reporter
This commit is contained in:
262
#10 Download reports/40 import_fio_history.py
Normal file
262
#10 Download reports/40 import_fio_history.py
Normal file
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import io
|
||||
import time
|
||||
import json
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import mysql.connector
|
||||
from mysql.connector import Error
|
||||
|
||||
# ====================================================================
|
||||
# UTF-8 OUTPUT
|
||||
# ====================================================================
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
# ====================================================================
|
||||
# CONFIGURATION
|
||||
# ====================================================================
|
||||
|
||||
BASE_DIR = Path(r"z:\Dropbox\!!!Days\Downloads Z230\Fio")
|
||||
|
||||
DB = {
|
||||
"host": "192.168.1.76",
|
||||
"port": 3307,
|
||||
"user": "root",
|
||||
"password": "Vlado9674+",
|
||||
"database": "fio",
|
||||
"charset": "utf8mb4",
|
||||
}
|
||||
|
||||
BATCH_SIZE = 500
|
||||
DEBUG_ON_ERROR = True
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# HELPERS
|
||||
# ====================================================================
|
||||
|
||||
def safe_col(t: dict, n: int):
|
||||
key = f"column{n}"
|
||||
val = t.get(key)
|
||||
return val.get("value") if val else None
|
||||
|
||||
|
||||
def clean_date(dt_str: str):
|
||||
if not dt_str:
|
||||
return None
|
||||
return dt_str[:10]
|
||||
|
||||
|
||||
def generate_fallback_id(account_id: str, t: dict) -> str:
|
||||
raw_date = clean_date(safe_col(t, 0)) or ""
|
||||
amount = str(safe_col(t, 1) or "")
|
||||
protiucet = str(safe_col(t, 2) or "")
|
||||
vs = str(safe_col(t, 5) or "")
|
||||
src = f"{account_id}|{raw_date}|{amount}|{protiucet}|{vs}"
|
||||
digest = hashlib.sha1(src.encode("utf-8")).hexdigest()
|
||||
return digest[:20]
|
||||
|
||||
|
||||
def load_json_file(p: Path):
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f" ❌ Nelze načíst JSON: {p} → {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# MAIN
|
||||
# ====================================================================
|
||||
|
||||
def main():
|
||||
start_all = time.time()
|
||||
|
||||
print("=== Fio HISTORICKÝ IMPORT (ze všech JSON na disku) ===\n", flush=True)
|
||||
print(f"Hledám JSON soubory v: {BASE_DIR}", flush=True)
|
||||
|
||||
# Najdeme všechny JSONy ve všech podadresářích
|
||||
all_json_paths = list(BASE_DIR.rglob("*.json"))
|
||||
print(f"Nalezeno JSON souborů: {len(all_json_paths)}\n", flush=True)
|
||||
|
||||
if not all_json_paths:
|
||||
print("Nenalezeny žádné JSON soubory. Konec.")
|
||||
return
|
||||
|
||||
# DB připojení
|
||||
try:
|
||||
conn = mysql.connector.connect(
|
||||
host=DB["host"],
|
||||
port=DB["port"],
|
||||
user=DB["user"],
|
||||
password=DB["password"],
|
||||
database=DB["database"],
|
||||
charset=DB["charset"]
|
||||
)
|
||||
cur = conn.cursor()
|
||||
except Error as e:
|
||||
print(f"FATAL DB ERROR: {e}")
|
||||
return
|
||||
|
||||
sql = """
|
||||
INSERT INTO transactions
|
||||
(
|
||||
cislo_uctu, id_operace, transaction_date, amount, currency,
|
||||
protiucet, kod_banky, nazev_protiuctu, nazev_banky, typ,
|
||||
vs, ks, ss, uziv_identifikace, zprava_pro_prijemce,
|
||||
provedl, id_pokynu, komentar, upr_objem_mena, api_bic, reference_platce
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
%(cislo_uctu)s, %(id_operace)s, %(transaction_date)s, %(amount)s, %(currency)s,
|
||||
%(protiucet)s, %(kod_banky)s, %(nazev_protiuctu)s, %(nazev_banky)s, %(typ)s,
|
||||
%(vs)s, %(ks)s, %(ss)s, %(uziv_identifikace)s, %(zprava_pro_prijemce)s,
|
||||
%(provedl)s, %(id_pokynu)s, %(komentar)s, %(upr_objem_mena)s, %(api_bic)s, %(reference_platce)s
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
transaction_date = VALUES(transaction_date),
|
||||
amount = VALUES(amount),
|
||||
currency = VALUES(currency),
|
||||
protiucet = VALUES(protiucet),
|
||||
kod_banky = VALUES(kod_banky),
|
||||
nazev_protiuctu = VALUES(nazev_protiuctu),
|
||||
nazev_banky = VALUES(nazev_banky),
|
||||
typ = VALUES(typ),
|
||||
vs = VALUES(vs),
|
||||
ks = VALUES(ks),
|
||||
ss = VALUES(ss),
|
||||
uziv_identifikace = VALUES(uziv_identifikace),
|
||||
zprava_pro_prijemce = VALUES(zprava_pro_prijemce),
|
||||
provedl = VALUES(provedl),
|
||||
id_pokynu = VALUES(id_pokynu),
|
||||
komentar = VALUES(komentar),
|
||||
upr_objem_mena = VALUES(upr_objem_mena),
|
||||
api_bic = VALUES(api_bic),
|
||||
reference_platce = VALUES(reference_platce)
|
||||
"""
|
||||
|
||||
total_processed_files = 0
|
||||
total_rows_inserted = 0
|
||||
total_rows_skipped = 0
|
||||
|
||||
# ============================================
|
||||
# PROCES JSON SOUBOR PO SOUBORU
|
||||
# ============================================
|
||||
for p in all_json_paths:
|
||||
total_processed_files += 1
|
||||
print(f"--- Soubor {total_processed_files}/{len(all_json_paths)}: {p}", flush=True)
|
||||
|
||||
data = load_json_file(p)
|
||||
if not data:
|
||||
continue
|
||||
|
||||
# JSON má strukturu jako při fetchnutí z API
|
||||
account_info = data.get("accountStatement", {}).get("info", {})
|
||||
account_id = account_info.get("accountId")
|
||||
|
||||
if not account_id:
|
||||
print(" ⚠ Nelze zjistit cislo_uctu z JSON! Přeskakuji.")
|
||||
continue
|
||||
|
||||
tlist = data.get("accountStatement", {}).get("transactionList", {}).get("transaction", [])
|
||||
if isinstance(tlist, dict):
|
||||
tlist = [tlist]
|
||||
|
||||
print(f" Počet transakcí: {len(tlist)}", flush=True)
|
||||
|
||||
if not tlist:
|
||||
continue
|
||||
|
||||
rows = []
|
||||
skipped_local = 0
|
||||
|
||||
# Převést transakce → DB řádky
|
||||
for t in tlist:
|
||||
id_operace_val = safe_col(t, 22)
|
||||
if id_operace_val is None:
|
||||
id_operace_val = generate_fallback_id(account_id, t)
|
||||
|
||||
transaction_date = clean_date(safe_col(t, 0))
|
||||
if not transaction_date:
|
||||
skipped_local += 1
|
||||
continue
|
||||
|
||||
id_pokynu_val = safe_col(t, 19)
|
||||
|
||||
row = {
|
||||
"cislo_uctu": account_id,
|
||||
"id_operace": str(id_operace_val),
|
||||
"transaction_date": transaction_date,
|
||||
"amount": safe_col(t, 1),
|
||||
"currency": safe_col(t, 14),
|
||||
"protiucet": safe_col(t, 2),
|
||||
"kod_banky": safe_col(t, 3),
|
||||
"nazev_protiuctu": safe_col(t, 10),
|
||||
"nazev_banky": safe_col(t, 12),
|
||||
"api_bic": safe_col(t, 26),
|
||||
"typ": safe_col(t, 8),
|
||||
"provedl": safe_col(t, 9),
|
||||
"vs": safe_col(t, 5),
|
||||
"ks": safe_col(t, 4),
|
||||
"ss": safe_col(t, 6),
|
||||
"zprava_pro_prijemce": safe_col(t, 16),
|
||||
"uziv_identifikace": safe_col(t, 7),
|
||||
"komentar": safe_col(t, 25),
|
||||
"upr_objem_mena": safe_col(t, 18),
|
||||
"id_pokynu": str(id_pokynu_val) if id_pokynu_val else None,
|
||||
"reference_platce": safe_col(t, 27),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
total_rows_skipped += skipped_local
|
||||
print(f" Přeskočeno transakcí bez data/PK: {skipped_local}")
|
||||
|
||||
# Batch insert
|
||||
inserted = 0
|
||||
|
||||
for i in range(0, len(rows), BATCH_SIZE):
|
||||
chunk = rows[i: i + BATCH_SIZE]
|
||||
try:
|
||||
cur.executemany(sql, chunk)
|
||||
conn.commit()
|
||||
inserted += len(chunk)
|
||||
except Error as e:
|
||||
print(f" ❌ Batch insert error: {e}")
|
||||
conn.rollback()
|
||||
|
||||
if DEBUG_ON_ERROR:
|
||||
print(" ► Per-row insert for debugging…")
|
||||
for row in chunk:
|
||||
try:
|
||||
cur.execute(sql, row)
|
||||
conn.commit()
|
||||
inserted += 1
|
||||
except Error as e_row:
|
||||
conn.rollback()
|
||||
print(f" ✗ Chyba transakce id_operace={row['id_operace']} → {e_row}")
|
||||
|
||||
total_rows_inserted += inserted
|
||||
print(f" ✓ Zapsáno/aktualizováno: {inserted}")
|
||||
|
||||
# ======================
|
||||
# ZÁVĚR
|
||||
# ======================
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
elapsed = time.time() - start_all
|
||||
print("\n===== HOTOVO =====", flush=True)
|
||||
print(f"Souborů zpracováno: {total_processed_files}")
|
||||
print(f"Transakcí zapsáno/aktualizováno: {total_rows_inserted}")
|
||||
print(f"Transakcí přeskočeno: {total_rows_skipped}")
|
||||
print(f"Celkový čas: {elapsed:.2f} s")
|
||||
print("==================", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user