#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys import io import time import json from pathlib import Path import hashlib import mysql.connector from mysql.connector import Error # ==================================================================== # UTF-8 OUTPUT # ==================================================================== sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace') # ==================================================================== # CONFIGURATION # ==================================================================== BASE_DIR = Path(r"z:\Dropbox\!!!Days\Downloads Z230\Fio") DB = { "host": "192.168.1.76", "port": 3307, "user": "root", "password": "Vlado9674+", "database": "fio", "charset": "utf8mb4", } BATCH_SIZE = 500 DEBUG_ON_ERROR = True # ==================================================================== # HELPERS # ==================================================================== def safe_col(t: dict, n: int): key = f"column{n}" val = t.get(key) return val.get("value") if val else None def clean_date(dt_str: str): if not dt_str: return None return dt_str[:10] def generate_fallback_id(account_id: str, t: dict) -> str: raw_date = clean_date(safe_col(t, 0)) or "" amount = str(safe_col(t, 1) or "") protiucet = str(safe_col(t, 2) or "") vs = str(safe_col(t, 5) or "") src = f"{account_id}|{raw_date}|{amount}|{protiucet}|{vs}" digest = hashlib.sha1(src.encode("utf-8")).hexdigest() return digest[:20] def load_json_file(p: Path): try: with open(p, "r", encoding="utf-8") as f: return json.load(f) except Exception as e: print(f" ❌ Nelze načíst JSON: {p} → {e}") return None # ==================================================================== # MAIN # ==================================================================== def main(): start_all = time.time() print("=== Fio HISTORICKÝ IMPORT (ze všech JSON na disku) ===\n", flush=True) print(f"Hledám JSON soubory v: {BASE_DIR}", flush=True) # Najdeme všechny JSONy ve všech podadresářích all_json_paths = list(BASE_DIR.rglob("*.json")) print(f"Nalezeno JSON souborů: {len(all_json_paths)}\n", flush=True) if not all_json_paths: print("Nenalezeny žádné JSON soubory. Konec.") return # DB připojení try: conn = mysql.connector.connect( host=DB["host"], port=DB["port"], user=DB["user"], password=DB["password"], database=DB["database"], charset=DB["charset"] ) cur = conn.cursor() except Error as e: print(f"FATAL DB ERROR: {e}") return sql = """ INSERT INTO transactions ( cislo_uctu, id_operace, transaction_date, amount, currency, protiucet, kod_banky, nazev_protiuctu, nazev_banky, typ, vs, ks, ss, uziv_identifikace, zprava_pro_prijemce, provedl, id_pokynu, komentar, upr_objem_mena, api_bic, reference_platce ) VALUES ( %(cislo_uctu)s, %(id_operace)s, %(transaction_date)s, %(amount)s, %(currency)s, %(protiucet)s, %(kod_banky)s, %(nazev_protiuctu)s, %(nazev_banky)s, %(typ)s, %(vs)s, %(ks)s, %(ss)s, %(uziv_identifikace)s, %(zprava_pro_prijemce)s, %(provedl)s, %(id_pokynu)s, %(komentar)s, %(upr_objem_mena)s, %(api_bic)s, %(reference_platce)s ) ON DUPLICATE KEY UPDATE transaction_date = VALUES(transaction_date), amount = VALUES(amount), currency = VALUES(currency), protiucet = VALUES(protiucet), kod_banky = VALUES(kod_banky), nazev_protiuctu = VALUES(nazev_protiuctu), nazev_banky = VALUES(nazev_banky), typ = VALUES(typ), vs = VALUES(vs), ks = VALUES(ks), ss = VALUES(ss), uziv_identifikace = VALUES(uziv_identifikace), zprava_pro_prijemce = VALUES(zprava_pro_prijemce), provedl = VALUES(provedl), id_pokynu = VALUES(id_pokynu), komentar = VALUES(komentar), upr_objem_mena = VALUES(upr_objem_mena), api_bic = VALUES(api_bic), reference_platce = VALUES(reference_platce) """ total_processed_files = 0 total_rows_inserted = 0 total_rows_skipped = 0 # ============================================ # PROCES JSON SOUBOR PO SOUBORU # ============================================ for p in all_json_paths: total_processed_files += 1 print(f"--- Soubor {total_processed_files}/{len(all_json_paths)}: {p}", flush=True) data = load_json_file(p) if not data: continue # JSON má strukturu jako při fetchnutí z API account_info = data.get("accountStatement", {}).get("info", {}) account_id = account_info.get("accountId") if not account_id: print(" ⚠ Nelze zjistit cislo_uctu z JSON! Přeskakuji.") continue tlist = data.get("accountStatement", {}).get("transactionList", {}).get("transaction", []) if isinstance(tlist, dict): tlist = [tlist] print(f" Počet transakcí: {len(tlist)}", flush=True) if not tlist: continue rows = [] skipped_local = 0 # Převést transakce → DB řádky for t in tlist: id_operace_val = safe_col(t, 22) if id_operace_val is None: id_operace_val = generate_fallback_id(account_id, t) transaction_date = clean_date(safe_col(t, 0)) if not transaction_date: skipped_local += 1 continue id_pokynu_val = safe_col(t, 19) row = { "cislo_uctu": account_id, "id_operace": str(id_operace_val), "transaction_date": transaction_date, "amount": safe_col(t, 1), "currency": safe_col(t, 14), "protiucet": safe_col(t, 2), "kod_banky": safe_col(t, 3), "nazev_protiuctu": safe_col(t, 10), "nazev_banky": safe_col(t, 12), "api_bic": safe_col(t, 26), "typ": safe_col(t, 8), "provedl": safe_col(t, 9), "vs": safe_col(t, 5), "ks": safe_col(t, 4), "ss": safe_col(t, 6), "zprava_pro_prijemce": safe_col(t, 16), "uziv_identifikace": safe_col(t, 7), "komentar": safe_col(t, 25), "upr_objem_mena": safe_col(t, 18), "id_pokynu": str(id_pokynu_val) if id_pokynu_val else None, "reference_platce": safe_col(t, 27), } rows.append(row) total_rows_skipped += skipped_local print(f" Přeskočeno transakcí bez data/PK: {skipped_local}") # Batch insert inserted = 0 for i in range(0, len(rows), BATCH_SIZE): chunk = rows[i: i + BATCH_SIZE] try: cur.executemany(sql, chunk) conn.commit() inserted += len(chunk) except Error as e: print(f" ❌ Batch insert error: {e}") conn.rollback() if DEBUG_ON_ERROR: print(" ► Per-row insert for debugging…") for row in chunk: try: cur.execute(sql, row) conn.commit() inserted += 1 except Error as e_row: conn.rollback() print(f" ✗ Chyba transakce id_operace={row['id_operace']} → {e_row}") total_rows_inserted += inserted print(f" ✓ Zapsáno/aktualizováno: {inserted}") # ====================== # ZÁVĚR # ====================== cur.close() conn.close() elapsed = time.time() - start_all print("\n===== HOTOVO =====", flush=True) print(f"Souborů zpracováno: {total_processed_files}") print(f"Transakcí zapsáno/aktualizováno: {total_rows_inserted}") print(f"Transakcí přeskočeno: {total_rows_skipped}") print(f"Celkový čas: {elapsed:.2f} s") print("==================", flush=True) if __name__ == "__main__": main()