263 lines
8.4 KiB
Python
263 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import sys
|
|
import io
|
|
import time
|
|
import json
|
|
from pathlib import Path
|
|
import hashlib
|
|
import mysql.connector
|
|
from mysql.connector import Error
|
|
|
|
# ====================================================================
|
|
# UTF-8 OUTPUT
|
|
# ====================================================================
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
|
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
|
|
|
# ====================================================================
|
|
# CONFIGURATION
|
|
# ====================================================================
|
|
|
|
BASE_DIR = Path(r"z:\Dropbox\!!!Days\Downloads Z230\Fio")
|
|
|
|
DB = {
|
|
"host": "192.168.1.76",
|
|
"port": 3307,
|
|
"user": "root",
|
|
"password": "Vlado9674+",
|
|
"database": "fio",
|
|
"charset": "utf8mb4",
|
|
}
|
|
|
|
BATCH_SIZE = 500
|
|
DEBUG_ON_ERROR = True
|
|
|
|
|
|
# ====================================================================
|
|
# HELPERS
|
|
# ====================================================================
|
|
|
|
def safe_col(t: dict, n: int):
|
|
key = f"column{n}"
|
|
val = t.get(key)
|
|
return val.get("value") if val else None
|
|
|
|
|
|
def clean_date(dt_str: str):
|
|
if not dt_str:
|
|
return None
|
|
return dt_str[:10]
|
|
|
|
|
|
def generate_fallback_id(account_id: str, t: dict) -> str:
|
|
raw_date = clean_date(safe_col(t, 0)) or ""
|
|
amount = str(safe_col(t, 1) or "")
|
|
protiucet = str(safe_col(t, 2) or "")
|
|
vs = str(safe_col(t, 5) or "")
|
|
src = f"{account_id}|{raw_date}|{amount}|{protiucet}|{vs}"
|
|
digest = hashlib.sha1(src.encode("utf-8")).hexdigest()
|
|
return digest[:20]
|
|
|
|
|
|
def load_json_file(p: Path):
|
|
try:
|
|
with open(p, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except Exception as e:
|
|
print(f" ❌ Nelze načíst JSON: {p} → {e}")
|
|
return None
|
|
|
|
|
|
# ====================================================================
|
|
# MAIN
|
|
# ====================================================================
|
|
|
|
def main():
|
|
start_all = time.time()
|
|
|
|
print("=== Fio HISTORICKÝ IMPORT (ze všech JSON na disku) ===\n", flush=True)
|
|
print(f"Hledám JSON soubory v: {BASE_DIR}", flush=True)
|
|
|
|
# Najdeme všechny JSONy ve všech podadresářích
|
|
all_json_paths = list(BASE_DIR.rglob("*.json"))
|
|
print(f"Nalezeno JSON souborů: {len(all_json_paths)}\n", flush=True)
|
|
|
|
if not all_json_paths:
|
|
print("Nenalezeny žádné JSON soubory. Konec.")
|
|
return
|
|
|
|
# DB připojení
|
|
try:
|
|
conn = mysql.connector.connect(
|
|
host=DB["host"],
|
|
port=DB["port"],
|
|
user=DB["user"],
|
|
password=DB["password"],
|
|
database=DB["database"],
|
|
charset=DB["charset"]
|
|
)
|
|
cur = conn.cursor()
|
|
except Error as e:
|
|
print(f"FATAL DB ERROR: {e}")
|
|
return
|
|
|
|
sql = """
|
|
INSERT INTO transactions
|
|
(
|
|
cislo_uctu, id_operace, transaction_date, amount, currency,
|
|
protiucet, kod_banky, nazev_protiuctu, nazev_banky, typ,
|
|
vs, ks, ss, uziv_identifikace, zprava_pro_prijemce,
|
|
provedl, id_pokynu, komentar, upr_objem_mena, api_bic, reference_platce
|
|
)
|
|
VALUES
|
|
(
|
|
%(cislo_uctu)s, %(id_operace)s, %(transaction_date)s, %(amount)s, %(currency)s,
|
|
%(protiucet)s, %(kod_banky)s, %(nazev_protiuctu)s, %(nazev_banky)s, %(typ)s,
|
|
%(vs)s, %(ks)s, %(ss)s, %(uziv_identifikace)s, %(zprava_pro_prijemce)s,
|
|
%(provedl)s, %(id_pokynu)s, %(komentar)s, %(upr_objem_mena)s, %(api_bic)s, %(reference_platce)s
|
|
)
|
|
ON DUPLICATE KEY UPDATE
|
|
transaction_date = VALUES(transaction_date),
|
|
amount = VALUES(amount),
|
|
currency = VALUES(currency),
|
|
protiucet = VALUES(protiucet),
|
|
kod_banky = VALUES(kod_banky),
|
|
nazev_protiuctu = VALUES(nazev_protiuctu),
|
|
nazev_banky = VALUES(nazev_banky),
|
|
typ = VALUES(typ),
|
|
vs = VALUES(vs),
|
|
ks = VALUES(ks),
|
|
ss = VALUES(ss),
|
|
uziv_identifikace = VALUES(uziv_identifikace),
|
|
zprava_pro_prijemce = VALUES(zprava_pro_prijemce),
|
|
provedl = VALUES(provedl),
|
|
id_pokynu = VALUES(id_pokynu),
|
|
komentar = VALUES(komentar),
|
|
upr_objem_mena = VALUES(upr_objem_mena),
|
|
api_bic = VALUES(api_bic),
|
|
reference_platce = VALUES(reference_platce)
|
|
"""
|
|
|
|
total_processed_files = 0
|
|
total_rows_inserted = 0
|
|
total_rows_skipped = 0
|
|
|
|
# ============================================
|
|
# PROCES JSON SOUBOR PO SOUBORU
|
|
# ============================================
|
|
for p in all_json_paths:
|
|
total_processed_files += 1
|
|
print(f"--- Soubor {total_processed_files}/{len(all_json_paths)}: {p}", flush=True)
|
|
|
|
data = load_json_file(p)
|
|
if not data:
|
|
continue
|
|
|
|
# JSON má strukturu jako při fetchnutí z API
|
|
account_info = data.get("accountStatement", {}).get("info", {})
|
|
account_id = account_info.get("accountId")
|
|
|
|
if not account_id:
|
|
print(" ⚠ Nelze zjistit cislo_uctu z JSON! Přeskakuji.")
|
|
continue
|
|
|
|
tlist = data.get("accountStatement", {}).get("transactionList", {}).get("transaction", [])
|
|
if isinstance(tlist, dict):
|
|
tlist = [tlist]
|
|
|
|
print(f" Počet transakcí: {len(tlist)}", flush=True)
|
|
|
|
if not tlist:
|
|
continue
|
|
|
|
rows = []
|
|
skipped_local = 0
|
|
|
|
# Převést transakce → DB řádky
|
|
for t in tlist:
|
|
id_operace_val = safe_col(t, 22)
|
|
if id_operace_val is None:
|
|
id_operace_val = generate_fallback_id(account_id, t)
|
|
|
|
transaction_date = clean_date(safe_col(t, 0))
|
|
if not transaction_date:
|
|
skipped_local += 1
|
|
continue
|
|
|
|
id_pokynu_val = safe_col(t, 19)
|
|
|
|
row = {
|
|
"cislo_uctu": account_id,
|
|
"id_operace": str(id_operace_val),
|
|
"transaction_date": transaction_date,
|
|
"amount": safe_col(t, 1),
|
|
"currency": safe_col(t, 14),
|
|
"protiucet": safe_col(t, 2),
|
|
"kod_banky": safe_col(t, 3),
|
|
"nazev_protiuctu": safe_col(t, 10),
|
|
"nazev_banky": safe_col(t, 12),
|
|
"api_bic": safe_col(t, 26),
|
|
"typ": safe_col(t, 8),
|
|
"provedl": safe_col(t, 9),
|
|
"vs": safe_col(t, 5),
|
|
"ks": safe_col(t, 4),
|
|
"ss": safe_col(t, 6),
|
|
"zprava_pro_prijemce": safe_col(t, 16),
|
|
"uziv_identifikace": safe_col(t, 7),
|
|
"komentar": safe_col(t, 25),
|
|
"upr_objem_mena": safe_col(t, 18),
|
|
"id_pokynu": str(id_pokynu_val) if id_pokynu_val else None,
|
|
"reference_platce": safe_col(t, 27),
|
|
}
|
|
rows.append(row)
|
|
|
|
total_rows_skipped += skipped_local
|
|
print(f" Přeskočeno transakcí bez data/PK: {skipped_local}")
|
|
|
|
# Batch insert
|
|
inserted = 0
|
|
|
|
for i in range(0, len(rows), BATCH_SIZE):
|
|
chunk = rows[i: i + BATCH_SIZE]
|
|
try:
|
|
cur.executemany(sql, chunk)
|
|
conn.commit()
|
|
inserted += len(chunk)
|
|
except Error as e:
|
|
print(f" ❌ Batch insert error: {e}")
|
|
conn.rollback()
|
|
|
|
if DEBUG_ON_ERROR:
|
|
print(" ► Per-row insert for debugging…")
|
|
for row in chunk:
|
|
try:
|
|
cur.execute(sql, row)
|
|
conn.commit()
|
|
inserted += 1
|
|
except Error as e_row:
|
|
conn.rollback()
|
|
print(f" ✗ Chyba transakce id_operace={row['id_operace']} → {e_row}")
|
|
|
|
total_rows_inserted += inserted
|
|
print(f" ✓ Zapsáno/aktualizováno: {inserted}")
|
|
|
|
# ======================
|
|
# ZÁVĚR
|
|
# ======================
|
|
cur.close()
|
|
conn.close()
|
|
|
|
elapsed = time.time() - start_all
|
|
print("\n===== HOTOVO =====", flush=True)
|
|
print(f"Souborů zpracováno: {total_processed_files}")
|
|
print(f"Transakcí zapsáno/aktualizováno: {total_rows_inserted}")
|
|
print(f"Transakcí přeskočeno: {total_rows_skipped}")
|
|
print(f"Celkový čas: {elapsed:.2f} s")
|
|
print("==================", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|