notebook
This commit is contained in:
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
FIO JSON ARCHIVE → MYSQL IMPORTER (ONE-TIME REIMPORT)
|
||||
=====================================================
|
||||
|
||||
CO TOHLE DĚLÁ
|
||||
-------------
|
||||
Tento skript je určený pro JEDNORÁZOVÝ import historických transakcí,
|
||||
které už máš uložené jako RAW JSON soubory na disku (archiv).
|
||||
|
||||
✅ NEVOLÁ Fio API (žádné stahování)
|
||||
✅ Rekurzivně projde celý adresář:
|
||||
U:\Dropbox\!!!Days\Downloads Z230\Fio\
|
||||
a najde všechny *.json soubory (typicky ve složkách podle účtu)
|
||||
|
||||
✅ Z každého JSONu:
|
||||
- vytáhne accountId (ID účtu z JSON hlavičky)
|
||||
- vytáhne transaction list
|
||||
- mapuje Fio "columnN" hodnoty na sloupce tabulky v MySQL
|
||||
|
||||
✅ Poté vkládá data do MySQL tabulky:
|
||||
fio.transactions
|
||||
|
||||
IDEMPOTENTNÍ CHOVÁNÍ (DŮLEŽITÉ)
|
||||
------------------------------
|
||||
Tabulka má primární klíč:
|
||||
PRIMARY KEY (cislo_uctu, id_operace)
|
||||
|
||||
Proto je INSERT napsaný jako:
|
||||
INSERT ... ON DUPLICATE KEY UPDATE ...
|
||||
|
||||
To znamená:
|
||||
- pokud (cislo_uctu, id_operace) ještě v DB není → vloží nový řádek
|
||||
- pokud už tam je → neudělá duplicitu, ale může aktualizovat vybraná pole
|
||||
|
||||
Takže skript můžeš spustit opakovaně bez rizika duplikací.
|
||||
|
||||
CO MUSÍ EXISTOVAT
|
||||
-----------------
|
||||
1) MySQL DB: fio
|
||||
2) Tabulka: fio.transactions (podle DDL co jsme skládali z ibd2sdi SDI)
|
||||
|
||||
VÝKON
|
||||
-----
|
||||
- Používá executemany() a dávky (BATCH_SIZE), aby to bylo rychlé
|
||||
- Commituje po dávkách
|
||||
|
||||
POZNÁMKA K CHYBÁM
|
||||
-----------------
|
||||
- Pokud narazí na rozbitý JSON nebo neočekávanou strukturu, soubor přeskočí
|
||||
a pokračuje dál (aby jednorázový import doběhl celý).
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pymysql
|
||||
|
||||
|
||||
# ======================================================
|
||||
# CONFIG
|
||||
# ======================================================
|
||||
|
||||
# Základní adresář s archivními JSON soubory z Fio API
|
||||
# Očekáváme strukturu například:
|
||||
# U:\Dropbox\!!!Days\Downloads Z230\Fio\2100046291_2010\2025-01-01_to_2025-03-31.json
|
||||
JSON_BASE_DIR = Path(r"u:\Dropbox\!!!Days\Downloads Z230\Fio")
|
||||
|
||||
# Připojení do MySQL (pozor: port 3307 dle tvého prostředí)
|
||||
DB = {
|
||||
"host": "192.168.1.76",
|
||||
"port": 3306,
|
||||
"user": "root",
|
||||
"password": "Vlado9674+",
|
||||
"database": "fio",
|
||||
"charset": "utf8mb4",
|
||||
}
|
||||
|
||||
# Kolik řádků posílat do DB v jedné dávce (performance)
|
||||
BATCH_SIZE = 500
|
||||
|
||||
|
||||
# ======================================================
|
||||
# HELPERS
|
||||
# ======================================================
|
||||
|
||||
def safe_col(tx: dict, n: int):
|
||||
"""
|
||||
Bezpečné čtení hodnoty Fio JSON "columnN".
|
||||
|
||||
Fio JSON pro jednu transakci vypadá typicky takto:
|
||||
{
|
||||
"column0": { "name": "Datum", "value": "2025-02-14+0100" },
|
||||
"column1": { "name": "Objem", "value": 123.45 },
|
||||
...
|
||||
}
|
||||
|
||||
Ale:
|
||||
- některé columnN vůbec nemusí existovat
|
||||
- nebo mohou být None
|
||||
|
||||
Tato funkce tedy:
|
||||
vrátí tx["columnN"]["value"]
|
||||
nebo:
|
||||
None (pokud tam columnN není)
|
||||
"""
|
||||
obj = tx.get(f"column{n}")
|
||||
if not obj:
|
||||
return None
|
||||
return obj.get("value")
|
||||
|
||||
|
||||
def clean_date(dt_str):
|
||||
"""
|
||||
Fio vrací datum často v podobě:
|
||||
"YYYY-MM-DD+0100"
|
||||
MySQL DATE chceme jen:
|
||||
"YYYY-MM-DD"
|
||||
|
||||
Pokud je dt_str None/empty, vrátí None.
|
||||
"""
|
||||
if not dt_str:
|
||||
return None
|
||||
return str(dt_str)[:10]
|
||||
|
||||
|
||||
def as_list(maybe_list_or_dict):
|
||||
"""
|
||||
Fio někdy vrací:
|
||||
- seznam transakcí (list)
|
||||
jindy:
|
||||
- jedinou transakci jako dict
|
||||
|
||||
Tady zaručíme, že vždy pracujeme se seznamem.
|
||||
"""
|
||||
if maybe_list_or_dict is None:
|
||||
return []
|
||||
if isinstance(maybe_list_or_dict, dict):
|
||||
return [maybe_list_or_dict]
|
||||
return list(maybe_list_or_dict)
|
||||
|
||||
|
||||
# ======================================================
|
||||
# SQL (MAPOVÁNÍ NA TVOU TABULKU)
|
||||
# ======================================================
|
||||
|
||||
# Vkládáme do tabulky fio.transactions (sloupce dle DDL z SDI).
|
||||
# Pozn.: sloupec stazeno_kdy neuvádíme -> vyplní se automaticky DEFAULT CURRENT_TIMESTAMP
|
||||
SQL_INSERT = """
|
||||
INSERT INTO transactions
|
||||
(
|
||||
id_operace,
|
||||
cislo_uctu,
|
||||
transaction_date,
|
||||
amount,
|
||||
currency,
|
||||
protiucet,
|
||||
kod_banky,
|
||||
nazev_protiuctu,
|
||||
nazev_banky,
|
||||
typ,
|
||||
vs,
|
||||
ks,
|
||||
ss,
|
||||
zprava_pro_prijemce,
|
||||
uziv_identifikace,
|
||||
provedl,
|
||||
id_pokynu,
|
||||
komentar,
|
||||
upr_objem_mena,
|
||||
api_bic,
|
||||
reference_platce
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
%(id_operace)s,
|
||||
%(cislo_uctu)s,
|
||||
%(transaction_date)s,
|
||||
%(amount)s,
|
||||
%(currency)s,
|
||||
%(protiucet)s,
|
||||
%(kod_banky)s,
|
||||
%(nazev_protiuctu)s,
|
||||
%(nazev_banky)s,
|
||||
%(typ)s,
|
||||
%(vs)s,
|
||||
%(ks)s,
|
||||
%(ss)s,
|
||||
%(zprava_pro_prijemce)s,
|
||||
%(uziv_identifikace)s,
|
||||
%(provedl)s,
|
||||
%(id_pokynu)s,
|
||||
%(komentar)s,
|
||||
%(upr_objem_mena)s,
|
||||
%(api_bic)s,
|
||||
%(reference_platce)s
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
-- když už transakce existuje, aktualizujeme vybraná pole
|
||||
transaction_date = VALUES(transaction_date),
|
||||
amount = VALUES(amount),
|
||||
currency = VALUES(currency),
|
||||
protiucet = VALUES(protiucet),
|
||||
kod_banky = VALUES(kod_banky),
|
||||
nazev_protiuctu = VALUES(nazev_protiuctu),
|
||||
nazev_banky = VALUES(nazev_banky),
|
||||
typ = VALUES(typ),
|
||||
vs = VALUES(vs),
|
||||
ks = VALUES(ks),
|
||||
ss = VALUES(ss),
|
||||
zprava_pro_prijemce= VALUES(zprava_pro_prijemce),
|
||||
uziv_identifikace = VALUES(uziv_identifikace),
|
||||
provedl = VALUES(provedl),
|
||||
id_pokynu = VALUES(id_pokynu),
|
||||
komentar = VALUES(komentar),
|
||||
upr_objem_mena = VALUES(upr_objem_mena),
|
||||
api_bic = VALUES(api_bic),
|
||||
reference_platce = VALUES(reference_platce)
|
||||
"""
|
||||
|
||||
|
||||
# ======================================================
|
||||
# MAIN
|
||||
# ======================================================
|
||||
|
||||
def main():
|
||||
"""
|
||||
Hlavní běh:
|
||||
|
||||
1) Najde všechny *.json soubory pod JSON_BASE_DIR (rekurzivně)
|
||||
2) Každý JSON načte a zkusí vytáhnout:
|
||||
- accountId
|
||||
- transactionList.transaction
|
||||
3) Pro každou transakci složí dict hodnot podle SQL_INSERT
|
||||
4) Vkládá do DB po dávkách (BATCH_SIZE) přes executemany()
|
||||
"""
|
||||
|
||||
print("=== FIO JSON ARCHIVE → MYSQL IMPORTER ===")
|
||||
print(f"Base dir: {JSON_BASE_DIR}")
|
||||
|
||||
# 1) Najdi všechny JSON soubory rekurzivně
|
||||
json_files = sorted(JSON_BASE_DIR.rglob("*.json"))
|
||||
print(f"Nalezeno JSON souborů: {len(json_files)}")
|
||||
|
||||
if not json_files:
|
||||
print("Nic k importu. Končím.")
|
||||
return
|
||||
|
||||
# 2) Připoj se k DB
|
||||
conn = pymysql.connect(**DB)
|
||||
cur = conn.cursor()
|
||||
|
||||
total_rows_seen = 0 # kolik transakcí jsme celkem naparsovali (všech souborů)
|
||||
total_rows_sent = 0 # kolik jsme celkem poslali do DB (insert/update attempt)
|
||||
files_ok = 0
|
||||
files_skipped = 0
|
||||
|
||||
try:
|
||||
# 3) Zpracuj každý JSON soubor
|
||||
for idx, jf in enumerate(json_files, start=1):
|
||||
print(f"\n[{idx}/{len(json_files)}] Soubor: {jf}")
|
||||
|
||||
# 3a) načtení JSON souboru
|
||||
try:
|
||||
data = json.loads(jf.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
print(f" ❌ Nejde načíst JSON ({e}) → přeskočeno")
|
||||
files_skipped += 1
|
||||
continue
|
||||
|
||||
# 3b) vytažení listu transakcí a accountId
|
||||
try:
|
||||
fio_acc_id = data["accountStatement"]["info"]["accountId"]
|
||||
t_raw = data["accountStatement"]["transactionList"].get("transaction", [])
|
||||
except Exception:
|
||||
print(" ❌ Neočekávaná struktura JSON (chybí accountStatement/info/transactionList) → přeskočeno")
|
||||
files_skipped += 1
|
||||
continue
|
||||
|
||||
tlist = as_list(t_raw)
|
||||
print(f" Účet: {fio_acc_id} | transakcí v souboru: {len(tlist)}")
|
||||
|
||||
if not tlist:
|
||||
print(" (prázdný seznam transakcí) → OK, nic nevkládám")
|
||||
files_ok += 1
|
||||
continue
|
||||
|
||||
# 3c) naparsuj všechny transakce v souboru do listu rows
|
||||
rows = []
|
||||
for tx in tlist:
|
||||
# mapování columnN -> DB sloupce (dle tvé tabulky)
|
||||
row = {
|
||||
"id_operace": safe_col(tx, 22),
|
||||
"cislo_uctu": fio_acc_id,
|
||||
|
||||
"transaction_date": clean_date(safe_col(tx, 0)),
|
||||
"amount": safe_col(tx, 1),
|
||||
"currency": safe_col(tx, 14),
|
||||
|
||||
"protiucet": safe_col(tx, 2),
|
||||
"kod_banky": safe_col(tx, 3),
|
||||
|
||||
"nazev_protiuctu": safe_col(tx, 10),
|
||||
"nazev_banky": safe_col(tx, 15),
|
||||
|
||||
"typ": safe_col(tx, 8),
|
||||
|
||||
"vs": safe_col(tx, 5),
|
||||
"ks": safe_col(tx, 4),
|
||||
"ss": safe_col(tx, 6),
|
||||
|
||||
"zprava_pro_prijemce": safe_col(tx, 16),
|
||||
|
||||
# v SDI tabulce existuje sloupec uziv_identifikace (TEXT)
|
||||
# ve starém importeru se brala hodnota z column12
|
||||
"uziv_identifikace": safe_col(tx, 12),
|
||||
|
||||
# Pozor: ve tvé tabulce je "provedl" (VARCHAR(50)).
|
||||
# Starý importer bral "provedl" typicky z column9.
|
||||
"provedl": safe_col(tx, 9),
|
||||
|
||||
"id_pokynu": safe_col(tx, 19),
|
||||
|
||||
# textová poznámka / komentář (starý importer používal column25 jako "poznamka")
|
||||
"komentar": safe_col(tx, 25),
|
||||
|
||||
# upřesnění objemu/měny (column20 / column21)
|
||||
# v tabulce máme jen jeden sloupec upr_objem_mena (varchar 255)
|
||||
# a zvlášť "currency" už je column14, takže sem dáváme column20 (Upřesnění)
|
||||
"upr_objem_mena": safe_col(tx, 20),
|
||||
|
||||
# BIC a reference plátce (dle komentářů v SDI)
|
||||
"api_bic": safe_col(tx, 26),
|
||||
"reference_platce": safe_col(tx, 27),
|
||||
}
|
||||
|
||||
# Minimální sanity check:
|
||||
# PRIMARY KEY je (cislo_uctu, id_operace),
|
||||
# takže pokud id_operace chybí, nebude to unikátní transakce.
|
||||
# Takové řádky přeskočíme (jinak by to dělalo problémy).
|
||||
if not row["id_operace"]:
|
||||
continue
|
||||
|
||||
rows.append(row)
|
||||
|
||||
total_rows_seen += len(tlist)
|
||||
print(f" Naparsováno validních řádků (s id_operace): {len(rows)}")
|
||||
|
||||
if not rows:
|
||||
print(" (po filtraci nic nezbylo) → OK")
|
||||
files_ok += 1
|
||||
continue
|
||||
|
||||
# 3d) insert do DB po dávkách
|
||||
inserted_attempt = 0
|
||||
for i in range(0, len(rows), BATCH_SIZE):
|
||||
chunk = rows[i:i + BATCH_SIZE]
|
||||
cur.executemany(SQL_INSERT, chunk)
|
||||
conn.commit()
|
||||
inserted_attempt += len(chunk)
|
||||
|
||||
total_rows_sent += inserted_attempt
|
||||
files_ok += 1
|
||||
print(f" ✓ Odesláno do DB (insert/update attempt): {inserted_attempt}")
|
||||
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
print("\n=== HOTOVO ===")
|
||||
print(f"Souborů OK: {files_ok}")
|
||||
print(f"Souborů přeskočeno: {files_skipped}")
|
||||
print(f"Celkem transakcí nalezeno v JSON (raw): {total_rows_seen}")
|
||||
print(f"Celkem posláno do DB (insert/update attempt): {total_rows_sent}")
|
||||
print("Pozn.: 'posláno do DB' neznamená čisté INSERTy — část mohla být UPDATE (ON DUPLICATE KEY).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import io
|
||||
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import pymysql
|
||||
|
||||
# =========================================
|
||||
# CONFIGURATION
|
||||
# =========================================
|
||||
|
||||
ACCOUNTS_FILE = r"../accounts.json"
|
||||
JSON_BASE_DIR = r"u:\Dropbox\!!!Days\Downloads Z230\Fio"
|
||||
|
||||
DB = {
|
||||
"host": "192.168.1.76",
|
||||
"port": 3306,
|
||||
"user": "root",
|
||||
"password": "Vlado9674+",
|
||||
"database": "fio",
|
||||
"charset": "utf8mb4",
|
||||
}
|
||||
|
||||
BATCH_SIZE = 500
|
||||
DAYS_BACK = 90
|
||||
|
||||
# =========================================
|
||||
# HELPERS
|
||||
# =========================================
|
||||
|
||||
def load_accounts(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
accounts = json.load(f)
|
||||
|
||||
for acc in accounts:
|
||||
for key in ("name", "account_number", "token"):
|
||||
if key not in acc:
|
||||
raise ValueError(f"Missing '{key}' in account config: {acc}")
|
||||
|
||||
return accounts
|
||||
|
||||
|
||||
def fio_url_for_period(token, d_from, d_to):
|
||||
return f"https://fioapi.fio.cz/v1/rest/periods/{token}/{d_from:%Y-%m-%d}/{d_to:%Y-%m-%d}/transactions.json"
|
||||
|
||||
|
||||
def fetch_fio_json(token, d_from, d_to):
|
||||
url = fio_url_for_period(token, d_from, d_to)
|
||||
resp = requests.get(url, timeout=30)
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f"❌ HTTP {resp.status_code} from Fio")
|
||||
return None
|
||||
|
||||
return resp.json()
|
||||
|
||||
|
||||
def safe_col(t, n):
|
||||
val = t.get(f"column{n}")
|
||||
return None if not val else val.get("value")
|
||||
|
||||
|
||||
def clean_date(dt_str):
|
||||
return None if not dt_str else dt_str[:10]
|
||||
|
||||
|
||||
def ensure_dir(path):
|
||||
Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def save_json_for_account(base_dir, account_cfg, data, d_from, d_to):
|
||||
acc_folder = account_cfg["account_number"].replace("/", "_")
|
||||
out_dir = Path(base_dir) / acc_folder
|
||||
ensure_dir(out_dir)
|
||||
|
||||
filename = f"{d_from:%Y-%m-%d}_to_{d_to:%Y-%m-%d}.json"
|
||||
out_path = out_dir / filename
|
||||
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return out_path
|
||||
|
||||
|
||||
# =========================================
|
||||
# MAIN
|
||||
# =========================================
|
||||
|
||||
def main():
|
||||
|
||||
today = date.today()
|
||||
d_from = today - timedelta(days=DAYS_BACK)
|
||||
d_to = today
|
||||
|
||||
print(f"=== Fio import {d_from} → {d_to} ===")
|
||||
|
||||
accounts = load_accounts(ACCOUNTS_FILE)
|
||||
|
||||
conn = pymysql.connect(**DB)
|
||||
cur = conn.cursor()
|
||||
|
||||
sql = """
|
||||
INSERT INTO transactions
|
||||
(
|
||||
id_operace,
|
||||
cislo_uctu,
|
||||
transaction_date,
|
||||
amount,
|
||||
currency,
|
||||
protiucet,
|
||||
kod_banky,
|
||||
nazev_protiuctu,
|
||||
nazev_banky,
|
||||
typ,
|
||||
vs,
|
||||
ks,
|
||||
ss,
|
||||
zprava_pro_prijemce,
|
||||
uziv_identifikace,
|
||||
provedl,
|
||||
id_pokynu,
|
||||
komentar,
|
||||
upr_objem_mena,
|
||||
api_bic,
|
||||
reference_platce
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
%(id_operace)s,
|
||||
%(cislo_uctu)s,
|
||||
%(transaction_date)s,
|
||||
%(amount)s,
|
||||
%(currency)s,
|
||||
%(protiucet)s,
|
||||
%(kod_banky)s,
|
||||
%(nazev_protiuctu)s,
|
||||
%(nazev_banky)s,
|
||||
%(typ)s,
|
||||
%(vs)s,
|
||||
%(ks)s,
|
||||
%(ss)s,
|
||||
%(zprava)s,
|
||||
%(uziv_identifikace)s,
|
||||
%(provedl)s,
|
||||
%(id_pokynu)s,
|
||||
%(komentar)s,
|
||||
%(upr_objem_mena)s,
|
||||
%(api_bic)s,
|
||||
%(reference_platce)s
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
transaction_date = VALUES(transaction_date),
|
||||
amount = VALUES(amount),
|
||||
currency = VALUES(currency),
|
||||
protiucet = VALUES(protiucet),
|
||||
kod_banky = VALUES(kod_banky),
|
||||
nazev_protiuctu = VALUES(nazev_protiuctu),
|
||||
nazev_banky = VALUES(nazev_banky),
|
||||
typ = VALUES(typ),
|
||||
vs = VALUES(vs),
|
||||
ks = VALUES(ks),
|
||||
ss = VALUES(ss),
|
||||
zprava_pro_prijemce = VALUES(zprava_pro_prijemce),
|
||||
uziv_identifikace = VALUES(uziv_identifikace),
|
||||
provedl = VALUES(provedl),
|
||||
id_pokynu = VALUES(id_pokynu),
|
||||
komentar = VALUES(komentar),
|
||||
upr_objem_mena = VALUES(upr_objem_mena),
|
||||
api_bic = VALUES(api_bic),
|
||||
reference_platce = VALUES(reference_platce)
|
||||
"""
|
||||
|
||||
total_inserted = 0
|
||||
|
||||
for acc in accounts:
|
||||
|
||||
print(f"\n--- {acc['name']} ---")
|
||||
|
||||
data = fetch_fio_json(acc["token"], d_from, d_to)
|
||||
if not data:
|
||||
continue
|
||||
|
||||
save_json_for_account(JSON_BASE_DIR, acc, data, d_from, d_to)
|
||||
|
||||
tlist = data["accountStatement"]["transactionList"].get("transaction", [])
|
||||
if isinstance(tlist, dict):
|
||||
tlist = [tlist]
|
||||
|
||||
fio_acc_id = data["accountStatement"]["info"]["accountId"]
|
||||
|
||||
rows = []
|
||||
|
||||
for t in tlist:
|
||||
|
||||
row = {
|
||||
"id_operace": safe_col(t, 22),
|
||||
"cislo_uctu": fio_acc_id,
|
||||
|
||||
"transaction_date": clean_date(safe_col(t, 0)),
|
||||
"amount": safe_col(t, 1),
|
||||
"currency": safe_col(t, 14),
|
||||
|
||||
"protiucet": safe_col(t, 2),
|
||||
"kod_banky": safe_col(t, 3),
|
||||
|
||||
"nazev_protiuctu": safe_col(t, 10),
|
||||
"nazev_banky": safe_col(t, 15),
|
||||
|
||||
"typ": safe_col(t, 8),
|
||||
|
||||
"vs": safe_col(t, 5),
|
||||
"ks": safe_col(t, 4),
|
||||
"ss": safe_col(t, 6),
|
||||
|
||||
"zprava": safe_col(t, 16),
|
||||
"uziv_identifikace": safe_col(t, 12),
|
||||
"provedl": safe_col(t, 12),
|
||||
|
||||
"id_pokynu": safe_col(t, 19),
|
||||
"komentar": safe_col(t, 25),
|
||||
"upr_objem_mena": safe_col(t, 20),
|
||||
|
||||
"api_bic": safe_col(t, 26),
|
||||
"reference_platce": safe_col(t, 27),
|
||||
}
|
||||
|
||||
if row["id_operace"]:
|
||||
rows.append(row)
|
||||
|
||||
inserted = 0
|
||||
|
||||
for i in range(0, len(rows), BATCH_SIZE):
|
||||
chunk = rows[i:i+BATCH_SIZE]
|
||||
cur.executemany(sql, chunk)
|
||||
conn.commit()
|
||||
inserted += len(chunk)
|
||||
|
||||
print(f"✓ Insert/Update: {inserted}")
|
||||
total_inserted += inserted
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
print(f"\n=== DONE → {total_inserted} rows processed ===")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -41,7 +41,7 @@ import json
|
||||
|
||||
# MySQL server parameters
|
||||
DB_HOST = "192.168.1.76"
|
||||
DB_PORT = 3307
|
||||
DB_PORT = 3306
|
||||
DB_USER = "root"
|
||||
DB_PASS = "Vlado9674+"
|
||||
DB_NAME = "fio"
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import io
|
||||
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
import mysql.connector
|
||||
from mysql.connector import Error
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
|
||||
from datetime import datetime
|
||||
import os
|
||||
import glob
|
||||
import json
|
||||
|
||||
# ======================================================
|
||||
# CONFIGURATION
|
||||
# ======================================================
|
||||
|
||||
DB_HOST = "192.168.1.76"
|
||||
DB_PORT = 3306
|
||||
DB_USER = "root"
|
||||
DB_PASS = "Vlado9674+"
|
||||
DB_NAME = "fio"
|
||||
|
||||
OUTPUT_DIR = r"u:\Dropbox\!!!Days\Downloads Z230"
|
||||
ACCOUNTS_JSON = r"accounts.json"
|
||||
|
||||
TEXT_COLUMNS = ["cislo_uctu", "protiucet", "kod_banky", "ks", "vs", "ss"]
|
||||
|
||||
# ======================================================
|
||||
# SQL VIEW COMPATIBILITY LAYER
|
||||
# ======================================================
|
||||
|
||||
SELECT_LEGACY = """
|
||||
SELECT
|
||||
transaction_date AS datum,
|
||||
amount AS objem,
|
||||
currency AS mena,
|
||||
cislo_uctu,
|
||||
protiucet,
|
||||
kod_banky,
|
||||
ks,
|
||||
vs,
|
||||
ss,
|
||||
zprava_pro_prijemce,
|
||||
komentar AS poznamka,
|
||||
id_operace,
|
||||
id_pokynu,
|
||||
nazev_banky,
|
||||
nazev_protiuctu,
|
||||
typ,
|
||||
upr_objem_mena AS upresneni_objem,
|
||||
NULL AS upresneni_mena,
|
||||
provedl AS zadal,
|
||||
api_bic,
|
||||
reference_platce,
|
||||
stazeno_kdy
|
||||
FROM transactions
|
||||
"""
|
||||
|
||||
# ======================================================
|
||||
# REMOVE OLD EXPORT FILES
|
||||
# ======================================================
|
||||
|
||||
def cleanup_old_exports():
|
||||
|
||||
patterns = [
|
||||
os.path.join(OUTPUT_DIR, "*FIO*transaction*.xlsx"),
|
||||
os.path.join(OUTPUT_DIR, "*FIO*transactions*.xlsx"),
|
||||
os.path.join(OUTPUT_DIR, "*FIO_transactions*.xlsx"),
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
for file in glob.glob(pattern):
|
||||
try:
|
||||
os.remove(file)
|
||||
print(f"🗑 Deleted old export: {file}")
|
||||
except:
|
||||
pass
|
||||
|
||||
# ======================================================
|
||||
# FORMAT SHEET
|
||||
# ======================================================
|
||||
|
||||
def format_sheet(ws, rows, headers):
|
||||
|
||||
for col_idx in range(1, len(headers) + 1):
|
||||
cell = ws.cell(row=1, column=col_idx)
|
||||
cell.font = Font(bold=True)
|
||||
cell.fill = PatternFill(start_color="FFFF00", fill_type="solid")
|
||||
|
||||
for row in rows:
|
||||
excel_row = []
|
||||
for h in headers:
|
||||
val = row[h]
|
||||
if h in TEXT_COLUMNS and val is not None:
|
||||
excel_row.append(f'="{val}"')
|
||||
else:
|
||||
excel_row.append(val)
|
||||
|
||||
ws.append(excel_row)
|
||||
|
||||
fill_red = PatternFill(start_color="FFFFDDDD", fill_type="solid")
|
||||
fill_green = PatternFill(start_color="FFEEFFEE", fill_type="solid")
|
||||
|
||||
objem_col_index = headers.index("objem") + 1
|
||||
|
||||
for row_idx in range(2, len(rows) + 2):
|
||||
try:
|
||||
value = float(ws.cell(row=row_idx, column=objem_col_index).value)
|
||||
except:
|
||||
value = 0
|
||||
|
||||
fill = fill_red if value < 0 else fill_green
|
||||
|
||||
for col_idx in range(1, len(headers) + 1):
|
||||
ws.cell(row=row_idx, column=col_idx).fill = fill
|
||||
|
||||
fixed_widths = [
|
||||
10.29, 10.29, 10.71, 13.29, 13.29, 7.29, 15.14, 12.29,
|
||||
34.43, 49.29, 13.57, 11.29, 11.29, 8, 30.57, 28.29,
|
||||
11, 28.29, 15.29, 14.29, 20
|
||||
]
|
||||
|
||||
for i, width in enumerate(fixed_widths, start=1):
|
||||
ws.column_dimensions[chr(64 + i)].width = width
|
||||
|
||||
thin = Side(border_style="thin", color="000000")
|
||||
border = Border(left=thin, right=thin, top=thin, bottom=thin)
|
||||
align_center = Alignment(horizontal="center")
|
||||
|
||||
total_rows = len(rows) + 1
|
||||
total_cols = len(headers)
|
||||
|
||||
for row_idx in range(1, total_rows + 1):
|
||||
for col_idx in range(1, total_cols + 1):
|
||||
cell = ws.cell(row=row_idx, column=col_idx)
|
||||
cell.border = border
|
||||
|
||||
align_left = Alignment(horizontal="left")
|
||||
|
||||
if row_idx > 1 and col_idx == 10:
|
||||
cell.alignment = align_left
|
||||
elif col_idx <= 9:
|
||||
cell.alignment = align_center
|
||||
|
||||
ws.freeze_panes = "A2"
|
||||
ws.auto_filter.ref = ws.dimensions
|
||||
|
||||
# ======================================================
|
||||
# EXPORT
|
||||
# ======================================================
|
||||
|
||||
def export_fio():
|
||||
|
||||
print("Connecting to MySQL...")
|
||||
|
||||
try:
|
||||
conn = mysql.connector.connect(
|
||||
host=DB_HOST,
|
||||
port=DB_PORT,
|
||||
user=DB_USER,
|
||||
password=DB_PASS,
|
||||
database=DB_NAME
|
||||
)
|
||||
except Error as e:
|
||||
print("❌ Failed to connect:", e)
|
||||
return
|
||||
|
||||
cur = conn.cursor(dictionary=True)
|
||||
|
||||
with open(ACCOUNTS_JSON, "r", encoding="utf-8") as f:
|
||||
accounts = json.load(f)
|
||||
|
||||
preferred_order = [
|
||||
"CZK rodina",
|
||||
"CZK ordinace",
|
||||
"CZK na jídlo",
|
||||
"CZK TrialHelp",
|
||||
"CZK maminka svojě věci"
|
||||
]
|
||||
|
||||
accounts_sorted = []
|
||||
|
||||
for pref in preferred_order:
|
||||
for acc in accounts:
|
||||
if acc["name"] == pref:
|
||||
accounts_sorted.append(acc)
|
||||
|
||||
for acc in accounts:
|
||||
if acc not in accounts_sorted:
|
||||
accounts_sorted.append(acc)
|
||||
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active)
|
||||
|
||||
# -------- ALL sheet --------
|
||||
cur.execute(SELECT_LEGACY + " ORDER BY datum DESC")
|
||||
all_rows = cur.fetchall()
|
||||
|
||||
if all_rows:
|
||||
headers = list(all_rows[0].keys())
|
||||
ws_all = wb.create_sheet(title="ALL")
|
||||
ws_all.append(headers)
|
||||
format_sheet(ws_all, all_rows, headers)
|
||||
|
||||
# -------- per account sheets --------
|
||||
for acc in accounts_sorted:
|
||||
|
||||
acc_num = acc["account_number"]
|
||||
sheet_name = acc["name"][:31]
|
||||
|
||||
print(f"➡ Creating sheet: {sheet_name}")
|
||||
|
||||
query = SELECT_LEGACY + f"""
|
||||
WHERE cislo_uctu = '{acc_num}'
|
||||
ORDER BY datum DESC
|
||||
"""
|
||||
|
||||
cur.execute(query)
|
||||
rows = cur.fetchall()
|
||||
|
||||
if not rows:
|
||||
continue
|
||||
|
||||
headers = list(rows[0].keys())
|
||||
ws = wb.create_sheet(title=sheet_name)
|
||||
ws.append(headers)
|
||||
format_sheet(ws, rows, headers)
|
||||
|
||||
conn.close()
|
||||
|
||||
cleanup_old_exports()
|
||||
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H-%M-%S")
|
||||
output_file = os.path.join(OUTPUT_DIR, f"{timestamp} FIO transactions.xlsx")
|
||||
|
||||
wb.save(output_file)
|
||||
|
||||
print(f"✅ Export complete:\n{output_file}")
|
||||
|
||||
# ======================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
export_fio()
|
||||
Reference in New Issue
Block a user