reporter
This commit is contained in:
@@ -1,42 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
FIO MULTI–ACCOUNT IMPORTER — FULLY COMMENTED VERSION
|
||||
====================================================
|
||||
|
||||
This script downloads transactions for **multiple Fio bank accounts**
|
||||
(using their API tokens) and imports them into a MySQL database
|
||||
(`fio.transactions` table).
|
||||
|
||||
It also saves the raw JSON responses into a folder structure
|
||||
for backup / debugging / later use.
|
||||
|
||||
Main features:
|
||||
• Reads all accounts from accounts.json
|
||||
• Downloads last N days (default 90)
|
||||
• Saves JSON files to disk
|
||||
• Extracts all transactions with safe parsing
|
||||
• Inserts into MySQL with ON DUPLICATE KEY UPDATE
|
||||
• Efficient batch insertion (executemany)
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import pymysql
|
||||
import requests # used to call Fio REST API
|
||||
import pymysql # MySQL driver
|
||||
|
||||
|
||||
# =========================================
|
||||
# CONFIG
|
||||
# CONFIGURATION
|
||||
# =========================================
|
||||
ACCOUNTS_FILE = r"u:\PycharmProjects\FIO\accounts.json"
|
||||
JSON_BASE_DIR = r"u:\Dropbox\!!!Days\Downloads Z230\Fio" # kam se budou ukládat JSONy
|
||||
|
||||
# JSON file containing multiple account configs:
|
||||
# [
|
||||
# { "name": "CZK rodina", "account_number": "2100046291", "token": "xxx" },
|
||||
# ...
|
||||
# ]
|
||||
ACCOUNTS_FILE = r"c:\users\vlado\PycharmProjects\FIO\accounts.json"
|
||||
|
||||
# Directory where raw JSON files from Fio API will be stored.
|
||||
JSON_BASE_DIR = r"z:\Dropbox\!!!Days\Downloads Z230\Fio"
|
||||
|
||||
# MySQL connection parameters
|
||||
DB = {
|
||||
"host": "192.168.1.76",
|
||||
"port": 3307,
|
||||
"user": "root",
|
||||
"password": "Vlado9674+", # uprav podle sebe / dej do .env
|
||||
"password": "Vlado9674+",
|
||||
"database": "fio",
|
||||
"charset": "utf8mb4",
|
||||
}
|
||||
|
||||
# How many transactions insert per batch (performance tuning)
|
||||
BATCH_SIZE = 500
|
||||
|
||||
# How many days back we load from Fio (default = last 90 days)
|
||||
DAYS_BACK = 90
|
||||
|
||||
|
||||
# =========================================
|
||||
# HELPERS
|
||||
# =========================================
|
||||
|
||||
def load_accounts(path: str):
|
||||
"""
|
||||
Reads accounts.json and does simple validation to ensure
|
||||
each entry contains: name, account_number, token.
|
||||
"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
accounts = json.load(f)
|
||||
|
||||
# jednoduchá validace
|
||||
for acc in accounts:
|
||||
for key in ("name", "account_number", "token"):
|
||||
if key not in acc:
|
||||
@@ -46,17 +83,28 @@ def load_accounts(path: str):
|
||||
|
||||
|
||||
def fio_url_for_period(token: str, d_from: date, d_to: date) -> str:
|
||||
"""
|
||||
Constructs the exact URL for Fio REST API "periods" endpoint.
|
||||
Example:
|
||||
https://fioapi.fio.cz/v1/rest/periods/<token>/2025-01-01/2025-01-31/transactions.json
|
||||
"""
|
||||
from_str = d_from.strftime("%Y-%m-%d")
|
||||
to_str = d_to.strftime("%Y-%m-%d")
|
||||
return f"https://fioapi.fio.cz/v1/rest/periods/{token}/{from_str}/{to_str}/transactions.json"
|
||||
|
||||
|
||||
def fetch_fio_json(token: str, d_from: date, d_to: date):
|
||||
"""
|
||||
Calls Fio API and fetches JSON.
|
||||
Handles HTTP errors and JSON decoding errors.
|
||||
"""
|
||||
url = fio_url_for_period(token, d_from, d_to)
|
||||
resp = requests.get(url, timeout=30)
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f" ❌ HTTP {resp.status_code} from Fio: {url}")
|
||||
return None
|
||||
|
||||
try:
|
||||
return resp.json()
|
||||
except json.JSONDecodeError:
|
||||
@@ -66,11 +114,16 @@ def fetch_fio_json(token: str, d_from: date, d_to: date):
|
||||
|
||||
def safe_col(t: dict, n: int):
|
||||
"""
|
||||
Safely read t['columnN']['value'], i.e. Fio column.
|
||||
Handles:
|
||||
- missing columnN
|
||||
- columnN is None
|
||||
- missing 'value'
|
||||
SAFE ACCESSOR for Fio transaction column numbers.
|
||||
|
||||
Fio JSON schema example:
|
||||
"column5": { "name": "VS", "value": "123456" }
|
||||
|
||||
But the structure is NOT guaranteed to exist.
|
||||
So this function prevents KeyError or NoneType errors.
|
||||
|
||||
Returns:
|
||||
t["columnN"]["value"] or None
|
||||
"""
|
||||
key = f"column{n}"
|
||||
val = t.get(key)
|
||||
@@ -81,8 +134,8 @@ def safe_col(t: dict, n: int):
|
||||
|
||||
def clean_date(dt_str: str):
|
||||
"""
|
||||
Convert Fio date '2025-10-26+0200' -> '2025-10-26'
|
||||
Fio spec: date is always rrrr-mm-dd+GMT.
|
||||
Fio returns dates like: "2025-02-14+0100"
|
||||
We strip timezone → "2025-02-14"
|
||||
"""
|
||||
if not dt_str:
|
||||
return None
|
||||
@@ -90,15 +143,19 @@ def clean_date(dt_str: str):
|
||||
|
||||
|
||||
def ensure_dir(path: Path):
|
||||
"""Creates directory if it doesn’t exist."""
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def save_json_for_account(base_dir: str, account_cfg: dict, data: dict, d_from: date, d_to: date):
|
||||
"""
|
||||
Uloží JSON do podsložky dle čísla účtu, název souboru podle období.
|
||||
Saves raw JSON to:
|
||||
<base_dir>/<account_number>/YYYY-MM-DD_to_YYYY-MM-DD.json
|
||||
|
||||
Useful for debugging, backups, or re-imports.
|
||||
"""
|
||||
acc_num_raw = account_cfg["account_number"]
|
||||
acc_folder_name = acc_num_raw.replace("/", "_") # 2101234567_2700
|
||||
acc_folder_name = acc_num_raw.replace("/", "_") # sanitize dir name for filesystem
|
||||
|
||||
out_dir = Path(base_dir) / acc_folder_name
|
||||
ensure_dir(out_dir)
|
||||
@@ -113,28 +170,31 @@ def save_json_for_account(base_dir: str, account_cfg: dict, data: dict, d_from:
|
||||
|
||||
|
||||
# =========================================
|
||||
# MAIN IMPORT
|
||||
# MAIN IMPORT LOGIC
|
||||
# =========================================
|
||||
|
||||
def main():
|
||||
start_all = time.time()
|
||||
|
||||
# období posledních 90 dní
|
||||
# Calculate time range (last N days)
|
||||
today = date.today()
|
||||
d_from = today - timedelta(days=DAYS_BACK)
|
||||
d_to = today
|
||||
|
||||
print(f"=== Fio multi-account import ===")
|
||||
print("=== Fio multi-account import ===")
|
||||
print(f"Období: {d_from} až {d_to}")
|
||||
print("Načítám účty z JSON konfigurace...")
|
||||
|
||||
# Load all accounts from accounts.json
|
||||
accounts = load_accounts(ACCOUNTS_FILE)
|
||||
print(f" Účtů v konfiguraci: {len(accounts)}\n")
|
||||
|
||||
# Připojení do DB
|
||||
# Connect to database
|
||||
conn = pymysql.connect(**DB)
|
||||
cur = conn.cursor()
|
||||
|
||||
# SQL s ON DUPLICATE KEY UPDATE
|
||||
# SQL INSERT with ON DUPLICATE KEY UPDATE
|
||||
# This means: if transaction already exists (same unique key), update it.
|
||||
sql = """
|
||||
INSERT INTO transactions
|
||||
(
|
||||
@@ -174,6 +234,9 @@ def main():
|
||||
|
||||
total_inserted = 0
|
||||
|
||||
# ======================================================
|
||||
# PROCESS EACH ACCOUNT IN accounts.json
|
||||
# ======================================================
|
||||
for acc in accounts:
|
||||
name = acc["name"]
|
||||
cfg_acc_num = acc["account_number"]
|
||||
@@ -182,17 +245,20 @@ def main():
|
||||
print(f"--- Účet: {name} ({cfg_acc_num}) ---")
|
||||
t0 = time.time()
|
||||
|
||||
# --- 1) Download JSON from Fio API
|
||||
data = fetch_fio_json(token, d_from, d_to)
|
||||
if data is None:
|
||||
print(" Přeskakuji, žádná data / chyba API.\n")
|
||||
continue
|
||||
|
||||
# volitelné uložení JSON
|
||||
# --- 2) Save raw JSON file to disk
|
||||
json_path = save_json_for_account(JSON_BASE_DIR, acc, data, d_from, d_to)
|
||||
print(f" JSON uložen do: {json_path}")
|
||||
|
||||
# extrakce transakcí
|
||||
# --- 3) Extract transactions from JSON tree
|
||||
tlist = data["accountStatement"]["transactionList"].get("transaction", [])
|
||||
|
||||
# FIO can return single transaction as an object (not list)
|
||||
if isinstance(tlist, dict):
|
||||
tlist = [tlist]
|
||||
|
||||
@@ -202,13 +268,15 @@ def main():
|
||||
print(" Žádné transakce, jdu dál.\n")
|
||||
continue
|
||||
|
||||
# FIO returns account ID under accountStatement.info.accountId
|
||||
fio_acc_id = data["accountStatement"]["info"]["accountId"]
|
||||
|
||||
# Warn if account ID in JSON doesn't match config (informational only)
|
||||
if cfg_acc_num and cfg_acc_num.split("/")[0] not in fio_acc_id:
|
||||
# jen varování, ne fatální chyba
|
||||
print(f" ⚠ Upozornění: accountId z Fio ({fio_acc_id}) "
|
||||
f"se neshoduje s account_number v konfiguraci ({cfg_acc_num})")
|
||||
|
||||
# připravit řádky pro batch insert
|
||||
# --- 4) Build list of MySQL rows
|
||||
rows = []
|
||||
for t in tlist:
|
||||
row = {
|
||||
@@ -245,25 +313,33 @@ def main():
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
# batch insert
|
||||
# --- 5) INSERT rows into MySQL in batches
|
||||
inserted = 0
|
||||
|
||||
for i in range(0, len(rows), BATCH_SIZE):
|
||||
chunk = rows[i:i + BATCH_SIZE]
|
||||
cur.executemany(sql, chunk)
|
||||
chunk = rows[i : i + BATCH_SIZE]
|
||||
cur.executemany(sql, chunk) # fast multi-row insert/update
|
||||
conn.commit()
|
||||
inserted += len(chunk)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
total_inserted += inserted
|
||||
|
||||
print(f" ✓ Zapsáno (insert/update): {inserted} řádků do DB za {elapsed:.2f} s\n")
|
||||
|
||||
# Close DB
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
total_elapsed = time.time() - start_all
|
||||
|
||||
print(f"=== Hotovo. Celkem zapsáno {total_inserted} transakcí. "
|
||||
f"Celkový čas: {total_elapsed:.2f} s ===")
|
||||
|
||||
|
||||
# ======================================================
|
||||
# ENTRY POINT
|
||||
# ======================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user