reporty/40 fio 01.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Import Fio banka CSV export (UTF-8, ; separated, quoted)
into MySQL database `fio.transactions`.

Unique key = (Číslo účtu, ID operace, ID pokynu)
Duplicates are skipped silently.
"""

import csv
from pathlib import Path
from datetime import datetime
import pymysql
from pymysql.cursors import DictCursor

# ======== CONFIG ========
CSV_PATH = Path(r"u:\Dropbox\!!!Days\Downloads Z230\Vyhledane pohyby (3).csv")
TABLE_NAME = "transactions"

MYSQL_CONFIG = {
    "host": "192.168.1.76",
    "port": 3307,
    "user": "root",
    "password": "Vlado9674+",
    "database": "fio",
    "charset": "utf8mb4",
    "cursorclass": DictCursor,
    "autocommit": True,
}


# ======== HELPERS ========
def clean(s: str):
    """Trim and normalize text values."""
    if not s:
        return None
    return s.strip() or None


def parse_date(raw: str):
    """Convert dd.mm.yyyy → date"""
    raw = (raw or "").strip()
    if not raw:
        return None
    try:
        return datetime.strptime(raw, "%d.%m.%Y").date()
    except ValueError:
        return None


def parse_float(raw: str):
    """Convert comma/space separated numbers to float"""
    raw = (raw or "").replace(" ", "").replace(",", ".")
    try:
        return float(raw)
    except ValueError:
        return None


# ======== DB ========
def get_mysql_connection():
    return pymysql.connect(**MYSQL_CONFIG)


def ensure_table_exists(conn):
    """Create table if it doesn’t exist, with unique key on (cislo_uctu, id_operace, id_pokynu)."""
    sql = f"""
    CREATE TABLE IF NOT EXISTS `{TABLE_NAME}` (
        id INT AUTO_INCREMENT PRIMARY KEY,
        datum DATE,
        castka DECIMAL(14,2),
        akce VARCHAR(100),
        cislo_uctu VARCHAR(40),
        id_operace VARCHAR(50),
        id_pokynu VARCHAR(50),
        protiucet VARCHAR(40),
        nazev_protiuctu VARCHAR(200),
        kod_banky VARCHAR(20),
        ks VARCHAR(20),
        vs VARCHAR(20),
        ss VARCHAR(20),
        zprava_pro_prijemce VARCHAR(500),
        poznamka VARCHAR(500),
        reference_platce VARCHAR(200),
        typ VARCHAR(100),
        upresneni VARCHAR(500),
        zadal VARCHAR(200),
        zdrojovy_ucet VARCHAR(50),
        nazev_banky VARCHAR(100),
        imported_at DATETIME DEFAULT CURRENT_TIMESTAMP,
        UNIQUE KEY uniq_tx (cislo_uctu, id_operace, id_pokynu)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    """
    with conn.cursor() as cur:
        cur.execute(sql)
    print(f"✅ Tabulka `{TABLE_NAME}` zkontrolována nebo vytvořena.")


# ======== IMPORT ========
def import_fio_csv():
    with open(CSV_PATH, "r", encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f, delimiter=";", quotechar='"')
        rows = list(reader)

    total_rows = len(rows)
    print(f"📄 Načteno {total_rows} řádků ze souboru {CSV_PATH.name}")

    with get_mysql_connection() as conn:
        ensure_table_exists(conn)
        inserted, skipped = 0, 0

        for i, row in enumerate(rows, start=1):
            data = {
                "datum": parse_date(row.get("Datum")),
                "castka": parse_float(row.get("Částka")),
                "akce": clean(row.get("Akce")),
                "cislo_uctu": clean(row.get("Číslo účtu")),
                "id_operace": clean(row.get("ID operace")),
                "id_pokynu": clean(row.get("ID pokynu")),
                "protiucet": clean(row.get("Protiúčet")),
                "nazev_protiuctu": clean(row.get("Název protiúčtu")),
                "kod_banky": clean(row.get("Kód banky")),
                "ks": clean(row.get("KS")),
                "vs": clean(row.get("VS")),
                "ss": clean(row.get("SS")),
                "zprava_pro_prijemce": clean(row.get("Zpráva pro příjemce")),
                "poznamka": clean(row.get("Poznámka")),
                "reference_platce": clean(row.get("Reference plátce")),
                "typ": clean(row.get("Typ")),
                "upresneni": clean(row.get("Upřesnění")),
                "zadal": clean(row.get("Zadal")),
                "zdrojovy_ucet": clean(row.get("Zdrojový účet")),
                "nazev_banky": clean(row.get("Název banky")),
            }

            cols = ", ".join(data.keys())
            placeholders = ", ".join(["%s"] * len(data))
            sql = f"INSERT IGNORE INTO `{TABLE_NAME}` ({cols}) VALUES ({placeholders})"

            with conn.cursor() as cur:
                affected = cur.execute(sql, list(data.values()))
                if affected:
                    inserted += 1
                else:
                    skipped += 1

            # --- progress output ---
            if i % 500 == 0 or i == total_rows:
                print(f"  {i}/{total_rows} zpracováno... ({inserted} vloženo, {skipped} duplicit)")

        # summary
        with conn.cursor() as cur:
            cur.execute(f"SELECT COUNT(*) AS cnt FROM `{TABLE_NAME}`")
            total_db = cur.fetchone()["cnt"]

        print(f"\n✅ Import dokončen: {inserted} nových, {skipped} duplicit přeskočeno.")
        print(f"📊 Celkem v databázi: {total_db} záznamů.")


# ======== MAIN ========
if __name__ == "__main__":
    if not CSV_PATH.exists():
        raise SystemExit(f"❌ Soubor {CSV_PATH} nenalezen.")
    import_fio_csv()