reporty/40 fio 01.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Fio CSV import → MySQL (dev version)
------------------------------------
- Always drops & recreates `transactions` table
- Uses real CSV headers as seen in "Vyhledane pohyby (3).csv"
- Unique key = (Číslo účtu, ID operace, ID pokynu)
"""

import csv
from pathlib import Path
from datetime import datetime
import pymysql
from pymysql.cursors import DictCursor
import re

# ======== CONFIG ========
CSV_PATH = Path(r"u:\Dropbox\!!!Days\Downloads Z230\Vyhledane pohyby (3).csv")
TABLE_NAME = "transactions"

MYSQL_CONFIG = {
    "host": "192.168.1.76",
    "port": 3307,
    "user": "root",
    "password": "Vlado9674+",
    "database": "fio",
    "charset": "utf8mb4",
    "cursorclass": DictCursor,
    "autocommit": True,
}


# ======== HELPERS ========
def clean(s: str):
    if not s:
        return None
    return s.strip() or None


def parse_date(raw: str):
    raw = (raw or "").strip()
    if not raw:
        return None
    try:
        return datetime.strptime(raw, "%d.%m.%Y").date()
    except ValueError:
        return None


def parse_float(raw: str):
    if raw is None:
        return None
    s = str(raw).strip()
    for ch in (" ", "\u00A0", "\u202F", "\u2007"):
        s = s.replace(ch, "")
    s = s.replace(",", ".")
    s = re.sub(r"[^0-9.+-]", "", s)
    try:
        return float(s)
    except ValueError:
        return None


# ======== DB ========
def get_mysql_connection():
    return pymysql.connect(**MYSQL_CONFIG)


def recreate_table(conn):
    """Drop and recreate table with schema matching CSV structure."""
    sql = f"""
    DROP TABLE IF EXISTS `{TABLE_NAME}`;
    CREATE TABLE `{TABLE_NAME}` (
        id INT AUTO_INCREMENT PRIMARY KEY,
        datum DATE,
        objem DECIMAL(14,2),
        mena CHAR(3),
        cislo_uctu VARCHAR(40),
        protiucet VARCHAR(40),
        kod_banky VARCHAR(20),
        ks VARCHAR(20),
        vs VARCHAR(20),
        ss VARCHAR(20),
        zprava_pro_prijemce VARCHAR(500),
        poznamka VARCHAR(500),
        id_operace VARCHAR(50),
        id_pokynu VARCHAR(50),
        ks_1 VARCHAR(20),
        nazev_banky VARCHAR(100),
        nazev_protiuctu VARCHAR(200),
        ss_1 VARCHAR(20),
        typ VARCHAR(100),
        upresneni_objem VARCHAR(100),
        upresneni_mena VARCHAR(20),
        vs_1 VARCHAR(20),
        zadal VARCHAR(200),
        imported_at DATETIME DEFAULT CURRENT_TIMESTAMP,
        UNIQUE KEY uniq_tx (cislo_uctu, id_operace, id_pokynu)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    """
    with conn.cursor() as cur:
        for stmt in sql.strip().split(";"):
            if stmt.strip():
                cur.execute(stmt)
    print(f"✅ Tabulka `{TABLE_NAME}` znovu vytvořena podle CSV struktury.")


# ======== IMPORT ========
def import_fio_csv():
    with open(CSV_PATH, "r", encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f, delimiter=";", quotechar='"')
        rows = list(reader)

    total = len(rows)
    print(f"📄 Načteno {total} řádků ze souboru {CSV_PATH.name}")

    with get_mysql_connection() as conn:
        recreate_table(conn)
        inserted, skipped = 0, 0

        for i, row in enumerate(rows, start=1):
            data = {
                "datum": parse_date(row.get("Datum")),
                "objem": parse_float(row.get("Objem")),
                "mena": clean(row.get("Měna")),
                "cislo_uctu": clean(row.get("Číslo účtu")),
                "protiucet": clean(row.get("Protiúčet")),
                "kod_banky": clean(row.get("Kód banky")),
                "ks": clean(row.get("KS")),
                "vs": clean(row.get("VS")),
                "ss": clean(row.get("SS")),
                "zprava_pro_prijemce": clean(row.get("Zpráva pro příjemce")),
                "poznamka": clean(row.get("Poznámka")),
                "id_operace": clean(row.get("ID operace")),
                "id_pokynu": clean(row.get("ID pokynu")),
                "ks_1": clean(row.get("KS.1")),
                "nazev_banky": clean(row.get("Název banky")),
                "nazev_protiuctu": clean(row.get("Název protiúčtu")),
                "ss_1": clean(row.get("SS.1")),
                "typ": clean(row.get("Typ")),
                "upresneni_objem": clean(row.get("Upřesnění - objem")),
                "upresneni_mena": clean(row.get("Upřesnění - měna")),
                "vs_1": clean(row.get("VS.1")),
                "zadal": clean(row.get("Zadal")),
            }

            cols = ", ".join(data.keys())
            placeholders = ", ".join(["%s"] * len(data))
            sql = f"INSERT IGNORE INTO `{TABLE_NAME}` ({cols}) VALUES ({placeholders})"

            with conn.cursor() as cur:
                affected = cur.execute(sql, list(data.values()))
                if affected:
                    inserted += 1
                else:
                    skipped += 1

            if i % 500 == 0 or i == total:
                print(f"  {i}/{total} zpracováno... ({inserted} vloženo, {skipped} duplicit)")

        print(f"\n✅ Import dokončen: {inserted} nových, {skipped} duplicit přeskočeno.")


# ======== MAIN ========
if __name__ == "__main__":
    import_fio_csv()