z230
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -1,3 +1,7 @@
|
||||
.venv/*
|
||||
.idea/
|
||||
__pycache__/
|
||||
__pycache__/
|
||||
# Neignorovat tyto typy
|
||||
!*.jsonl
|
||||
!*.txt
|
||||
!*.log
|
||||
136
Syncthing/01 LocalCleanerBasedOnRemoteHash.py
Normal file
136
Syncthing/01 LocalCleanerBasedOnRemoteHash.py
Normal file
@@ -0,0 +1,136 @@
|
||||
import os
|
||||
import pymysql
|
||||
import sys
|
||||
import blake3 # Nutné: pip install blake3
|
||||
|
||||
# ==============================
|
||||
# ⚙️ NASTAVENÍ
|
||||
# ==============================
|
||||
LOCAL_ROOT = r"U:\#Syncthing"
|
||||
|
||||
# !!! AŽ SI BUDEŠ JISTÝ, ZMĚŇ NA True !!!
|
||||
DELETE_MODE = False
|
||||
|
||||
# Hledáme shodu pouze v souborech, které jsou na serveru v této cestě:
|
||||
REMOTE_FOLDER_FILTER = "/#ColdData/Porno/"
|
||||
|
||||
# DB Nastavení
|
||||
DB_HOST = "192.168.1.76"
|
||||
DB_PORT = 3307
|
||||
DB_USER = "root"
|
||||
DB_PASS = "Vlado9674+"
|
||||
DB_NAME = "torrents"
|
||||
DB_TABLE = "file_md5_index"
|
||||
|
||||
|
||||
# ==============================
|
||||
|
||||
def get_db_connection():
|
||||
try:
|
||||
return pymysql.connect(
|
||||
host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASS,
|
||||
database=DB_NAME, cursorclass=pymysql.cursors.DictCursor
|
||||
)
|
||||
except Exception as e:
|
||||
sys.exit(f"❌ Chyba DB: {e}")
|
||||
|
||||
|
||||
def calculate_blake3(file_path):
|
||||
"""Spočítá BLAKE3 hash souboru"""
|
||||
h = blake3.blake3()
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(4 * 1024 * 1024) # 4MB chunks
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.digest()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print(f"🚀 TERMINÁTOR: Mazání podle obsahu (BLAKE3)")
|
||||
print(f" 📂 Zdroj: {LOCAL_ROOT}")
|
||||
print(f" 🎯 DB filtr: {REMOTE_FOLDER_FILTER}")
|
||||
print(f" 🛠 Mód: {'🔴 OSTRÝ (MAZÁNÍ)' if DELETE_MODE else '🟢 SIMULACE (BEZPEČNÝ)'}")
|
||||
print("=====================================================")
|
||||
|
||||
# 1. NAČTENÍ HASHŮ Z DB
|
||||
print("📡 Stahuji hashe ze serveru...")
|
||||
remote_hashes = set()
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cursor:
|
||||
sql = f"SELECT blake3 FROM {DB_TABLE} WHERE full_path LIKE %s"
|
||||
cursor.execute(sql, (f"%{REMOTE_FOLDER_FILTER}%",))
|
||||
rows = cursor.fetchall()
|
||||
|
||||
for row in rows:
|
||||
remote_hashes.add(row['blake3'])
|
||||
|
||||
print(f"✅ V paměti mám {len(remote_hashes):,} unikátních hashů ze serveru.")
|
||||
print("=====================================================")
|
||||
|
||||
# 2. SKENOVÁNÍ A MAZÁNÍ
|
||||
count_scanned = 0
|
||||
count_match = 0
|
||||
size_freed = 0
|
||||
|
||||
for root, dirs, files in os.walk(LOCAL_ROOT):
|
||||
# Přeskočit syncthing složky
|
||||
if ".stfolder" in dirs: dirs.remove(".stfolder")
|
||||
if ".stversions" in dirs: dirs.remove(".stversions")
|
||||
|
||||
for filename in files:
|
||||
file_path = os.path.join(root, filename)
|
||||
count_scanned += 1
|
||||
|
||||
# Výpis práce
|
||||
print(f"⏳ [{count_scanned}] Zpracovávám: {filename}")
|
||||
|
||||
# Spočítat lokální hash
|
||||
local_hash = calculate_blake3(file_path)
|
||||
|
||||
if local_hash is None:
|
||||
print(f" ⚠️ Chyba čtení souboru!")
|
||||
continue
|
||||
|
||||
# Zobrazíme kousek hashe pro kontrolu (převedeme na hex text)
|
||||
hash_preview = local_hash.hex()[:8]
|
||||
|
||||
# JE TEN HASH V DATABÁZI?
|
||||
if local_hash in remote_hashes:
|
||||
count_match += 1
|
||||
file_size = os.path.getsize(file_path)
|
||||
size_freed += file_size
|
||||
|
||||
print(f" ✅ SHODA (Hash: {hash_preview}...) -> {'🗑️ MAŽU' if DELETE_MODE else '📦 NAŠEL BYCH'}")
|
||||
|
||||
if DELETE_MODE:
|
||||
try:
|
||||
os.remove(file_path)
|
||||
# Pokus o smazání prázdné složky
|
||||
try:
|
||||
os.rmdir(root)
|
||||
except:
|
||||
pass
|
||||
except OSError as e:
|
||||
print(f" ❌ Chyba mazání: {e}")
|
||||
else:
|
||||
print(f" ❌ Žádná shoda (Hash: {hash_preview}...) -> Nechávám být")
|
||||
|
||||
print("\n\n📊 VÝSLEDEK")
|
||||
print("=====================================================")
|
||||
print(f"🔎 Prošlo rukama: {count_scanned:,} souborů")
|
||||
print(f"🗑️ Smazáno/Shoda: {count_match:,} souborů")
|
||||
print(f"💾 Ušetřené místo: {size_freed / (1024 * 1024):.2f} MB")
|
||||
print("=====================================================")
|
||||
|
||||
if not DELETE_MODE and count_match > 0:
|
||||
print("💡 TIP: Až budeš připraven, změň v kódu: DELETE_MODE = True")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
134
Syncthing/02 SyncthingBlakeDoSouboru.py
Normal file
134
Syncthing/02 SyncthingBlakeDoSouboru.py
Normal file
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import blake3
|
||||
import sys
|
||||
import json
|
||||
from tqdm import tqdm
|
||||
|
||||
# ==============================
|
||||
# ⚙️ NASTAVENÍ
|
||||
# ==============================
|
||||
LOCAL_ROOT = r"U:\#Syncthing"
|
||||
CACHE_FILE = "local_hashes.jsonl"
|
||||
ERROR_LOG = "errors_scan.txt"
|
||||
|
||||
|
||||
def calculate_blake3_hex(file_path):
|
||||
h = blake3.blake3()
|
||||
# Zde ošetříme chyby při otevírání/čtení souboru
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(4 * 1024 * 1024)
|
||||
if not chunk: break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
except Exception as e:
|
||||
# Vyhodíme výjimku nahoru, abychom ji zachytili v hlavní smyčce i s názvem souboru
|
||||
raise e
|
||||
|
||||
|
||||
def count_files(path):
|
||||
print("⏳ Sčítám soubory...", end="\r")
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(path):
|
||||
if ".stfolder" in dirs: dirs.remove(".stfolder")
|
||||
if ".stversions" in dirs: dirs.remove(".stversions")
|
||||
total += len(files)
|
||||
print(f"✅ Celkem souborů: {total:,}")
|
||||
return total
|
||||
|
||||
|
||||
def load_processed_files(cache_file):
|
||||
"""Načte cesty souborů, které už máme hotové"""
|
||||
processed = set()
|
||||
if os.path.exists(cache_file):
|
||||
print("📥 Načítám již hotové záznamy pro navázání...")
|
||||
try:
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
try:
|
||||
data = json.loads(line)
|
||||
processed.add(data["p"]) # "p" je cesta
|
||||
except:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"⚠️ Chyba při čtení cache (nevadí, jedeme dál): {e}")
|
||||
|
||||
print(f"⏭ Přeskočím {len(processed):,} již hotových souborů.")
|
||||
return processed
|
||||
|
||||
|
||||
def main():
|
||||
print("🚀 KROK 1: Generování cache (RESUMABLE + SAFE MODE)")
|
||||
print(f"📂 Zdroj: {LOCAL_ROOT}")
|
||||
print(f"💾 Výstup: {CACHE_FILE}")
|
||||
print("===================================")
|
||||
|
||||
# 1. Zjistit celkový počet
|
||||
total_files = count_files(LOCAL_ROOT)
|
||||
|
||||
# 2. Zjistit, co už máme hotové
|
||||
processed_paths = load_processed_files(CACHE_FILE)
|
||||
|
||||
errors = 0
|
||||
newly_processed = 0
|
||||
|
||||
# Otevřeme v režimu 'a' (APPEND) - připisování na konec
|
||||
with open(CACHE_FILE, "a", encoding="utf-8") as f_out:
|
||||
|
||||
with tqdm(total=total_files, unit="file", dynamic_ncols=True) as pbar:
|
||||
|
||||
# Pokud už máme nějaká data, aktualizujeme progress bar
|
||||
if processed_paths:
|
||||
pbar.update(len(processed_paths))
|
||||
|
||||
for root, dirs, files in os.walk(LOCAL_ROOT):
|
||||
if ".stfolder" in dirs: dirs.remove(".stfolder")
|
||||
if ".stversions" in dirs: dirs.remove(".stversions")
|
||||
|
||||
for filename in files:
|
||||
file_path = os.path.join(root, filename)
|
||||
|
||||
# === SKIPPER: Pokud už to máme, jdeme dál ===
|
||||
if file_path in processed_paths:
|
||||
continue
|
||||
# ============================================
|
||||
|
||||
# Pro dlouhé názvy ořízneme text v progress baru
|
||||
short_name = (filename[:30] + '...') if len(filename) > 30 else filename
|
||||
pbar.set_description(f"Hash: {short_name}")
|
||||
|
||||
# === SAFE BLOCK: Try-Except uvnitř smyčky ===
|
||||
try:
|
||||
hex_hash = calculate_blake3_hex(file_path)
|
||||
|
||||
if hex_hash:
|
||||
record = {"h": hex_hash, "p": file_path}
|
||||
f_out.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
# Flush, aby se to zapsalo na disk hned (pro jistotu)
|
||||
if newly_processed % 100 == 0:
|
||||
f_out.flush()
|
||||
newly_processed += 1
|
||||
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
error_msg = f"❌ CHYBA: {filename} -> {e}\n"
|
||||
# Zapíšeme chybu do logu, aby to nerušilo output
|
||||
with open(ERROR_LOG, "a", encoding="utf-8") as ferr:
|
||||
ferr.write(error_msg)
|
||||
|
||||
# Posuneme progress bar
|
||||
pbar.update(1)
|
||||
|
||||
print("\n\n✅ HOTOVO.")
|
||||
print(f"📄 Nově přidáno: {newly_processed}")
|
||||
print(f"⏭ Přeskočeno: {len(processed_paths)}")
|
||||
|
||||
if errors > 0:
|
||||
print(f"⚠️ Počet chyb: {errors} (Detaily viz soubor '{ERROR_LOG}')")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
150
Syncthing/03 Terminator.py
Normal file
150
Syncthing/03 Terminator.py
Normal file
@@ -0,0 +1,150 @@
|
||||
import os
|
||||
import pymysql
|
||||
import sys
|
||||
import json
|
||||
from tqdm import tqdm
|
||||
|
||||
# ==============================
|
||||
# ⚙️ NASTAVENÍ
|
||||
# ==============================
|
||||
CACHE_FILE = "local_hashes.jsonl"
|
||||
|
||||
# !!! ZMĚŇ NA True PRO SKUTEČNÉ MAZÁNÍ !!!
|
||||
DELETE_MODE = True
|
||||
|
||||
# Filtr DB (Hledáme shodu jen v této složce na serveru)
|
||||
REMOTE_FOLDER_FILTER = "/#ColdData/Porno/"
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": "192.168.1.76",
|
||||
"port": 3307,
|
||||
"user": "root",
|
||||
"password": "Vlado9674+",
|
||||
"database": "torrents",
|
||||
}
|
||||
|
||||
|
||||
# ==============================
|
||||
|
||||
def get_remote_hashes():
|
||||
print("📡 Stahuji hashe z DB (filtr: " + REMOTE_FOLDER_FILTER + ")...")
|
||||
hashes = set()
|
||||
try:
|
||||
conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
|
||||
with conn.cursor() as cursor:
|
||||
# Stáhneme hashe souborů, které jsou v cílové složce
|
||||
sql = f"SELECT blake3 FROM file_md5_index WHERE full_path LIKE %s"
|
||||
cursor.execute(sql, (f"%{REMOTE_FOLDER_FILTER}%",))
|
||||
|
||||
for row in cursor.fetchall():
|
||||
# DB vrací bytes, my potřebujeme HEX string (stejný jako v JSONL)
|
||||
hashes.add(row['blake3'].hex())
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
sys.exit(f"❌ Chyba DB: {e}")
|
||||
return hashes
|
||||
|
||||
|
||||
def count_lines(filename):
|
||||
"""Spočítá řádky pro progress bar"""
|
||||
print("⏳ Sčítám záznamy v cache...", end="\r")
|
||||
with open(filename, "r", encoding="utf-8") as f:
|
||||
return sum(1 for _ in f)
|
||||
|
||||
|
||||
def main():
|
||||
print("🚀 KROK 2: Terminátor (Mazání podle cache)")
|
||||
print(f"📂 Cache soubor: {CACHE_FILE}")
|
||||
print(f"🛠 Mód: {'🔴 OSTRÝ (MAZÁNÍ)' if DELETE_MODE else '🟢 SIMULACE (JEN VÝPIS)'}")
|
||||
print("===================================")
|
||||
|
||||
if not os.path.exists(CACHE_FILE):
|
||||
sys.exit("❌ Cache soubor neexistuje! Spusť nejdřív KROK 1.")
|
||||
|
||||
# 1. Stáhnout DB
|
||||
remote_hashes = get_remote_hashes()
|
||||
print(f"✅ DB načtena: {len(remote_hashes):,} unikátních hashů ze serveru.")
|
||||
|
||||
# 2. Spočítat řádky pro progress bar
|
||||
total_lines = count_lines(CACHE_FILE)
|
||||
print(f"✅ Cache obsahuje: {total_lines:,} lokálních souborů ke kontrole.")
|
||||
|
||||
print("🔍 Jdu porovnávat...")
|
||||
|
||||
count_deleted = 0
|
||||
freed_space = 0
|
||||
errors = 0
|
||||
|
||||
# Otevření souboru pro logování smazaných (volitelné)
|
||||
log_name = "deleted_files.log" if DELETE_MODE else "would_delete.log"
|
||||
|
||||
with open(CACHE_FILE, "r", encoding="utf-8") as f_in, \
|
||||
open(log_name, "w", encoding="utf-8") as f_log:
|
||||
|
||||
# Progress bar
|
||||
with tqdm(total=total_lines, unit="file", dynamic_ncols=True) as pbar:
|
||||
|
||||
for line in f_in:
|
||||
pbar.update(1)
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
|
||||
try:
|
||||
data = json.loads(line)
|
||||
local_hash = data["h"]
|
||||
file_path = data["p"]
|
||||
except:
|
||||
continue # Vadný řádek
|
||||
|
||||
# Kontrola existence souboru (mohli jsme ho už smazat ručně)
|
||||
if not os.path.exists(file_path):
|
||||
continue
|
||||
|
||||
# --- HLAVNÍ LOGIKA ---
|
||||
if local_hash in remote_hashes:
|
||||
# SHODA! Soubor je na serveru -> SMAZAT lokálně
|
||||
|
||||
try:
|
||||
size = os.path.getsize(file_path)
|
||||
|
||||
if DELETE_MODE:
|
||||
os.remove(file_path)
|
||||
status = "SMAZÁNO"
|
||||
# Pokus o smazání prázdné složky
|
||||
try:
|
||||
os.rmdir(os.path.dirname(file_path))
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
status = "NAŠEL BYCH"
|
||||
|
||||
count_deleted += 1
|
||||
freed_space += size
|
||||
|
||||
# Zápis do logu
|
||||
f_log.write(f"{file_path}\n")
|
||||
|
||||
# Aktualizace popisku (jen občas, aby to neblikalo moc rychle)
|
||||
if count_deleted % 10 == 0:
|
||||
pbar.set_description(f"Mazání: {count_deleted} files")
|
||||
|
||||
except OSError as e:
|
||||
errors += 1
|
||||
# print(f"❌ Chyba u {file_path}: {e}")
|
||||
|
||||
print("\n\n📊 VÝSLEDEK")
|
||||
print("===================================")
|
||||
print(f"🔎 Zkontrolováno: {total_lines:,}")
|
||||
print(f"🗑️ {'Smazáno' if DELETE_MODE else 'K smazání'}: {count_deleted:,}")
|
||||
print(f"💾 Ušetřeno místo: {freed_space / (1024 * 1024 * 1024):.2f} GB")
|
||||
print(f"📝 Seznam souborů: '{log_name}'")
|
||||
|
||||
if errors > 0:
|
||||
print(f"⚠️ Chyb při mazání: {errors}")
|
||||
|
||||
if not DELETE_MODE and count_deleted > 0:
|
||||
print("\n💡 LÍBÍ SE TI VÝSLEDEK? Změň nahoře: DELETE_MODE = True")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user