112 lines
3.8 KiB
Python
112 lines
3.8 KiB
Python
# ============================================================
|
|
# export_from_seaweed_v1.0.py
|
|
# Verze: 1.0
|
|
# Datum: 2026-06-15
|
|
# Popis: Sestaví na disk strom dokumentů jedné úrovně studie
|
|
# ze SeaweedFS (zdroj pravdy je Mongo VTMF.documents +
|
|
# objekty v SeaweedFS Fileru). Pojmenování podle původních
|
|
# Dropbox pravidel (pipeline v1.5):
|
|
#
|
|
# <OUTPUT_ROOT>\<Type>\<Subtype>\
|
|
# "YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>"
|
|
#
|
|
# Datum/verze se vynechají, když chybí. Stahuje jen
|
|
# deleted=False, downloaded=True, placeholder!=True,
|
|
# se seaweed_path. Idempotentní — existující soubory
|
|
# přeskakuje (resume), takže lze pouštět opakovaně pro
|
|
# „aktuální verzi" čehokoli.
|
|
#
|
|
# Konfigurace (konstanty níže): STUDY, LEVEL, OUTPUT_ROOT.
|
|
# Pro jinou úroveň/studii stačí změnit tyto tři.
|
|
#
|
|
# Spuštění:
|
|
# & "...\.venv\Scripts\python.exe" "...\export_from_seaweed_v1.0.py"
|
|
# ============================================================
|
|
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from pymongo import MongoClient, ASCENDING
|
|
|
|
# --- Konfigurace -------------------------------------------------------
|
|
|
|
STUDY = "77242113UCO3001"
|
|
LEVEL = "study" # study | country | site
|
|
OUTPUT_ROOT = Path(r"U:\77242113UCO3001_STUDYLEVEL")
|
|
|
|
MONGO_URI = "mongodb://192.168.1.76:27017"
|
|
MONGO_DB = "VTMF"
|
|
MONGO_COLL = "documents"
|
|
|
|
OVERWRITE = False # True = přepsat i existující soubory
|
|
|
|
BAD_CHARS_RE = re.compile(r"[<>:\"/\\|?*\x00-\x1f�]")
|
|
|
|
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
|
|
|
|
def clean(s):
|
|
s = BAD_CHARS_RE.sub("_", str(s or ""))
|
|
s = re.sub(r"\s+", " ", s)
|
|
s = re.sub(r"_{2,}", "_", s)
|
|
return s.strip(" ._")
|
|
|
|
|
|
def target_path(doc):
|
|
"""<OUTPUT_ROOT>\\<Type>\\<Subtype>\\
|
|
'YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>'."""
|
|
ext = Path(doc.get("seaweed_path", "")).suffix
|
|
date = doc.get("date") or ""
|
|
date_prefix = (date + " ") if date else ""
|
|
version = f" [{doc['version']}]" if doc.get("version") else ""
|
|
desc = clean(doc.get("desc")) or clean(doc.get("vtmf"))
|
|
filename = f"{date_prefix}{desc} [{doc['vtmf']}]{version}{ext}"
|
|
typ = clean(doc.get("type")) or "_"
|
|
sub = clean(doc.get("subtype")) or "_"
|
|
return OUTPUT_ROOT / typ / sub / filename
|
|
|
|
|
|
def main():
|
|
coll = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)[MONGO_DB][MONGO_COLL]
|
|
q = {"level": LEVEL, "deleted": False, "downloaded": True,
|
|
"placeholder": {"$ne": True}, "seaweed_path": {"$ne": None}}
|
|
docs = list(coll.find(q).sort([("vtmf", ASCENDING), ("version", ASCENDING)]))
|
|
log(f"[i] {LEVEL}-level dokumentů studie {STUDY}: {len(docs)}")
|
|
log(f"[i] Cíl: {OUTPUT_ROOT}\n")
|
|
|
|
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
|
written = skipped = failed = 0
|
|
total_bytes = 0
|
|
|
|
for n, doc in enumerate(docs, 1):
|
|
dest = target_path(doc)
|
|
if dest.exists() and not OVERWRITE:
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
with urllib.request.urlopen(doc["seaweed_url"], timeout=120) as r:
|
|
data = r.read()
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(data)
|
|
written += 1
|
|
total_bytes += len(data)
|
|
kb = len(data) / 1024
|
|
size = f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB"
|
|
log(f"[{n}/{len(docs)}] {dest.relative_to(OUTPUT_ROOT)} ({size})")
|
|
except Exception as e:
|
|
failed += 1
|
|
log(f"[!] {doc['_id']}: {e}")
|
|
|
|
mb = total_bytes / 1024 / 1024
|
|
log(f"\n[ok] Hotovo: {written} zapsáno ({mb:.1f} MB), "
|
|
f"{skipped} přeskočeno (už existuje), {failed} chyb.")
|
|
sys.exit(1 if failed else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|