187 lines
6.9 KiB
Python
187 lines
6.9 KiB
Python
# ============================================================
|
|
# export_from_seaweed_v1.2.py
|
|
# Verze: 1.2
|
|
# Datum: 2026-06-15
|
|
# Popis: Sestaví na disk strom dokumentů studie ze SeaweedFS
|
|
# (zdroj pravdy = Mongo VTMF.documents + objekty ve Fileru).
|
|
# Pojmenování podle původních Dropbox pravidel (pipeline v1.5):
|
|
#
|
|
# <základní podadresář>\<Type>\<Subtype>\
|
|
# "YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>"
|
|
#
|
|
# Základní podadresář pod OUTPUT_ROOT podle úrovně:
|
|
# study -> STUDY
|
|
# country -> COUNTRY
|
|
# site -> <číslo centra> (dokument se zapíše do složky
|
|
# KAŽDÉHO centra, do kterého je referencovaný)
|
|
#
|
|
# PŘÍSLUŠNOST KE STUDII = pole scopes[] (ne studies[]!).
|
|
# Dokument patří studii X na úrovni L, pokud má scope
|
|
# "L|X|..." — tj. byl reálně natažen reportem té studie/úrovně.
|
|
# studies[] je jen M:N reference (kam všude je přilinkovaný)
|
|
# a pro výběr „TMF studie X" se NEpoužívá.
|
|
#
|
|
# RYCHLÉ NASTAVENÍ (nahoře): STUDIES + OUTPUT_ROOT_TMPL + přepínače
|
|
# EXPORT (True/False u study/country/site).
|
|
#
|
|
# Stahuje jen deleted=False, downloaded=True, placeholder!=True,
|
|
# se seaweed_path. Idempotentní — existující soubory přeskakuje
|
|
# (resume), takže lze pouštět opakovaně pro „aktuální verzi".
|
|
#
|
|
# v1.2: výběr podle scopes[] (ne studies[]); více studií najednou
|
|
# (seznam STUDIES). (v1.0/v1.1 v TRASH/.)
|
|
#
|
|
# Spuštění:
|
|
# & "...\.venv\Scripts\python.exe" "...\export_from_seaweed_v1.2.py"
|
|
# ============================================================
|
|
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from pymongo import MongoClient, ASCENDING
|
|
|
|
# ====================================================================
|
|
# RYCHLÉ NASTAVENÍ
|
|
# ====================================================================
|
|
# Které studie sestavit (podle scopes[]). Klidně víc najednou.
|
|
STUDIES = [
|
|
"77242113UCO3001",
|
|
"77242113UCO3002",
|
|
"77242113CRD3001",
|
|
"42847922MDD3003",
|
|
]
|
|
|
|
# Kam: {study} se nahradí kódem studie. Pod tím vzniknou podadresáře
|
|
# dle úrovně (STUDY / COUNTRY / číslo centra).
|
|
OUTPUT_ROOT_TMPL = r"U:\{study}"
|
|
|
|
# Co stáhnout — přepni True/False:
|
|
EXPORT = {
|
|
"study": True, # -> <root>\STUDY\<Type>\<Subtype>\...
|
|
"country": True, # -> <root>\COUNTRY\<Type>\<Subtype>\...
|
|
"site": True, # -> <root>\<číslo centra>\<Type>\<Subtype>\...
|
|
}
|
|
|
|
OVERWRITE = False # True = přepsat i existující soubory
|
|
# ====================================================================
|
|
|
|
MONGO_URI = "mongodb://192.168.1.76:27017"
|
|
MONGO_DB = "VTMF"
|
|
MONGO_COLL = "documents"
|
|
|
|
BAD_CHARS_RE = re.compile(r"[<>:\"/\\|?*\x00-\x1f�]")
|
|
|
|
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
|
|
|
|
def clean(s):
|
|
s = BAD_CHARS_RE.sub("_", str(s or ""))
|
|
s = re.sub(r"\s+", " ", s)
|
|
s = re.sub(r"_{2,}", "_", s)
|
|
return s.strip(" ._")
|
|
|
|
|
|
def filename_for(doc):
|
|
"""'YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>'."""
|
|
ext = Path(doc.get("seaweed_path", "")).suffix
|
|
date = doc.get("date") or ""
|
|
date_prefix = (date + " ") if date else ""
|
|
version = f" [{doc['version']}]" if doc.get("version") else ""
|
|
desc = clean(doc.get("desc")) or clean(doc.get("vtmf"))
|
|
return f"{date_prefix}{desc} [{doc['vtmf']}]{version}{ext}"
|
|
|
|
|
|
def base_dirs_for(doc, level, root):
|
|
"""Seznam základních podadresářů (pod root) pro daný dokument.
|
|
study -> [STUDY], country -> [COUNTRY], site -> [<každé centrum>]."""
|
|
if level == "study":
|
|
return [root / "STUDY"]
|
|
if level == "country":
|
|
return [root / "COUNTRY"]
|
|
sites = [clean(s) for s in doc.get("sites", []) if clean(s)]
|
|
return [root / s for s in sites] if sites else [root / "SITE_nezname"]
|
|
|
|
|
|
def target_paths(doc, level, root):
|
|
fname = filename_for(doc)
|
|
typ = clean(doc.get("type")) or "_"
|
|
sub = clean(doc.get("subtype")) or "_"
|
|
return [base / typ / sub / fname for base in base_dirs_for(doc, level, root)]
|
|
|
|
|
|
def export_study_level(coll, study, level, root):
|
|
# příslušnost ke studii/úrovni přes scope "level|study|..."
|
|
scope_prefix = re.escape(f"{level}|{study}|")
|
|
q = {"scopes": {"$regex": f"^{scope_prefix}"},
|
|
"deleted": False, "downloaded": True,
|
|
"placeholder": {"$ne": True}, "seaweed_path": {"$ne": None}}
|
|
docs = list(coll.find(q).sort([("vtmf", ASCENDING), ("version", ASCENDING)]))
|
|
log(f" [{level}] dokumentů: {len(docs)}")
|
|
if not docs:
|
|
return 0, 0, 0, 0
|
|
|
|
written = skipped = failed = 0
|
|
total_bytes = 0
|
|
for n, doc in enumerate(docs, 1):
|
|
dests = target_paths(doc, level, root)
|
|
need = dests if OVERWRITE else [d for d in dests if not d.exists()]
|
|
if not need:
|
|
skipped += len(dests)
|
|
continue
|
|
try:
|
|
with urllib.request.urlopen(doc["seaweed_url"], timeout=120) as r:
|
|
data = r.read()
|
|
for dest in need:
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(data)
|
|
written += 1
|
|
total_bytes += len(data)
|
|
skipped += len(dests) - len(need)
|
|
kb = len(data) / 1024
|
|
size = f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB"
|
|
extra = f" (+{len(need)} kopií)" if len(need) > 1 else ""
|
|
log(f" [{n}/{len(docs)}] {need[0].relative_to(root)} ({size}){extra}")
|
|
except Exception as e:
|
|
failed += 1
|
|
log(f" [!] {doc['_id']}: {e}")
|
|
return written, skipped, failed, total_bytes
|
|
|
|
|
|
def main():
|
|
coll = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)[MONGO_DB][MONGO_COLL]
|
|
levels = [lvl for lvl in ("study", "country", "site") if EXPORT.get(lvl)]
|
|
log(f"[i] Studie: {', '.join(STUDIES)}")
|
|
log(f"[i] Úrovně: {', '.join(levels) if levels else '(žádná)'}")
|
|
log(f"[i] Cíl: {OUTPUT_ROOT_TMPL}\n")
|
|
if not levels:
|
|
sys.exit(0)
|
|
|
|
gw = gs = gf = gb = 0
|
|
for study in STUDIES:
|
|
root = Path(OUTPUT_ROOT_TMPL.format(study=study))
|
|
# má studie přes scopes vůbec něco?
|
|
has = coll.count_documents({"scopes": {"$regex": f"^[a-z]+\\|{re.escape(study)}\\|"}})
|
|
log(f"=== {study} -> {root} (scoped dokumentů: {has}) ===")
|
|
if not has:
|
|
log(f"[i] {study}: přes scopes nic — pipeline pro tuto studii "
|
|
f"zatím neproběhla, přeskakuji.\n")
|
|
continue
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
for level in levels:
|
|
w, s, f, b = export_study_level(coll, study, level, root)
|
|
gw += w; gs += s; gf += f; gb += b
|
|
log("")
|
|
|
|
mb = gb / 1024 / 1024
|
|
log(f"[ok] Hotovo: {gw} souborů zapsáno ({mb:.1f} MB), "
|
|
f"{gs} přeskočeno (už existuje), {gf} chyb.")
|
|
sys.exit(1 if gf else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|