administrator/janssen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

z230

Browse Source
This commit is contained in:
vladimir.buzalka
2026-06-15 16:43:46 +02:00
parent 495cf8da21
commit 91f1825109
14 changed files with 547 additions and 1247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VTMFDownloadFiles/TRASH/download_state.csv.imported
+11
View File
@@ -0,0 +1,11 @@
vtmf,result,file,timestamp
VTMF-19077748,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Third Parties\Third Party Oversight\2023-05-16 Teckro SOP list associated with work performed with IPE [VTMF-19077748] [v1.0].pdf,2026-06-12T13:32:43
VTMF-18982659,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Third Parties\Third Party Oversight\2023-03-22 List of QD Solutions SOP list associated with work performed with IPE [VTMF-18982659] [v1.0].pdf,2026-06-12T13:32:46
VTMF-19008643,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Central Trial Documents\Subject Documents\2023-05-03 Janssen Institution Level_Cover Letter_Greenphire US Site Stipend and Travel Assistance Program Acknowledgement Form [VTMF-19008643] [v1.0].pdf,2026-06-12T13:32:50
VTMF-19008639,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Central Trial Documents\Subject Documents\2026-02-04 Janssen Institution Level_Cover Letter_Greenphire ClinCard Message Templates V10 Jan2022 [VTMF-19008639] [v2.0].pdf,2026-06-12T13:32:57
VTMF-22648123,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Trial Management\Trial Oversight\2025-02-05 CSC Timely Filing Metrics Review_4Q2024_V#1 [VTMF-22648123] [v1.0].pdf,2026-06-12T13:33:01
VTMF-23162872,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Trial Management\Trial Oversight\2025-04-07 CSC Timely Filing Metrics Review_1Q2025_V#1 [VTMF-23162872] [v1.0].pdf,2026-06-12T13:33:05
VTMF-22056820,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Trial Management\Trial Oversight\2024-11-12 CSC Timely Filing Metrics Review_Aug-Oct 2024_V#1 [VTMF-22056820] [v1.0].pdf,2026-06-12T13:33:08
VTMF-24352418,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Trial Management\Trial Oversight\2025-10-27 CSC Timely Filing Metrics Review - 3Q2025 [VTMF-24352418] [v1.0].pdf,2026-06-12T13:33:12
VTMF-5690421,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Central Trial Documents\Trial Documents\IB_en_JNJ-42847922_2025_ed14 [VTMF-5690421] [v18.0].pdf,2026-06-12T13:33:16
VTMF-22657978,ok,U:\Dropbox\!!!Days\Downloads Z230\VTMF-77242113UCO3001\Central Trial Documents\Trial Documents\IB_Ed[13]-Addendum-JNJ-42847922-AAA-1524042 [VTMF-22657978] [v1.0].pdf,2026-06-12T13:33:20
VTMFDownloadFiles/TRASH/export_from_seaweed_v1.0.py
+111
View File
@@ -0,0 +1,111 @@
# ============================================================
# export_from_seaweed_v1.0.py
# Verze: 1.0
# Datum: 2026-06-15
# Popis: Sestaví na disk strom dokumentů jedné úrovně studie
# ze SeaweedFS (zdroj pravdy je Mongo VTMF.documents +
# objekty v SeaweedFS Fileru). Pojmenování podle původních
# Dropbox pravidel (pipeline v1.5):
#
# <OUTPUT_ROOT>\<Type>\<Subtype>\
# "YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>"
#
# Datum/verze se vynechají, když chybí. Stahuje jen
# deleted=False, downloaded=True, placeholder!=True,
# se seaweed_path. Idempotentní — existující soubory
# přeskakuje (resume), takže lze pouštět opakovaně pro
# „aktuální verzi" čehokoli.
#
# Konfigurace (konstanty níže): STUDY, LEVEL, OUTPUT_ROOT.
# Pro jinou úroveň/studii stačí změnit tyto tři.
#
# Spuštění:
# & "...\.venv\Scripts\python.exe" "...\export_from_seaweed_v1.0.py"
# ============================================================
import re
import sys
import urllib.request
from pathlib import Path
from pymongo import MongoClient, ASCENDING
# --- Konfigurace -------------------------------------------------------
STUDY = "77242113UCO3001"
LEVEL = "study" # study | country | site
OUTPUT_ROOT = Path(r"U:\77242113UCO3001_STUDYLEVEL")
MONGO_URI = "mongodb://192.168.1.76:27017"
MONGO_DB = "VTMF"
MONGO_COLL = "documents"
OVERWRITE = False # True = přepsat i existující soubory
BAD_CHARS_RE = re.compile(r"[<>:\"/\\|?*\x00-\x1f�]")
def log(msg):
print(msg, flush=True)
def clean(s):
s = BAD_CHARS_RE.sub("_", str(s or ""))
s = re.sub(r"\s+", " ", s)
s = re.sub(r"_{2,}", "_", s)
return s.strip(" ._")
def target_path(doc):
"""<OUTPUT_ROOT>\\<Type>\\<Subtype>\\
'YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>'."""
ext = Path(doc.get("seaweed_path", "")).suffix
date = doc.get("date") or ""
date_prefix = (date + " ") if date else ""
version = f" [{doc['version']}]" if doc.get("version") else ""
desc = clean(doc.get("desc")) or clean(doc.get("vtmf"))
filename = f"{date_prefix}{desc} [{doc['vtmf']}]{version}{ext}"
typ = clean(doc.get("type")) or "_"
sub = clean(doc.get("subtype")) or "_"
return OUTPUT_ROOT / typ / sub / filename
def main():
coll = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)[MONGO_DB][MONGO_COLL]
q = {"level": LEVEL, "deleted": False, "downloaded": True,
"placeholder": {"$ne": True}, "seaweed_path": {"$ne": None}}
docs = list(coll.find(q).sort([("vtmf", ASCENDING), ("version", ASCENDING)]))
log(f"[i] {LEVEL}-level dokumentů studie {STUDY}: {len(docs)}")
log(f"[i] Cíl: {OUTPUT_ROOT}\n")
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
written = skipped = failed = 0
total_bytes = 0
for n, doc in enumerate(docs, 1):
dest = target_path(doc)
if dest.exists() and not OVERWRITE:
skipped += 1
continue
try:
with urllib.request.urlopen(doc["seaweed_url"], timeout=120) as r:
data = r.read()
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(data)
written += 1
total_bytes += len(data)
kb = len(data) / 1024
size = f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB"
log(f"[{n}/{len(docs)}] {dest.relative_to(OUTPUT_ROOT)} ({size})")
except Exception as e:
failed += 1
log(f"[!] {doc['_id']}: {e}")
mb = total_bytes / 1024 / 1024
log(f"\n[ok] Hotovo: {written} zapsáno ({mb:.1f} MB), "
f"{skipped} přeskočeno (už existuje), {failed} chyb.")
sys.exit(1 if failed else 0)
if __name__ == "__main__":
main()
VTMFDownloadFiles/TRASH/export_from_seaweed_v1.1.py
+167
View File
@@ -0,0 +1,167 @@
# ============================================================
# export_from_seaweed_v1.1.py
# Verze: 1.1
# Datum: 2026-06-15
# Popis: Sestaví na disk strom dokumentů studie ze SeaweedFS
# (zdroj pravdy = Mongo VTMF.documents + objekty ve Fileru).
# Pojmenování podle původních Dropbox pravidel (pipeline v1.5):
#
# <základní podadresář>\<Type>\<Subtype>\
# "YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>"
#
# Základní podadresář pod OUTPUT_ROOT podle úrovně:
# study -> STUDY
# country -> COUNTRY
# site -> <číslo centra> (dokument se zapíše do složky
# KAŽDÉHO centra, do kterého je referencovaný)
#
# RYCHLÉ NASTAVENÍ (nahoře): STUDY + OUTPUT_ROOT + přepínače EXPORT
# (True/False u study/country/site).
#
# Stahuje jen deleted=False, downloaded=True, placeholder!=True,
# se seaweed_path. Idempotentní — existující soubory přeskakuje
# (resume), takže lze pouštět opakovaně pro „aktuální verzi".
#
# v1.1: přepínače úrovní (True/False) + základní podadresář dle úrovně
# (STUDY / COUNTRY / číslo centra); site-level se kopíruje do
# složky každého centra. (v1.0 v TRASH/ — jen study, pevně.)
#
# Spuštění:
# & "...\.venv\Scripts\python.exe" "...\export_from_seaweed_v1.1.py"
# ============================================================
import re
import sys
import urllib.request
from pathlib import Path
from pymongo import MongoClient, ASCENDING
# ====================================================================
# RYCHLÉ NASTAVENÍ
# ====================================================================
STUDY = "77242113UCO3001" # která studie
OUTPUT_ROOT = Path(r"U:\77242113UCO3001") # kam (pod ní vzniknou
# podadresáře dle úrovně)
# Co stáhnout — přepni True/False:
EXPORT = {
"study": True, # -> OUTPUT_ROOT\STUDY\<Type>\<Subtype>\...
"country": False, # -> OUTPUT_ROOT\COUNTRY\<Type>\<Subtype>\...
"site": False, # -> OUTPUT_ROOT\<číslo centra>\<Type>\<Subtype>\...
}
OVERWRITE = False # True = přepsat i existující soubory
# ====================================================================
MONGO_URI = "mongodb://192.168.1.76:27017"
MONGO_DB = "VTMF"
MONGO_COLL = "documents"
BAD_CHARS_RE = re.compile(r"[<>:\"/\\|?*\x00-\x1f�]")
def log(msg):
print(msg, flush=True)
def clean(s):
s = BAD_CHARS_RE.sub("_", str(s or ""))
s = re.sub(r"\s+", " ", s)
s = re.sub(r"_{2,}", "_", s)
return s.strip(" ._")
def filename_for(doc):
"""'YYYY-MM-DD Description [VTMF-xxx] [v1.0].<přípona>'."""
ext = Path(doc.get("seaweed_path", "")).suffix
date = doc.get("date") or ""
date_prefix = (date + " ") if date else ""
version = f" [{doc['version']}]" if doc.get("version") else ""
desc = clean(doc.get("desc")) or clean(doc.get("vtmf"))
return f"{date_prefix}{desc} [{doc['vtmf']}]{version}{ext}"
def base_dirs_for(doc, level):
"""Seznam základních podadresářů (pod OUTPUT_ROOT) pro daný dokument.
study -> [STUDY], country -> [COUNTRY], site -> [<každé centrum>]."""
if level == "study":
return [OUTPUT_ROOT / "STUDY"]
if level == "country":
return [OUTPUT_ROOT / "COUNTRY"]
# site: jedna kopie do složky každého centra
sites = [clean(s) for s in doc.get("sites", []) if clean(s)]
if not sites:
return [OUTPUT_ROOT / "SITE_nezname"]
return [OUTPUT_ROOT / s for s in sites]
def target_paths(doc, level):
fname = filename_for(doc)
typ = clean(doc.get("type")) or "_"
sub = clean(doc.get("subtype")) or "_"
return [base / typ / sub / fname for base in base_dirs_for(doc, level)]
def export_level(coll, level):
q = {"level": level, "studies": STUDY, "deleted": False,
"downloaded": True, "placeholder": {"$ne": True},
"seaweed_path": {"$ne": None}}
docs = list(coll.find(q).sort([("vtmf", ASCENDING), ("version", ASCENDING)]))
log(f"\n=== {level.upper()} === dokumentů: {len(docs)}")
if not docs:
if level != "study":
log(f"[i] (žádné {level}-level dokumenty v Mongo — pipeline pro "
f"tuto úroveň zatím možná neproběhla)")
return 0, 0, 0, 0
written = skipped = failed = 0
total_bytes = 0
for n, doc in enumerate(docs, 1):
dests = target_paths(doc, level)
need = dests if OVERWRITE else [d for d in dests if not d.exists()]
if not need:
skipped += len(dests)
continue
try:
with urllib.request.urlopen(doc["seaweed_url"], timeout=120) as r:
data = r.read()
for dest in need:
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(data)
written += 1
total_bytes += len(data)
skipped += len(dests) - len(need)
kb = len(data) / 1024
size = f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB"
extra = f" (+{len(need)} kopií)" if len(need) > 1 else ""
log(f"[{n}/{len(docs)}] {need[0].relative_to(OUTPUT_ROOT)} ({size}){extra}")
except Exception as e:
failed += 1
log(f"[!] {doc['_id']}: {e}")
return written, skipped, failed, total_bytes
def main():
coll = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)[MONGO_DB][MONGO_COLL]
levels = [lvl for lvl in ("study", "country", "site") if EXPORT.get(lvl)]
log(f"[i] Studie: {STUDY}")
log(f"[i] Cíl: {OUTPUT_ROOT}")
log(f"[i] Úrovně: {', '.join(levels) if levels else '(žádná — zapni něco v EXPORT)'}")
if not levels:
sys.exit(0)
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
tw = ts = tf = tb = 0
for level in levels:
w, s, f, b = export_level(coll, level)
tw += w; ts += s; tf += f; tb += b
mb = tb / 1024 / 1024
log(f"\n[ok] Hotovo: {tw} souborů zapsáno ({mb:.1f} MB), "
f"{ts} přeskočeno (už existuje), {tf} chyb.")
sys.exit(1 if tf else 0)
if __name__ == "__main__":
main()
VTMFDownloadFiles/TRASH/seaweed_backfill_v1.1.py
+140
View File
@@ -0,0 +1,140 @@
# ============================================================
# seaweed_backfill_v1.1.py
# Verze: 1.1
# Datum: 2026-06-15
# v1.1: retry 3x s 5s pauzou při HTTP 5xx (přechodná chyba serveru)
# Popis: Jednorázový backfill — nahraje do SeaweedFS Filer
# všechny dokumenty z VTMF.documents, které jsou na disku
# (downloaded=True, file!=null) ale ještě nemají seaweed_path.
# Placeholdery a záznamy bez souboru přeskočí.
# Lze spustit opakovaně — HEAD check zajistí dedup,
# přerušení kdykoli naváže příště.
# ============================================================
import hashlib
import mimetypes
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
from pymongo import MongoClient, ASCENDING
MONGO_URI = "mongodb://192.168.1.76:27017"
MONGO_DB = "VTMF"
MONGO_COLL = "documents"
SEAWEED_FILER = "http://192.168.1.50:8888"
SEAWEED_PREFIX = "/vtmf-documents"
def log(msg):
print(msg, flush=True)
def sw_path(sha256):
return f"{SEAWEED_PREFIX}/{sha256[:2]}/{sha256[2:4]}/{sha256}"
MAX_ATTEMPTS = 3
RETRY_PAUSE = 5 # sekund mezi pokusy při 5xx
def seaweed_store(data, mime="application/octet-stream"):
"""HEAD check + PUT s retry při 5xx. Vrací (path, url, uploaded)."""
sha256 = hashlib.sha256(data).hexdigest()
path = sw_path(sha256)
url = SEAWEED_FILER + path
try:
urllib.request.urlopen(
urllib.request.Request(url, method="HEAD"), timeout=10)
return path, url, False # dedup hit
except urllib.error.HTTPError as e:
if e.code != 404:
raise
last_err = None
for attempt in range(1, MAX_ATTEMPTS + 1):
try:
urllib.request.urlopen(
urllib.request.Request(url, data=data, method="PUT",
headers={"Content-Type": mime}),
timeout=120)
return path, url, True
except urllib.error.HTTPError as e:
if e.code < 500:
raise # 4xx — nema smysl opakovat
last_err = e
if attempt < MAX_ATTEMPTS:
log(f" [!] HTTP {e.code} (pokus {attempt}/{MAX_ATTEMPTS}), čekám {RETRY_PAUSE}s...")
time.sleep(RETRY_PAUSE)
raise last_err
def main():
client = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)
client.admin.command("ping")
coll = client[MONGO_DB][MONGO_COLL]
log(f"[ok] Mongo připojeno: {MONGO_URI} / {MONGO_DB}.{MONGO_COLL}")
query = {
"downloaded": True,
"placeholder": {"$ne": True},
"seaweed_path": None,
"file": {"$ne": None},
}
todo = list(coll.find(query).sort([("vtmf", ASCENDING), ("version", ASCENDING)]))
log(f"[i] Ke zpracování: {len(todo)} dokumentů\n")
uploaded = dedup = skipped = failed = 0
for n, doc in enumerate(todo, 1):
key = doc["_id"]
path = doc.get("file")
if not path or not Path(path).exists():
log(f"[{n}/{len(todo)}] {key} [!] Soubor nenalezen na disku — přeskočeno.")
skipped += 1
continue
try:
data = Path(path).read_bytes()
size_kb = len(data) / 1024
size_str = f"{size_kb:.0f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB"
log(f"[{n}/{len(todo)}] {key} ({size_str} {Path(path).suffix.lstrip('.').upper()}) {doc.get('desc', '')[:60]}")
mime = mimetypes.guess_type(path)[0] or "application/octet-stream"
sha256_hex = hashlib.sha256(data).hexdigest()
sw_p, sw_url, was_new = seaweed_store(data, mime)
coll.update_one({"_id": key}, {"$set": {
"sha256": sha256_hex,
"seaweed_path": sw_p,
"seaweed_url": sw_url,
"seaweed_synced_at": datetime.now(),
}})
if was_new:
uploaded += 1
log(f" [ok] Nahráno ({size_str}) → {sw_p}")
else:
dedup += 1
log(f" [i] Dedup hit ({size_str}) → {sw_p}")
except Exception as e:
failed += 1
log(f" [!] Chyba: {e}")
log(f"\n{'='*60}")
log(f" Hotovo: {uploaded} nahráno, {dedup} dedup, "
f"{skipped} bez souboru, {failed} chyb.")
log(f"{'='*60}")
sys.exit(1 if failed else 0)
if __name__ == "__main__":
main()
VTMFDownloadFiles/TRASH/test_seaweed.py
+46
View File
@@ -0,0 +1,46 @@
"""Rychlý test SeaweedFS Filer (port 8888) — PUT / HEAD / GET / DELETE."""
import hashlib
import urllib.error
import urllib.request
FILER = "http://192.168.1.50:8888"
PAYLOAD = b"SeaweedFS VTMF test " + b"x" * 1000
SHA256 = hashlib.sha256(PAYLOAD).hexdigest()
PATH = f"/vtmf-documents/_test/{SHA256[:8]}"
URL = FILER + PATH
def req(method, data=None):
r = urllib.request.Request(URL, method=method, data=data,
headers={"Content-Type": "text/plain"} if data else {})
try:
with urllib.request.urlopen(r, timeout=10) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as e:
return e.code, b""
print(f"Filer: {FILER}")
print(f"Path: {PATH}\n")
status, _ = req("PUT", PAYLOAD)
assert status in (200, 201), f"PUT selhal: {status}"
print(f"[ok] PUT → {status}")
status, _ = req("HEAD")
assert status == 200, f"HEAD selhal: {status}"
print(f"[ok] HEAD → {status}")
status, body = req("GET")
assert status == 200 and body == PAYLOAD, f"GET selhal: {status}, délka={len(body)}"
print(f"[ok] GET → {status}, {len(body)} B")
status, _ = req("DELETE")
assert status in (200, 204), f"DELETE selhal: {status}"
print(f"[ok] DELETE → {status}")
status, _ = req("HEAD")
assert status == 404, f"Po DELETE HEAD vrátil {status}, čekal 404"
print(f"[ok] HEAD po DELETE → 404 (soubor odstraněn)\n")
print("SeaweedFS Filer OK.")