z230
This commit is contained in:
@@ -0,0 +1,937 @@
|
||||
# ============================================================
|
||||
# vtmf_pipeline_v1.6.py
|
||||
# Verze: 1.6
|
||||
# Datum: 2026-06-15
|
||||
# Popis: Kompletní workflow V-TMF (J&J Veeva Vault) pro studii
|
||||
# 77242113UCO3001 přes VŠECHNY TŘI ÚROVNĚ dokumentů
|
||||
# (STUDY / COUNTRY / SITE). Jeden běh udělá pro každý
|
||||
# report ze seznamu REPORTS:
|
||||
# 1) login do Vaultu (persistentní session + ruční 2FA),
|
||||
# 2) export reportu do Excelu (Data Only) do WhatToDownload/,
|
||||
# 3) parse + scoped sync do MongoDB (db VTMF, kolekce
|
||||
# documents; klíč _id = "číslo|verze"),
|
||||
# a nakonec jeden průchod stažení všech dosud nestažených
|
||||
# dokumentů PŘÍMO do SeaweedFS (žádný Dropbox/disk).
|
||||
#
|
||||
# ZÁSADNÍ ZMĚNY proti v1.5:
|
||||
#
|
||||
# • Hierarchie dokumentů ve VTMF je STUDY -> COUNTRY -> SITE.
|
||||
# Dokument je do studií/zemí/center jen REFERENCOVANÝ (M:N) —
|
||||
# např. Master Confidentiality Agreement v nemocnici je jeden
|
||||
# dokument referencovaný do všech studií i center té nemocnice.
|
||||
# Proto: jeden dokument = jeden záznam = jeden SeaweedFS objekt;
|
||||
# příslušnost je jen metadatová pole studies[]/countries[]/sites[].
|
||||
#
|
||||
# • REPORTS = seznam (level, study, country, url). Country i site
|
||||
# report filtrují jen na zemi (CZ), ne na studii -> při ukládání
|
||||
# se row bere jen pokud cílová studie je v jeho Study sloupci
|
||||
# (prakticky no-op, vše vrácené UCO3001 obsahuje).
|
||||
#
|
||||
# • Zobecněný parser: study report má 15 sloupců (+ Document Date),
|
||||
# country/site mají 17 (+ Created By, Study Country, Site; bez
|
||||
# Document Date). Sloupce se hledají podle NÁZVU, datum má
|
||||
# fallback Document Date -> Approval Complete Date -> Version
|
||||
# Creation Date. Study/Study Country/Site se parsují na pole.
|
||||
#
|
||||
# • Scoped sync: mazání už NEkouká na celou kolekci. Každý report
|
||||
# má scope = (level|study|country); dokument nese pole scopes[].
|
||||
# Když z reportu daného scope zmizí, scope se odebere; teprve
|
||||
# když nemá žádný scope -> deleted=True.
|
||||
#
|
||||
# • Evidence reportů: kolekce report_runs (level, study, country,
|
||||
# url, exported_at, file, row_count, doc_keys).
|
||||
#
|
||||
# • ÚLOŽIŠTĚ = JEN SeaweedFS, klíč číslo dokumentu + verze:
|
||||
# /vtmf-documents/<vtmf>/<verze>.<přípona>
|
||||
# Žádné ukládání dokumentů na disk/Dropbox — stahují se přes
|
||||
# dočasný soubor Playwrightu rovnou do Fileru. SHA-256 se počítá
|
||||
# a ukládá do Mongo jen jako kontrolní součet. (Aktuální verzi
|
||||
# čehokoli do Dropboxu zařídí samostatný export skript ze SeaweedFS.)
|
||||
#
|
||||
# Heslo se NIKDY nedává natvrdo do skriptu — čte se z .env
|
||||
# v rootu projektu Janssen (VAULT_USER / VAULT_PASS).
|
||||
#
|
||||
# Migrace stávajících study-level dat na toto schéma: migrate_to_v16.py
|
||||
# Předchůdce: vtmf_pipeline_v1.5 (v TRASH/).
|
||||
# ============================================================
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
|
||||
from pymongo import MongoClient, ASCENDING
|
||||
|
||||
# --- Konfigurace -------------------------------------------------------
|
||||
|
||||
LOGIN_URL = ("https://fedlogin.jnj.com/idp/eyJ2c2lkIjoiam5qX3ZlZXZhIn0/"
|
||||
"startSSO.ping?PartnerSpId=janssenetmf.veevavault.com"
|
||||
"&IdpAdapterId=CompIWALDAPEXTFORM"
|
||||
"&TargetResource=https%3A%2F%2Fvtmf.veevavault.com%2F")
|
||||
|
||||
# Studie, jejíž TMF stavíme (cíl ořezu country/site reportů).
|
||||
TARGET_STUDY = "77242113UCO3001"
|
||||
|
||||
# ====================================================================
|
||||
# SEZNAM REPORTŮ KE ZPRACOVÁNÍ
|
||||
# --------------------------------------------------------------------
|
||||
# Každý řádek = jeden report. Pole:
|
||||
# enabled = True/False -> přepni na False a report se v dalším běhu
|
||||
# NEnačte (zůstane v seznamu jako dokumentace)
|
||||
# name = popisek do logu (co to je za report)
|
||||
# level = "study" | "country" | "site" (úroveň + scope)
|
||||
# study = kód cílové studie (scope + ořez na tuto studii)
|
||||
# country = země scope (None u study-level)
|
||||
# url = přímý odkaz na report viewer ve Vaultu
|
||||
#
|
||||
# Přidání jiné studie = prostě dopiš další 3 řádky s jejím kódem
|
||||
# a URL; běh je zpracuje vedle stávajících.
|
||||
# ====================================================================
|
||||
REPORTS = [
|
||||
{"enabled": True, "name": "UCO3001 — STUDY level",
|
||||
"level": "study", "study": TARGET_STUDY, "country": None,
|
||||
"url": "https://vtmf.veevavault.com/ui/#reporting/viewer/"
|
||||
"0RP000000000182?study__v%2C%2C%2CIN=0ST000000137008"},
|
||||
|
||||
{"enabled": True, "name": "UCO3001 — COUNTRY level (Czech Republic)",
|
||||
"level": "country", "study": TARGET_STUDY, "country": "Czech Republic",
|
||||
"url": "https://vtmf.veevavault.com/ui/#reporting/viewer/"
|
||||
"0RP000000000319?study_country__v%2C%2C%2CIN=0SC00000017T056"},
|
||||
|
||||
{"enabled": False, "name": "UCO3001 — SITE level (all sites in Czech Republic)",
|
||||
"level": "site", "study": TARGET_STUDY, "country": "Czech Republic",
|
||||
"url": "https://vtmf.veevavault.com/ui/#reporting/viewer/"
|
||||
"0RP000000000762?study_country__v%2C%2C%2CEQ=0SC00000017T056"},
|
||||
]
|
||||
|
||||
VAULT_UI_PATTERN = "**vtmf.veevavault.com/ui**" # úspěšný vstup do Vaultu
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
PROFILE_DIR = SCRIPT_DIR / "vault_profile" # perzistentní session
|
||||
ENV_FILE = SCRIPT_DIR.parent / ".env" # root projektu Janssen
|
||||
DEBUG_DIR = SCRIPT_DIR / "debug" # diagnostické výstupy
|
||||
EXCEL_DIR = SCRIPT_DIR / "WhatToDownload" # stažené reporty (jen Excel)
|
||||
PROCESSED_DIR = EXCEL_DIR / "Zpracovano" # archiv zpracovaných
|
||||
|
||||
MONGO_URI = "mongodb://192.168.1.76:27017"
|
||||
MONGO_DB = "VTMF"
|
||||
MONGO_COLL = "documents"
|
||||
RUNS_COLL = "report_runs"
|
||||
|
||||
# Kolik dokumentů stáhnout v tomto běhu (None = všechny zbývající)
|
||||
LIMIT = None
|
||||
# Pole, jejichž změny se verzují do history[]
|
||||
TRACKED_FIELDS = ("name", "status", "type", "subtype", "classification",
|
||||
"desc", "date", "url", "studies", "countries", "sites",
|
||||
"level")
|
||||
|
||||
MAX_ATTEMPTS = 2 # pokusy na jeden dokument
|
||||
RETRY_PAUSE_MS = 5000 # pauza před opakováním
|
||||
BETWEEN_DOCS_MS = 500 # pauza mezi dokumenty
|
||||
|
||||
SEAWEED_FILER = "http://192.168.1.50:8888"
|
||||
SEAWEED_PREFIX = "/vtmf-documents"
|
||||
|
||||
|
||||
class PlaceholderDocument(Exception):
|
||||
"""Dokument existuje jen jako placeholder — "This placeholder has no content"."""
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
def load_env_file(path):
|
||||
"""Načte KEY=VALUE řádky z .env do os.environ.
|
||||
Už nastavené env proměnné mají přednost, .env je nepřepisuje."""
|
||||
if not path.exists():
|
||||
log(f"[!] .env nenalezen: {path}")
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key, value = key.strip(), value.strip().strip('"').strip("'")
|
||||
if value and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
ENV_SECTION_HEADER = "# --- Veeva Vault (J&J V-TMF) — VTMFDownloadFiles/download_vault ---"
|
||||
ENV_KEYS = ("VAULT_USER", "VAULT_PASS")
|
||||
|
||||
|
||||
def ensure_credentials():
|
||||
"""Načte .env; pokud VAULT_USER/VAULT_PASS chybí, založí/doplní
|
||||
v .env šablonu, vyzve uživatele k doplnění a ukončí skript."""
|
||||
load_env_file(ENV_FILE)
|
||||
if all(os.environ.get(k) for k in ENV_KEYS):
|
||||
return
|
||||
|
||||
existing = ENV_FILE.read_text(encoding="utf-8") if ENV_FILE.exists() else ""
|
||||
missing_lines = [f"{k}=" for k in ENV_KEYS
|
||||
if not re.search(rf"^\s*{k}\s*=", existing, re.M)]
|
||||
|
||||
if not ENV_FILE.exists():
|
||||
ENV_FILE.write_text(
|
||||
"# .env — lokální přihlašovací údaje (NEVERZOVAT, je v .gitignore)\n\n"
|
||||
+ ENV_SECTION_HEADER + "\n"
|
||||
+ "\n".join(missing_lines) + "\n",
|
||||
encoding="utf-8")
|
||||
log(f"[i] Založil jsem nový .env: {ENV_FILE}")
|
||||
elif missing_lines:
|
||||
with open(ENV_FILE, "a", encoding="utf-8") as f:
|
||||
f.write("\n" + ENV_SECTION_HEADER + "\n"
|
||||
+ "\n".join(missing_lines) + "\n")
|
||||
log(f"[i] Doplnil jsem chybějící řádky do .env: {ENV_FILE}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(" CHYBÍ PŘIHLAŠOVACÍ ÚDAJE.")
|
||||
print(f" Doplň VAULT_USER a VAULT_PASS do souboru:")
|
||||
print(f" {ENV_FILE}")
|
||||
print(" a spusť skript znovu.")
|
||||
print("=" * 60)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# --- Parsování Excelu --------------------------------------------------
|
||||
|
||||
HYPERLINK_RE = re.compile(r'HYPERLINK\("([^"]+)"\s*,\s*"([^"]+)"\)')
|
||||
VERSION_RE = re.compile(r"\((v[^)]+)\)\s*$")
|
||||
DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
||||
# nepovolené znaky názvů + řídicí znaky + unicode artefakt �
|
||||
BAD_CHARS_RE = re.compile(r"[<>:\"/\\|?*\x00-\x1f�]")
|
||||
|
||||
|
||||
def clean_text(s):
|
||||
"""Očistí string na rozumný název (bez nepovolených znaků)."""
|
||||
s = BAD_CHARS_RE.sub("_", str(s))
|
||||
s = re.sub(r"\s+", " ", s)
|
||||
s = re.sub(r"_{2,}", "_", s)
|
||||
return s.strip(" ._")
|
||||
|
||||
|
||||
def display_text(cell):
|
||||
"""Zobrazený text buňky — u =HYPERLINK vzorce druhý argument."""
|
||||
raw = str(cell.value or "").strip()
|
||||
m = HYPERLINK_RE.search(raw)
|
||||
return m.group(2).strip() if m else raw
|
||||
|
||||
|
||||
def split_multi(text):
|
||||
"""Comma-separated seznam -> list (strip, bez prázdných, dedup pořadí)."""
|
||||
out, seen = [], set()
|
||||
for part in str(text or "").split(","):
|
||||
p = part.strip()
|
||||
if p and p not in seen:
|
||||
seen.add(p)
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def cell_date(cell):
|
||||
"""Z buňky vytáhne datum jako 'YYYY-MM-DD' (datetime i string), nebo ''."""
|
||||
v = cell.value if cell is not None else None
|
||||
if hasattr(v, "strftime"):
|
||||
return v.strftime("%Y-%m-%d")
|
||||
m = DATE_RE.search(str(v or ""))
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def extract_doc_url(raw):
|
||||
"""Z HYPERLINK hodnoty (nebo i rozbité URL) vytáhne čistou doc URL
|
||||
ve tvaru https://<host>/ui/#doc_info/<id>/<major>/<minor>."""
|
||||
m = re.search(r"(https://[^/\"]+/ui/#doc_info/\d+/\d+/\d+)", str(raw))
|
||||
if not m:
|
||||
raise ValueError(f"Nenašel jsem doc URL v: {raw!r}")
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def read_documents_from_excel(path, level):
|
||||
"""Načte dokumenty z .xlsx reportu dané úrovně (study/country/site).
|
||||
Sloupce se hledají podle NÁZVU (study má 15, country/site 17).
|
||||
Document Name/Number jsou =HYPERLINK vzorce -> URL i text regexem.
|
||||
Report má rozbité deklarované rozměry -> přímá iterace řádků."""
|
||||
from openpyxl import load_workbook
|
||||
|
||||
log(f"[i] Parsování reportu ({level}): {path.name}")
|
||||
wb = load_workbook(path, data_only=False) # potřebujeme vzorce
|
||||
ws = wb[wb.sheetnames[0]]
|
||||
|
||||
rows = ws.iter_rows()
|
||||
header = [c.value for c in next(rows)]
|
||||
idx = {h: i for i, h in enumerate(header) if h is not None}
|
||||
|
||||
required = ("Document Number", "Document Name", "Document Status",
|
||||
"Type", "Subtype", "Description", "Study")
|
||||
missing = [c for c in required if c not in idx]
|
||||
if missing:
|
||||
raise RuntimeError(f"V reportu chybí očekávané sloupce: {missing}")
|
||||
|
||||
i_num, i_name = idx["Document Number"], idx["Document Name"]
|
||||
i_status, i_type, i_sub = idx["Document Status"], idx["Type"], idx["Subtype"]
|
||||
i_desc, i_study = idx["Description"], idx["Study"]
|
||||
i_class = idx.get("Classification")
|
||||
i_proc = idx.get("Process Name")
|
||||
i_extsys = idx.get("External System Name")
|
||||
i_created = idx.get("Created By")
|
||||
i_modby = idx.get("Last Modified By")
|
||||
i_verby = idx.get("Version Created By")
|
||||
i_country = idx.get("Study Country")
|
||||
i_site = idx.get("Site")
|
||||
i_date_cols = [idx.get(c) for c in
|
||||
("Document Date", "Approval Complete Date", "Version Creation Date")
|
||||
if idx.get(c) is not None]
|
||||
|
||||
def g(row, i):
|
||||
return display_text(row[i]) if i is not None else ""
|
||||
|
||||
docs, bad = [], []
|
||||
for row in rows:
|
||||
cell = row[i_num]
|
||||
if cell.value is None:
|
||||
continue
|
||||
raw = str(cell.value)
|
||||
m = HYPERLINK_RE.search(raw)
|
||||
if m:
|
||||
url_raw, vtmf = m.group(1), m.group(2)
|
||||
elif cell.hyperlink:
|
||||
url_raw, vtmf = cell.hyperlink.target, raw
|
||||
else:
|
||||
bad.append(raw)
|
||||
continue
|
||||
try:
|
||||
url = extract_doc_url(url_raw)
|
||||
except ValueError:
|
||||
bad.append(raw)
|
||||
continue
|
||||
|
||||
name = display_text(row[i_name])
|
||||
vm = VERSION_RE.search(name)
|
||||
version = vm.group(1) if vm else "v?"
|
||||
|
||||
desc = clean_text(g(row, i_desc))
|
||||
if not desc:
|
||||
desc = clean_text(VERSION_RE.sub("", name))
|
||||
|
||||
date = ""
|
||||
for i_d in i_date_cols:
|
||||
date = cell_date(row[i_d])
|
||||
if date:
|
||||
break
|
||||
|
||||
docs.append({
|
||||
"vtmf": vtmf.strip(),
|
||||
"version": version,
|
||||
"url": url,
|
||||
"level": level,
|
||||
"name": name,
|
||||
"status": g(row, i_status),
|
||||
"type": clean_text(g(row, i_type)),
|
||||
"subtype": clean_text(g(row, i_sub)),
|
||||
"classification": g(row, i_class),
|
||||
"desc": desc,
|
||||
"process_name": g(row, i_proc),
|
||||
"external_system_name": g(row, i_extsys),
|
||||
"created_by": g(row, i_created),
|
||||
"last_modified_by": g(row, i_modby),
|
||||
"version_created_by": g(row, i_verby),
|
||||
"date": date,
|
||||
"studies": split_multi(g(row, i_study)),
|
||||
"countries": split_multi(g(row, i_country)) if i_country is not None else [],
|
||||
"sites": split_multi(g(row, i_site)) if i_site is not None else [],
|
||||
})
|
||||
|
||||
log(f"[i] Načteno {len(docs)} dokumentů"
|
||||
+ (f", {len(bad)} řádků bez použitelné URL (přeskočeno)" if bad else ""))
|
||||
return docs
|
||||
|
||||
|
||||
# --- MongoDB synchronizace ---------------------------------------------
|
||||
|
||||
def doc_key(vtmf, version):
|
||||
return f"{vtmf}|{version}"
|
||||
|
||||
|
||||
def scope_key(report):
|
||||
return f"{report['level']}|{report['study']}|{report.get('country') or ''}"
|
||||
|
||||
|
||||
def get_db():
|
||||
client = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000)
|
||||
client.admin.command("ping")
|
||||
db = client[MONGO_DB]
|
||||
coll = db[MONGO_COLL]
|
||||
coll.create_index([("vtmf", ASCENDING), ("version", ASCENDING)], unique=True)
|
||||
coll.create_index([("deleted", ASCENDING), ("downloaded", ASCENDING)])
|
||||
coll.create_index([("scopes", ASCENDING)])
|
||||
coll.create_index([("studies", ASCENDING)])
|
||||
coll.create_index([("sites", ASCENDING)])
|
||||
coll.create_index([("level", ASCENDING)])
|
||||
runs = db[RUNS_COLL]
|
||||
runs.create_index([("level", ASCENDING), ("study", ASCENDING),
|
||||
("country", ASCENDING), ("exported_at", ASCENDING)])
|
||||
return db, coll, runs
|
||||
|
||||
|
||||
def sync_report_to_mongo(coll, runs, docs, report, report_file):
|
||||
"""Promítne report daného scope do kolekce documents.
|
||||
- nové založí, změny polí promítne (+ history[]),
|
||||
- každému dokumentu přidá scope do scopes[] (a level do levels[]),
|
||||
- dokument, který z TOHOTO scope zmizel, ztratí tento scope;
|
||||
bez jakéhokoli scope -> deleted=True.
|
||||
Scoped mazání = sync jednoho reportu NIKDY neoznačí dokumenty
|
||||
jiného scope (study/country/site) jako smazané. Žádné souborové
|
||||
operace (úložiště je SeaweedFS)."""
|
||||
now = datetime.now()
|
||||
sk = scope_key(report)
|
||||
stats = {"new": 0, "updated": 0, "unchanged": 0,
|
||||
"resurrected": 0, "scope_removed": 0, "marked_deleted": 0}
|
||||
current_keys = set()
|
||||
|
||||
for d in docs:
|
||||
key = doc_key(d["vtmf"], d["version"])
|
||||
current_keys.add(key)
|
||||
existing = coll.find_one({"_id": key})
|
||||
|
||||
if existing is None:
|
||||
coll.insert_one({
|
||||
"_id": key, **d,
|
||||
"levels": [d["level"]], "scopes": [sk],
|
||||
"first_seen": now, "last_seen": now,
|
||||
"deleted": False, "downloaded": False,
|
||||
"seaweed_path": None, "history": [],
|
||||
})
|
||||
stats["new"] += 1
|
||||
continue
|
||||
|
||||
changes = {}
|
||||
for fld in TRACKED_FIELDS:
|
||||
if existing.get(fld) != d.get(fld):
|
||||
changes[fld] = {"old": existing.get(fld), "new": d.get(fld)}
|
||||
|
||||
update = {"$set": {**d, "last_seen": now, "deleted": False},
|
||||
"$addToSet": {"scopes": sk, "levels": d["level"]}}
|
||||
if changes:
|
||||
update["$push"] = {"history": {"ts": now, "changes": changes}}
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["unchanged"] += 1
|
||||
if existing.get("deleted"):
|
||||
stats["resurrected"] += 1
|
||||
coll.update_one({"_id": key}, update)
|
||||
|
||||
# dokumenty dříve v TOMTO scope, které v reportu chybí -> odebrat scope
|
||||
for rec in coll.find({"scopes": sk, "_id": {"$nin": list(current_keys)}}):
|
||||
remaining = [s for s in rec.get("scopes", []) if s != sk]
|
||||
upd = {"scopes": remaining}
|
||||
op = {"$set": upd}
|
||||
stats["scope_removed"] += 1
|
||||
if not remaining: # už nikde -> smazáno
|
||||
upd["deleted"] = True
|
||||
upd["deleted_at"] = now
|
||||
op["$push"] = {"history": {"ts": now,
|
||||
"changes": {"deleted": {"old": False, "new": True}}}}
|
||||
stats["marked_deleted"] += 1
|
||||
coll.update_one({"_id": rec["_id"]}, op)
|
||||
|
||||
runs.insert_one({
|
||||
"level": report["level"], "study": report["study"],
|
||||
"country": report.get("country"), "url": report["url"],
|
||||
"scope": sk, "exported_at": now,
|
||||
"file": str(report_file), "row_count": len(docs),
|
||||
"doc_keys": sorted(current_keys),
|
||||
})
|
||||
|
||||
log(f"[ok] Mongo sync [{sk}]: {stats['new']} nových, {stats['updated']} změněných, "
|
||||
f"{stats['unchanged']} beze změny, {stats['resurrected']} obnovených, "
|
||||
f"{stats['scope_removed']} odebrán scope ({stats['marked_deleted']} úplně smazáno).")
|
||||
return stats
|
||||
|
||||
|
||||
# --- Přihlášení --------------------------------------------------------
|
||||
|
||||
def submit_login_form(page, password_box):
|
||||
"""Odešle login formulář. Zkouší postupně tlačítka Sign On / Login /
|
||||
OK / submit input; když žádné nenajde, stiskne Enter v poli hesla."""
|
||||
candidates = [
|
||||
page.get_by_role("button", name=re.compile("sign\\s*on", re.I)),
|
||||
page.get_by_role("button", name=re.compile("log\\s*in|sign\\s*in", re.I)),
|
||||
page.locator("input[type='submit']"),
|
||||
page.locator("button[type='submit']"),
|
||||
page.get_by_role("button", name=re.compile("^ok$", re.I)),
|
||||
]
|
||||
for loc in candidates:
|
||||
try:
|
||||
if loc.count() and loc.first.is_visible():
|
||||
label = (loc.first.inner_text() or
|
||||
loc.first.get_attribute("value") or "submit").strip()
|
||||
log(f"[i] Odesílám formulář tlačítkem '{label}'...")
|
||||
loc.first.click()
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
log("[i] Tlačítko nenalezeno, odesílám Enterem v poli hesla...")
|
||||
password_box.press("Enter")
|
||||
|
||||
|
||||
def login_if_needed(page):
|
||||
"""Otevře login URL, vyplní jméno+heslo, detekuje 2FA a počká na
|
||||
ruční potvrzení. Pokud perzistentní session žije, login přeskočí."""
|
||||
log(f"[i] Otevírám přihlašovací URL...")
|
||||
page.goto(LOGIN_URL, wait_until="domcontentloaded")
|
||||
|
||||
if "vtmf.veevavault.com/ui" in page.url:
|
||||
log("[i] Už přihlášen (perzistentní session).")
|
||||
return
|
||||
|
||||
user_box = page.locator("input[type='text']").first
|
||||
try:
|
||||
user_box.wait_for(timeout=8000)
|
||||
except PWTimeout:
|
||||
if "vtmf.veevavault.com/ui" in page.url:
|
||||
log("[i] Přihlášen bez formuláře (session redirect).")
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"Nenašel jsem login formulář ani Vault. Aktuální URL: {page.url}")
|
||||
|
||||
username = os.environ["VAULT_USER"]
|
||||
password = os.environ["VAULT_PASS"]
|
||||
|
||||
log("[i] Vyplňuji přihlašovací údaje...")
|
||||
user_box.fill(username)
|
||||
password_box = page.locator("input[type='password']").first
|
||||
password_box.fill(password)
|
||||
submit_login_form(page, password_box)
|
||||
|
||||
log("[i] Odeslán login, čekám na výsledek...")
|
||||
try:
|
||||
page.wait_for_url(VAULT_UI_PATTERN, timeout=15000)
|
||||
log("[ok] Přihlášen rovnou (bez 2FA).")
|
||||
return
|
||||
except PWTimeout:
|
||||
pass # nejsme ve Vaultu -> pravděpodobně 2FA výzva
|
||||
|
||||
err = page.locator("text=/invalid|incorrect|failed/i")
|
||||
try:
|
||||
if err.count() and err.first.is_visible():
|
||||
raise RuntimeError(f"Login selhal: {err.first.inner_text().strip()}")
|
||||
except PWTimeout:
|
||||
pass
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(" VYŽADOVÁNO OVĚŘENÍ NA TELEFONU (2FA).")
|
||||
print(" Potvrď přihlášení v mobilní aplikaci.")
|
||||
print("=" * 60)
|
||||
input(" Až to potvrdíš, stiskni ENTER pro pokračování... ")
|
||||
|
||||
page.wait_for_url(VAULT_UI_PATTERN, timeout=120000)
|
||||
log("[ok] Přihlášení dokončeno.")
|
||||
|
||||
|
||||
def verify_inside(page):
|
||||
"""Ověří, že jsme uvnitř Vaultu (URL na /ui)."""
|
||||
page.wait_for_url(VAULT_UI_PATTERN, timeout=30000)
|
||||
log(f"[ok] Uvnitř Vaultu: {page.url}")
|
||||
|
||||
|
||||
def dialog_visible(page):
|
||||
"""True, pokud je na stránce viditelný jQuery UI dialog."""
|
||||
try:
|
||||
dlg = page.locator(".ui-dialog")
|
||||
return bool(dlg.count() and dlg.first.is_visible())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def save_page_debug(page, tag):
|
||||
"""Uloží diagnostiku stránky: screenshot, HTML všech frames a výpis
|
||||
kandidátů na tlačítka. Vrátí cestu složky."""
|
||||
out = DEBUG_DIR / datetime.now().strftime(f"%Y-%m-%d_%H-%M-%S_{tag}")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
page.screenshot(path=str(out / "screenshot.png"), full_page=False)
|
||||
except Exception as e:
|
||||
(out / "screenshot_error.txt").write_text(str(e), encoding="utf-8")
|
||||
report = []
|
||||
for i, frame in enumerate(page.frames):
|
||||
report.append(f"=== frame[{i}] url={frame.url}")
|
||||
try:
|
||||
(out / f"frame_{i}.html").write_text(frame.content(), encoding="utf-8")
|
||||
for sel in (".ui-dialog", "a.ok.vv_button",
|
||||
".ui-dialog-titlebar-close",
|
||||
"button", "input[type='button']",
|
||||
"[title]", "[aria-label]"):
|
||||
n = frame.locator(sel).count()
|
||||
if n:
|
||||
report.append(f" {sel}: {n}x")
|
||||
for attr in ("title", "aria-label"):
|
||||
vals = frame.locator(f"[{attr}]").evaluate_all(
|
||||
f"els => els.map(e => e.getAttribute('{attr}'))")
|
||||
uniq = sorted({v for v in vals if v})[:80]
|
||||
report.append(f" {attr}: {uniq}")
|
||||
except Exception as e:
|
||||
report.append(f" [chyba čtení framu: {e}]")
|
||||
(out / "frames_report.txt").write_text("\n".join(report), encoding="utf-8")
|
||||
log(f"[!] Diagnostika stránky uložena do: {out}")
|
||||
return out
|
||||
|
||||
|
||||
# Viditelné OK tlačítko dialogu — je to <a>, ne <button>!
|
||||
# Křížek .ui-dialog-titlebar-close je display:none → NEPOUŽÍVAT.
|
||||
DIALOG_OK_SELECTOR = (".ui-dialog a.ok.vv_button, "
|
||||
".vv_login_msg_dialog .vv_button.ok")
|
||||
|
||||
|
||||
def dismiss_maintenance_popup(page, timeout=8000):
|
||||
"""Zavře Veeva login/maintenance dialog kliknutím na viditelné OK
|
||||
(<a class='ok vv_button'>). Dialog se objevuje SE ZPOŽDĚNÍM,
|
||||
proto se na něj krátce čeká. Bezpečné volat vždy."""
|
||||
ok = page.locator(DIALOG_OK_SELECTOR)
|
||||
try:
|
||||
ok.first.wait_for(state="visible", timeout=timeout)
|
||||
except PWTimeout:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
closed = 0
|
||||
for _ in range(5): # dialogy umí být ve frontě
|
||||
try:
|
||||
if ok.count() and ok.first.is_visible():
|
||||
ok.first.click()
|
||||
page.wait_for_timeout(300)
|
||||
closed += 1
|
||||
log("[i] Maintenance/login dialog zavřen (OK).")
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
if not dialog_visible(page):
|
||||
return bool(closed)
|
||||
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(500)
|
||||
log("[i] Zkusil jsem dialog zavřít klávesou Escape.")
|
||||
|
||||
if dialog_visible(page):
|
||||
save_page_debug(page, "dialog")
|
||||
print("\n" + "=" * 60)
|
||||
print(" DIALOG SE NEPODAŘILO ZAVŘÍT AUTOMATICKY.")
|
||||
print(" Zavři ho prosím ručně v prohlížeči.")
|
||||
print("=" * 60)
|
||||
input(" Po ručním zavření stiskni ENTER... ")
|
||||
return bool(closed)
|
||||
|
||||
|
||||
# --- Export reportu ----------------------------------------------------
|
||||
|
||||
def _first_visible(page, builders):
|
||||
"""Vrátí (locator, popis) prvního viditelného kandidáta. Hledá na
|
||||
hlavní stránce i ve všech frames."""
|
||||
for frame in page.frames:
|
||||
for build, desc in builders:
|
||||
try:
|
||||
loc = build(frame)
|
||||
if loc.count() and loc.first.is_visible():
|
||||
return loc.first, desc
|
||||
except Exception:
|
||||
continue
|
||||
return None, None
|
||||
|
||||
|
||||
def download_report(page, report):
|
||||
"""Stáhne daný report (Export to Excel, Data Only) do WhatToDownload/
|
||||
pod timestampovaným názvem. Vrátí cestu k souboru.
|
||||
Při selhání uloží diagnostiku stránky do debug/ a vyhodí výjimku."""
|
||||
log(f"\n[i] === Report {report['level'].upper()} "
|
||||
f"({report.get('country') or report['study']}) ===")
|
||||
log("[i] Otevírám report...")
|
||||
page.goto(report["url"], wait_until="domcontentloaded")
|
||||
dismiss_maintenance_popup(page, timeout=4000)
|
||||
|
||||
try:
|
||||
page.wait_for_selector("text=Returned", timeout=30000)
|
||||
except PWTimeout:
|
||||
try:
|
||||
page.wait_for_selector("text=Document Status:", timeout=30000)
|
||||
except PWTimeout:
|
||||
save_page_debug(page, f"report_load_{report['level']}")
|
||||
raise RuntimeError(
|
||||
"Report se nenačetl (nenašel jsem 'Returned' ani "
|
||||
"'Document Status:'). Diagnostika v debug/.")
|
||||
log("[i] Report načten, otevírám menu akcí (⋯)...")
|
||||
|
||||
actions, desc = _first_visible(page, [
|
||||
(lambda f: f.locator(
|
||||
".actionMenuContainer .dropDown.vv_dropdown_toggle "
|
||||
"button.vv-icon-button"), ".actionMenuContainer button (ověřený)"),
|
||||
(lambda f: f.locator(".actionMenuContainer button"), ".actionMenuContainer button (volnější)"),
|
||||
(lambda f: f.locator("button[title='Actions'], [aria-label='Actions']"), "title/aria-label Actions"),
|
||||
])
|
||||
if actions is None:
|
||||
save_page_debug(page, f"report_menu_{report['level']}")
|
||||
raise RuntimeError("Nenašel jsem menu akcí (⋯) na reportu. Diagnostika v debug/.")
|
||||
log(f"[i] Menu nalezeno přes: {desc}")
|
||||
actions.click()
|
||||
|
||||
item = page.locator("a.ReportAction[data-action-name='ExcelExport']")
|
||||
try:
|
||||
item.first.wait_for(state="visible", timeout=15000)
|
||||
except PWTimeout:
|
||||
item = page.get_by_text("Export to Excel", exact=True)
|
||||
try:
|
||||
item.first.wait_for(state="visible", timeout=5000)
|
||||
except PWTimeout:
|
||||
save_page_debug(page, f"report_export_item_{report['level']}")
|
||||
raise RuntimeError("Menu se otevřelo, ale položku 'Export to "
|
||||
"Excel' jsem nenašel. Diagnostika v debug/.")
|
||||
log("[i] Klikám 'Export to Excel'...")
|
||||
item.first.click()
|
||||
log("[i] Dialog Excel Export Options...")
|
||||
|
||||
radio = page.locator("input[name='requiredRadioField'][value='STANDARD']")
|
||||
try:
|
||||
radio.first.wait_for(state="visible", timeout=10000)
|
||||
if not radio.first.is_checked():
|
||||
radio.first.check()
|
||||
log("[i] Přepnuto na 'Data Only'.")
|
||||
except PWTimeout:
|
||||
log("[!] Radio 'Data Only' nenalezeno — spoléhám na default dialogu.")
|
||||
|
||||
export_btn = page.get_by_role("button", name="Export", exact=True)
|
||||
try:
|
||||
export_btn.first.wait_for(state="visible", timeout=10000)
|
||||
except PWTimeout:
|
||||
save_page_debug(page, f"report_export_btn_{report['level']}")
|
||||
raise RuntimeError("Dialog exportu bez tlačítka Export. Diagnostika v debug/.")
|
||||
export_btn = export_btn.first
|
||||
with page.expect_download(timeout=120000) as dl_info:
|
||||
export_btn.click()
|
||||
download = dl_info.value
|
||||
|
||||
EXCEL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
dest = EXCEL_DIR / f"{ts} {report['level']} {download.suggested_filename}"
|
||||
download.save_as(str(dest))
|
||||
log(f"[ok] Report uložen: {dest}")
|
||||
return dest
|
||||
|
||||
|
||||
def archive_report(path):
|
||||
"""Po úspěšném zpracování přesune report do Zpracovano/."""
|
||||
PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
target = PROCESSED_DIR / path.name
|
||||
path.rename(target)
|
||||
log(f"[i] Report archivován: {target}")
|
||||
|
||||
|
||||
# --- SeaweedFS ---------------------------------------------------------
|
||||
|
||||
def seaweed_path(vtmf, version, ext):
|
||||
"""Cesta podle identity dokumentu: /vtmf-documents/<vtmf>/<verze><ext>."""
|
||||
ver = version or "vunknown"
|
||||
return f"{SEAWEED_PREFIX}/{vtmf}/{ver}{ext}"
|
||||
|
||||
|
||||
def seaweed_store(vtmf, version, ext, data, mime="application/octet-stream"):
|
||||
"""Upload do SeaweedFS Filer pod cestou <vtmf>/<verze><ext>.
|
||||
Vrací (path, url)."""
|
||||
path = seaweed_path(vtmf, version, ext)
|
||||
url = SEAWEED_FILER + path
|
||||
req = urllib.request.Request(url, data=data, method="PUT",
|
||||
headers={"Content-Type": mime})
|
||||
urllib.request.urlopen(req, timeout=120)
|
||||
return path, url
|
||||
|
||||
|
||||
# --- Stažení dokumentů -------------------------------------------------
|
||||
|
||||
def find_source_file_button(page):
|
||||
"""Najde ikonu Source File (list papíru se šipkou dolů, vpravo nahoře)."""
|
||||
for sel in ("[title='Source File']", "[aria-label='Source File']"):
|
||||
loc = page.locator(sel)
|
||||
if loc.count():
|
||||
return loc.first
|
||||
loc = page.get_by_role("button", name=re.compile("Source File", re.I))
|
||||
if loc.count():
|
||||
return loc.first
|
||||
return None
|
||||
|
||||
|
||||
def download_source_bytes(page, doc):
|
||||
"""Otevře dokument, stáhne Source File do dočasného souboru Playwrightu
|
||||
a vrátí (data: bytes, ext: str). Žádné trvalé uložení na disk.
|
||||
PlaceholderDocument když dokument nemá obsah."""
|
||||
vtmf = doc["vtmf"]
|
||||
log(f"[i] Otevírám dokument {vtmf} ({doc.get('version', '')}) ...")
|
||||
page.goto(doc["url"], wait_until="domcontentloaded")
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=30000)
|
||||
except PWTimeout:
|
||||
log("[!] networkidle nenastal do 30 s, zkouším pokračovat...")
|
||||
dismiss_maintenance_popup(page, timeout=2000)
|
||||
|
||||
ph = page.locator("div.vv_placeholder_text")
|
||||
if ph.count() and ph.first.is_visible():
|
||||
log(f"[i] {vtmf}: placeholder bez obsahu — přeskakuji.")
|
||||
raise PlaceholderDocument(vtmf)
|
||||
|
||||
target = find_source_file_button(page)
|
||||
if target is None:
|
||||
raise RuntimeError(
|
||||
f"Nenašel jsem ikonu 'Source File' na stránce dokumentu {vtmf}.")
|
||||
|
||||
log("[i] Klikám na Source File a čekám na download...")
|
||||
with page.expect_download(timeout=60000) as dl_info:
|
||||
target.click()
|
||||
try:
|
||||
item = page.get_by_role("menuitem", name=re.compile("Source File", re.I))
|
||||
if item.count() and item.first.is_visible():
|
||||
log("[i] Otevřel se dropdown, vybírám 'Source File'...")
|
||||
item.first.click()
|
||||
except Exception:
|
||||
pass
|
||||
download = dl_info.value
|
||||
|
||||
ext = Path(download.suggested_filename).suffix
|
||||
tmp = download.path() # dočasný soubor Playwrightu
|
||||
data = Path(tmp).read_bytes()
|
||||
return data, ext
|
||||
|
||||
|
||||
def download_missing(page, coll):
|
||||
"""Stáhne všechny nesmazané dokumenty bez downloaded=True PŘÍMO do
|
||||
SeaweedFS (žádný disk). Výsledek každého se ihned zapíše do Mongo."""
|
||||
todo = list(coll.find({"deleted": False, "downloaded": {"$ne": True}})
|
||||
.sort([("level", ASCENDING), ("vtmf", ASCENDING),
|
||||
("version", ASCENDING)]))
|
||||
if LIMIT:
|
||||
todo = todo[:LIMIT]
|
||||
log(f"\n[i] Ke stažení: {len(todo)} dokumentů"
|
||||
+ (f" (LIMIT={LIMIT})" if LIMIT else ""))
|
||||
|
||||
ok_count = fail_count = placeholder_count = 0
|
||||
for n, doc in enumerate(todo, 1):
|
||||
key = doc["_id"]
|
||||
log(f"\n--- [{n}/{len(todo)}] {key} | {doc.get('level', '?')} | {doc['desc'][:60]}")
|
||||
last_err = None
|
||||
for attempt in range(1, MAX_ATTEMPTS + 1):
|
||||
try:
|
||||
data, ext = download_source_bytes(page, doc)
|
||||
size_kb = len(data) / 1024
|
||||
size_str = f"{size_kb:.0f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB"
|
||||
sha256_hex = hashlib.sha256(data).hexdigest()
|
||||
mime = mimetypes.guess_type("f" + ext)[0] or "application/octet-stream"
|
||||
sw_path, sw_url = seaweed_store(doc["vtmf"], doc["version"], ext, data, mime)
|
||||
log(f"[ok] {size_str} -> SeaweedFS {sw_path}")
|
||||
coll.update_one({"_id": key}, {"$set": {
|
||||
"downloaded": True,
|
||||
"downloaded_at": datetime.now(),
|
||||
"sha256": sha256_hex,
|
||||
"seaweed_path": sw_path, "seaweed_url": sw_url,
|
||||
"seaweed_synced_at": datetime.now(),
|
||||
"last_error": None}})
|
||||
ok_count += 1
|
||||
last_err = None
|
||||
break
|
||||
except PlaceholderDocument:
|
||||
coll.update_one({"_id": key}, {"$set": {
|
||||
"downloaded": True, "placeholder": True,
|
||||
"downloaded_at": datetime.now(), "last_error": None}})
|
||||
placeholder_count += 1
|
||||
last_err = None
|
||||
break
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
log(f"[!] Pokus {attempt}/{MAX_ATTEMPTS} selhal: {e}")
|
||||
if attempt < MAX_ATTEMPTS:
|
||||
page.wait_for_timeout(RETRY_PAUSE_MS)
|
||||
if last_err is not None:
|
||||
coll.update_one({"_id": key}, {"$set": {
|
||||
"last_error": str(last_err), "error_at": datetime.now()}})
|
||||
fail_count += 1
|
||||
page.wait_for_timeout(BETWEEN_DOCS_MS)
|
||||
return ok_count, fail_count, placeholder_count
|
||||
|
||||
|
||||
# --- Main --------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ensure_credentials()
|
||||
db, coll, runs = get_db()
|
||||
log(f"[ok] Mongo připojeno: {MONGO_URI} / {MONGO_DB}.{MONGO_COLL}")
|
||||
|
||||
with sync_playwright() as p:
|
||||
ctx = p.chromium.launch_persistent_context(
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
headless=False,
|
||||
accept_downloads=True,
|
||||
no_viewport=True,
|
||||
args=["--start-maximized"],
|
||||
)
|
||||
page = ctx.pages[0] if ctx.pages else ctx.new_page()
|
||||
ok_count = fail_count = placeholder_count = 0
|
||||
pipeline_error = None
|
||||
try:
|
||||
# 1) login
|
||||
login_if_needed(page)
|
||||
verify_inside(page)
|
||||
dismiss_maintenance_popup(page)
|
||||
|
||||
# 2+3) pro každý ZAPNUTÝ report: export -> parse -> scoped sync
|
||||
log("\n[i] Plán reportů:")
|
||||
for r in REPORTS:
|
||||
flag = "ZAP" if r.get("enabled", True) else "VYP"
|
||||
log(f" [{flag}] {r.get('name', r['level'])}")
|
||||
for report in REPORTS:
|
||||
if not report.get("enabled", True):
|
||||
log(f"\n[i] Přeskakuji (enabled=False): {report.get('name', report['level'])}")
|
||||
continue
|
||||
report_path = download_report(page, report)
|
||||
docs = read_documents_from_excel(report_path, report["level"])
|
||||
before = len(docs)
|
||||
docs = [d for d in docs if report["study"] in d["studies"]]
|
||||
if before != len(docs):
|
||||
log(f"[i] Ořez na {report['study']}: {len(docs)}/{before} řádků.")
|
||||
if not docs:
|
||||
log(f"[!] Report {report['level']} prázdný (po ořezu) — "
|
||||
f"sync přeskočen, nic se nemaže.")
|
||||
archive_report(report_path)
|
||||
continue
|
||||
sync_report_to_mongo(coll, runs, docs, report, report_path)
|
||||
archive_report(report_path)
|
||||
|
||||
# 4) jeden průchod stažení všeho nestaženého do SeaweedFS
|
||||
ok_count, fail_count, placeholder_count = download_missing(page, coll)
|
||||
except KeyboardInterrupt:
|
||||
log("\n[!] Přerušeno uživatelem — stav je v Mongo, příští běh naváže.")
|
||||
except Exception as e:
|
||||
pipeline_error = e
|
||||
print("\n" + "=" * 60)
|
||||
print(" PIPELINE SELHALA!")
|
||||
print(f" {type(e).__name__}: {e}")
|
||||
print("=" * 60)
|
||||
finally:
|
||||
total = coll.count_documents({})
|
||||
active = coll.count_documents({"deleted": False})
|
||||
have = coll.count_documents({"deleted": False, "downloaded": True})
|
||||
log(f"\n[i] Výsledek běhu: {ok_count} staženo, "
|
||||
f"{placeholder_count} placeholderů, {fail_count} chyb"
|
||||
+ (f", PIPELINE SELHALA ({pipeline_error})" if pipeline_error else "."))
|
||||
log(f"[i] Mongo: {total} záznamů celkem, {active} aktivních, "
|
||||
f"z toho v SeaweedFS {have} ({active - have} zbývá).")
|
||||
log("[i] Zavírám prohlížeč.")
|
||||
ctx.close()
|
||||
sys.exit(2 if pipeline_error else (1 if fail_count else 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user