Merge remote-tracking branch 'origin/master'
This commit is contained in:
+52
-14
@@ -149,6 +149,50 @@ def filer_url(filer, seaweed_path):
|
|||||||
return filer.rstrip("/") + "/" + enc
|
return filer.rstrip("/") + "/" + enc
|
||||||
|
|
||||||
|
|
||||||
|
SLICE = 16 << 20 # 16 MB na Range usek (plny GET fileru je patologicky pomaly)
|
||||||
|
|
||||||
|
|
||||||
|
def download_resumable(url, dst, size, retries=8):
|
||||||
|
"""Stahuje po Range usecich do .part (filer servíruje plny GET ~50x pomaleji).
|
||||||
|
Pri vypadku navaze tam, kde skoncil. Vraci (ok, posledni_chyba)."""
|
||||||
|
tmp = dst.with_suffix(".part")
|
||||||
|
have = tmp.stat().st_size if tmp.exists() else 0
|
||||||
|
if have > size: # poskozeny zbytek -> od zacatku
|
||||||
|
tmp.unlink()
|
||||||
|
have = 0
|
||||||
|
last = None
|
||||||
|
fails = 0
|
||||||
|
while have < size:
|
||||||
|
end = min(have + SLICE, size) - 1
|
||||||
|
try:
|
||||||
|
with requests.get(url, headers={"Range": f"bytes={have}-{end}"},
|
||||||
|
stream=True, timeout=(15, 90)) as r:
|
||||||
|
if r.status_code not in (206, 200):
|
||||||
|
r.raise_for_status()
|
||||||
|
if r.status_code == 200: # filer ignoroval Range -> cely soubor
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
with open(tmp, "wb") as f:
|
||||||
|
for chunk in r.iter_content(1 << 20):
|
||||||
|
f.write(chunk)
|
||||||
|
break
|
||||||
|
with open(tmp, "ab") as f:
|
||||||
|
for chunk in r.iter_content(1 << 20):
|
||||||
|
f.write(chunk)
|
||||||
|
have = tmp.stat().st_size
|
||||||
|
fails = 0 # usek prosel -> reset retry
|
||||||
|
except Exception as e:
|
||||||
|
last = e
|
||||||
|
fails += 1
|
||||||
|
if fails > retries:
|
||||||
|
break
|
||||||
|
time.sleep(min(2 ** fails, 20))
|
||||||
|
have = tmp.stat().st_size if tmp.exists() else 0
|
||||||
|
ok = tmp.exists() and tmp.stat().st_size == size
|
||||||
|
if ok:
|
||||||
|
tmp.replace(dst)
|
||||||
|
return ok, last
|
||||||
|
|
||||||
|
|
||||||
def build_plan(db):
|
def build_plan(db):
|
||||||
kurzy = {k["_id"]: k for k in db.kurzy.find({})}
|
kurzy = {k["_id"]: k for k in db.kurzy.find({})}
|
||||||
vids = list(db.materialy.find(
|
vids = list(db.materialy.find(
|
||||||
@@ -231,24 +275,18 @@ def main():
|
|||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
print(f"[{n}/{len(plan)}] ↓ {size/1024**2:.1f}MB {fn}", flush=True)
|
print(f"[{n}/{len(plan)}] ↓ {size/1024**2:.1f}MB {fn}", flush=True)
|
||||||
try:
|
url = filer_url(args.filer, path)
|
||||||
url = filer_url(args.filer, path)
|
ts = time.time()
|
||||||
ts = time.time()
|
ok, err = download_resumable(url, dst, size)
|
||||||
# timeout=(connect, read) -> zaseknute spojeni spadne rychle
|
if ok:
|
||||||
with requests.get(url, stream=True, timeout=(15, 90)) as r:
|
|
||||||
r.raise_for_status()
|
|
||||||
tmp = dst.with_suffix(".part")
|
|
||||||
with open(tmp, "wb") as f:
|
|
||||||
for chunk in r.iter_content(1 << 20):
|
|
||||||
f.write(chunk)
|
|
||||||
tmp.replace(dst)
|
|
||||||
done += 1
|
done += 1
|
||||||
dl_bytes += size
|
dl_bytes += size
|
||||||
sp = size / 1024**2 / max(time.time() - ts, 0.1)
|
sp = size / 1024**2 / max(time.time() - ts, 0.1)
|
||||||
msg = f"[{n}/{len(plan)}] OK {size/1024**2:.1f}MB ({sp:.1f} MB/s) {fn}"
|
msg = (f"[{n}/{len(plan)}] OK {size/1024**2:.1f}MB "
|
||||||
except Exception as e:
|
f"({sp:.1f} MB/s) {fn}")
|
||||||
|
else:
|
||||||
failed += 1
|
failed += 1
|
||||||
msg = f"[{n}/{len(plan)}] FAIL {fn} :: {e}"
|
msg = f"[{n}/{len(plan)}] FAIL {fn} :: {err}"
|
||||||
print(msg, flush=True)
|
print(msg, flush=True)
|
||||||
log.write(msg + "\n")
|
log.write(msg + "\n")
|
||||||
log.flush()
|
log.flush()
|
||||||
|
|||||||
@@ -20,4 +20,5 @@ SeaweedFS „na hraní" na Tower1 Unraid (192.168.1.50, root, heslo Vlado7309208
|
|||||||
- Bloby: share `SeaweedFS` → `/mnt/user/SeaweedFS/data`, array disk9–11 pod paritou (useCache=no). Filer leveldb + master raft: `/mnt/cache2tb0225dec/appdata/seaweedfs/` (SSD, bez parity).
|
- Bloby: share `SeaweedFS` → `/mnt/user/SeaweedFS/data`, array disk9–11 pod paritou (useCache=no). Filer leveldb + master raft: `/mnt/cache2tb0225dec/appdata/seaweedfs/` (SSD, bez parity).
|
||||||
- Noční backup filer metadat: cron 03:30 na Tower1 (`/boot/config/plugins/dynamix/seaweedfs-backup.cron`) spouští `/boot/config/scripts/seaweedfs_meta_backup.sh` — `fs.meta.save` přes weed shell → scp na tower 192.168.1.76 do `/mnt/user/Backup/Critical/SeaweedFS/daily/` (drží 7), neděle navíc `weekly/` (drží 4). Log `/var/log/seaweedfs_backup.log`. Restore: `fs.meta.load`.
|
- Noční backup filer metadat: cron 03:30 na Tower1 (`/boot/config/plugins/dynamix/seaweedfs-backup.cron`) spouští `/boot/config/scripts/seaweedfs_meta_backup.sh` — `fs.meta.save` přes weed shell → scp na tower 192.168.1.76 do `/mnt/user/Backup/Critical/SeaweedFS/daily/` (drží 7), neděle navíc `weekly/` (drží 4). Log `/var/log/seaweedfs_backup.log`. Restore: `fs.meta.load`.
|
||||||
- Tower (.76, root heslo 7309208104) a Tower1 mají vyměněné SSH klíče — z Tower1 na tower jde psát přímo.
|
- Tower (.76, root heslo 7309208104) a Tower1 mají vyměněné SSH klíče — z Tower1 na tower jde psát přímo.
|
||||||
|
- **HW závada (2026-06, přetrvává od 26.5.):** SATA řadič/port `ata1` na Tower1 „lost interrupt (Status 0x50)" → kernel `Disabling IRQ #19`, ovladač spadne do pollingu, link degraduje na 1.5 Gbps. Na `ata1` visí **disk10 (sdj) a disk11 (sdk)** — obě TOSHIBA MG08ACA16TE (zbytek pole Seagate). Tyto dva disky pak čtou ~20–40 KB/s (iostat: 100% util, r_await 200–400 ms), `mdunraidd11` v D-state, iowait ~97 %. SMART obou PASSED (problém je řadič/kabel, ne ploténky). **Důsledek:** SeaweedFS bloby na disk9–11 se z disk10/11 čtou na ~0,2 MB/s → velké soubory (EUNI videa) se prakticky nestáhnou. Fix: reboot (povolí IRQ) jako dočasný, trvale přepojit obě Toshiby kabelem na jiný port/HBA než `ata1`.
|
||||||
- Pozor: Unraid flash `/boot` je VFAT (case-insensitive) — share cfg `seaweedfs.cfg` vs `SeaweedFS.cfg` je tentýž soubor. Share nastavení za běhu mění `emcmd "query&string"` (jeden argument, nutné `shareNameOrig` + `cmdEditShare=Apply`), live stav v `/var/local/emhttp/shares.ini`.
|
- Pozor: Unraid flash `/boot` je VFAT (case-insensitive) — share cfg `seaweedfs.cfg` vs `SeaweedFS.cfg` je tentýž soubor. Share nastavení za běhu mění `emcmd "query&string"` (jeden argument, nutné `shareNameOrig` + `cmdEditShare=Apply`), live stav v `/var/local/emhttp/shares.ini`.
|
||||||
|
|||||||
Reference in New Issue
Block a user