git

90 Import all torrents from all pages.py

@@ -47,11 +47,12 @@ cursor = db.cursor()
 # 2) Selenium setup
 # ============================================================
 COOKIE_FILE = Path("sktorrent_cookies.json")
-# Updated to standard torrents.php as requested
-BASE_URL = (
-    "https://sktorrent.eu/torrent/torrents.php"
-    "?active=0&category=24&order=data&by=DESC&zaner=&jazyk="
-)
+
+CATEGORIES = {
+    24: "Knihy a časopisy",
+    32: "Mluvené slovo"
+}
+MAX_PAGES = 10
 
 chrome_options = Options()
 chrome_options.add_argument("--start-maximized")
@@ -85,7 +86,15 @@ print("🔗 Requests session initialized.")
 
 
 # ============================================================
-# 4) Popup handler
+# 4) Build URL for category and page
+# ============================================================
+def get_torrent_url(category, page):
+    base = "https://sktorrent.eu/torrent/torrents.php"
+    return f"{base}?active=0&category={category}&order=data&by=DESC&zaner=&jazyk=&page={page}"
+
+
+# ============================================================
+# 5) Popup handler
 # ============================================================
 def close_popup_if_any():
     try:
@@ -96,7 +105,7 @@ def close_popup_if_any():
 
 
 # ============================================================
-# 5) Parse one torrent row (MODIFIED)
+# 6) Parse one torrent row (MODIFIED)
 # ============================================================
 
 def parse_row(cells):
@@ -204,7 +213,7 @@ def parse_row(cells):
         print(f"⚠️ parse_row logic failed: {e}")
         return None
 # ============================================================
-# 6) INSERT SQL (MODIFIED)
+# 7) INSERT SQL (MODIFIED)
 # ============================================================
 insert_sql = """
 INSERT INTO torrents (
@@ -228,13 +237,15 @@ ON DUPLICATE KEY UPDATE
 # keeps the old value if the new one is NULL,
 # but updates it if the old one was NULL and the new one is binary.
 # ============================================================
-# 7) PROCESS ALL PAGES
+# 8) PROCESS ALL PAGES AND CATEGORIES
 # ============================================================
-TOTAL_PAGES = 226
 
-for page_num in range(0, TOTAL_PAGES):
-    current_url = f"{BASE_URL}&page={page_num}"
-    print(f"\n🌐 Loading Page Index {page_num} (Page {page_num + 1}/{TOTAL_PAGES})")
+for category_id, category_name in CATEGORIES.items():
+    print(f"\n📚 Starting category: {category_name} (ID: {category_id})")
+
+    for page_num in range(0, MAX_PAGES):
+        current_url = get_torrent_url(category_id, page_num)
+        print(f"\n🌐 Loading page {page_num}/{MAX_PAGES - 1} for {category_name}")
 
         driver.get(current_url)
         time.sleep(2)
@@ -252,17 +263,14 @@ for page_num in range(0, TOTAL_PAGES):
                 real_rows.append(cells)
 
         if not real_rows:
-        print("⚠️ No data rows found on this page. Ending loop.")
+            print(f"⚠️ No data rows found. Stopping pagination for {category_name}")
             break
 
-    # === INSERT THIS LINE HERE ===
         page_new_items = 0
-    # =============================
 
         for cells in real_rows:
             try:
                 data = parse_row(cells)
-            # ... rest of your logic ...
             except Exception as e:
                 print(f"⚠️ parse_row failed: {e}")
                 continue
@@ -281,15 +289,10 @@ for page_num in range(0, TOTAL_PAGES):
 
             cursor.execute(insert_sql, data)
 
-    # # If an entire page is old news, we can stop the deep crawl
-    # if page_new_items == 0 and page_num > 0:
-    #     print("🛑 Page contained only known items. Sync complete.")
-    #     break
-
         time.sleep(1)
 
 # ============================================================
-# 8) SEND EMAIL REPORT
+# 9) SEND EMAIL REPORT
 # ============================================================
 RUN_END = datetime.datetime.now()
 subject = f"SKTorrent run – {RUN_START:%Y-%m-%d %H:%M}"

Seedbox/95 IncrementalImport.py

@@ -30,10 +30,11 @@ HOW_MANY_TO_CHECK = 0
 
 COOKIE_FILE = Path("sktorrent_cookies.json")
 
-BASE_URL = (
-    "https://sktorrent.eu/torrent/torrents.php"
-    "?active=0&category=23&order=data&by=DESC"
-)
+CATEGORIES = {
+    24: "Knihy a časopisy",
+    32: "Mluvené slovo"
+}
+MAX_PAGES = 10
 
 SLEEP_BETWEEN_PAGES   = 2.0   # pauza mezi stránkami
 SLEEP_BEFORE_DOWNLOAD = 1.5   # pauza před stažením každého .torrent souboru
@@ -221,6 +222,8 @@ def main():
     else:
         mode_desc = f"kontrola posledních {HOW_MANY_TO_CHECK} torrentů"
     print(f"Pořadí: nejnovější → nejstarší  |  {mode_desc}")
+    print(f"Kategorie: {', '.join(CATEGORIES.values())}")
+    print(f"Stránek na kategorii: {MAX_PAGES}")
     print("=" * 60)
 
     session = build_session()
@@ -230,12 +233,20 @@ def main():
     new_count      = 0
     checked_count  = 0
     skipped_count  = 0
-    page           = 0
     stop           = False
 
-    while not stop:
+    for category_id, category_name in CATEGORIES.items():
+        if stop:
+            break
 
-        url = f"{BASE_URL}&page={page}"
+        print(f"\n📚 Kategorie: {category_name} (ID: {category_id})")
+
+        for page in range(MAX_PAGES):
+
+            url = (
+                "https://sktorrent.eu/torrent/torrents.php"
+                f"?active=0&category={category_id}&order=data&by=DESC&page={page}"
+            )
             r = None
             for attempt in range(1, 6):
                 try:
@@ -301,8 +312,8 @@ def main():
                 cursor.execute(INSERT_SQL, item)
                 new_count += 1
 
-        if not stop:
-            page += 1
+            if stop:
+                break
             time.sleep(SLEEP_BETWEEN_PAGES)
 
     # ============================================================
@@ -314,7 +325,6 @@ def main():
     print(f"Nových torrentů uloženo : {new_count}")
     print(f"Zkontrolováno celkem   : {checked_count}")
     print(f"Přeskočeno (v DB)      : {skipped_count}")
-    print(f"Stránek prošlo          : {page}")
     print("=" * 60)
 
     db.close()