Compare commits
7 Commits
5f01729bce
...
b4e5c2f46e
| Author | SHA1 | Date | |
|---|---|---|---|
| b4e5c2f46e | |||
| a113f68e97 | |||
| bf75bffb02 | |||
| 0769bd2670 | |||
| 6e8395890d | |||
| bc35cbdfac | |||
| af295ff63c |
10
.idea/Torrents.iml
generated
Normal file
10
.idea/Torrents.iml
generated
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module type="PYTHON_MODULE" version="4">
|
||||||
|
<component name="NewModuleRootManager">
|
||||||
|
<content url="file://$MODULE_DIR$">
|
||||||
|
<excludeFolder url="file://$MODULE_DIR$/.venv" />
|
||||||
|
</content>
|
||||||
|
<orderEntry type="jdk" jdkName="Python 3.12 (torrents)" jdkType="Python SDK" />
|
||||||
|
<orderEntry type="sourceFolder" forTests="false" />
|
||||||
|
</component>
|
||||||
|
</module>
|
||||||
7
.idea/misc.xml
generated
Normal file
7
.idea/misc.xml
generated
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="Black">
|
||||||
|
<option name="sdkName" value="Python 3.12 (torrents)" />
|
||||||
|
</component>
|
||||||
|
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (torrents)" project-jdk-type="Python SDK" />
|
||||||
|
</project>
|
||||||
143
70 MD5.py
Normal file
143
70 MD5.py
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
"""
|
||||||
|
FAST MD5 indexer with in-memory cache
|
||||||
|
- prints every processed file
|
||||||
|
- skips unchanged files instantly
|
||||||
|
- restart-safe (no reprocessing same files)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import hashlib
|
||||||
|
from datetime import datetime
|
||||||
|
import pymysql
|
||||||
|
|
||||||
|
# ==============================
|
||||||
|
# CONFIG
|
||||||
|
# ==============================
|
||||||
|
|
||||||
|
ROOT_DIR = r"\\tower1\#ColdData\porno"
|
||||||
|
|
||||||
|
DB_CONFIG = {
|
||||||
|
"host": "192.168.1.76",
|
||||||
|
"port": 3307,
|
||||||
|
"user": "root",
|
||||||
|
"password": "Vlado9674+",
|
||||||
|
"database": "torrents",
|
||||||
|
"charset": "utf8mb4",
|
||||||
|
"autocommit": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
CHUNK_SIZE = 1024 * 1024 # 1 MB
|
||||||
|
PRINT_SKIPPED = True # set False if too noisy
|
||||||
|
|
||||||
|
# ==============================
|
||||||
|
# HELPERS
|
||||||
|
# ==============================
|
||||||
|
|
||||||
|
def compute_md5(path: str) -> str:
|
||||||
|
h = hashlib.md5()
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
for chunk in iter(lambda: f.read(CHUNK_SIZE), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
def format_size(size):
|
||||||
|
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
||||||
|
if size < 1024:
|
||||||
|
return f"{size:.1f} {unit}"
|
||||||
|
size /= 1024
|
||||||
|
return f"{size:.1f} PB"
|
||||||
|
|
||||||
|
# ==============================
|
||||||
|
# MAIN
|
||||||
|
# ==============================
|
||||||
|
|
||||||
|
def main():
|
||||||
|
db = pymysql.connect(**DB_CONFIG)
|
||||||
|
cur = db.cursor()
|
||||||
|
|
||||||
|
print("📥 Loading already indexed files into memory...")
|
||||||
|
|
||||||
|
cur.execute("""
|
||||||
|
SELECT full_path, file_size, UNIX_TIMESTAMP(mtime)
|
||||||
|
FROM file_md5_index
|
||||||
|
""")
|
||||||
|
|
||||||
|
indexed = {
|
||||||
|
(row[0], row[1], row[2])
|
||||||
|
for row in cur.fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"✅ Loaded {len(indexed):,} indexed entries")
|
||||||
|
print("======================================")
|
||||||
|
|
||||||
|
new_files = 0
|
||||||
|
skipped = 0
|
||||||
|
|
||||||
|
for root, _, files in os.walk(ROOT_DIR):
|
||||||
|
for fname in files:
|
||||||
|
full_path = os.path.join(root, fname)
|
||||||
|
|
||||||
|
try:
|
||||||
|
stat = os.stat(full_path)
|
||||||
|
except (OSError, FileNotFoundError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
mtime = int(stat.st_mtime)
|
||||||
|
size = stat.st_size
|
||||||
|
|
||||||
|
key = (full_path, size, mtime)
|
||||||
|
|
||||||
|
# FAST PATH
|
||||||
|
if key in indexed:
|
||||||
|
skipped += 1
|
||||||
|
if PRINT_SKIPPED:
|
||||||
|
print("⏭ SKIP")
|
||||||
|
print(f" File: {full_path}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
print("➕ NEW / UPDATED")
|
||||||
|
print(f" Size: {format_size(size)}")
|
||||||
|
print(f" File: {full_path}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
md5 = compute_md5(full_path)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ MD5 failed: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
cur.execute("""
|
||||||
|
INSERT INTO file_md5_index
|
||||||
|
(full_path, file_name, directory, file_size, mtime, md5)
|
||||||
|
VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s), %s)
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
file_size=VALUES(file_size),
|
||||||
|
mtime=VALUES(mtime),
|
||||||
|
md5=VALUES(md5),
|
||||||
|
updated_at=CURRENT_TIMESTAMP
|
||||||
|
""", (
|
||||||
|
full_path,
|
||||||
|
fname,
|
||||||
|
root,
|
||||||
|
size,
|
||||||
|
mtime,
|
||||||
|
md5,
|
||||||
|
))
|
||||||
|
|
||||||
|
new_files += 1
|
||||||
|
print(f" MD5 : {md5}")
|
||||||
|
print("--------------------------------------")
|
||||||
|
|
||||||
|
print("======================================")
|
||||||
|
print(f"✅ New / updated : {new_files}")
|
||||||
|
print(f"⏭ Skipped : {skipped}")
|
||||||
|
print("======================================")
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
"value": "646071",
|
"value": "646071",
|
||||||
"domain": "sktorrent.eu",
|
"domain": "sktorrent.eu",
|
||||||
"path": "/",
|
"path": "/",
|
||||||
"expires": 1799666388.762402,
|
"expires": 1798003565.462807,
|
||||||
"httpOnly": false,
|
"httpOnly": false,
|
||||||
"secure": false,
|
"secure": false,
|
||||||
"sameSite": "Lax"
|
"sameSite": "Lax"
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
"value": "91df6b497860582e09a7b333569d0187",
|
"value": "91df6b497860582e09a7b333569d0187",
|
||||||
"domain": "sktorrent.eu",
|
"domain": "sktorrent.eu",
|
||||||
"path": "/",
|
"path": "/",
|
||||||
"expires": 1799666388.762402,
|
"expires": 1798003565.463191,
|
||||||
"httpOnly": false,
|
"httpOnly": false,
|
||||||
"secure": false,
|
"secure": false,
|
||||||
"sameSite": "Lax"
|
"sameSite": "Lax"
|
||||||
|
|||||||
Reference in New Issue
Block a user