600 lines
19 KiB
Python
600 lines
19 KiB
Python
"""ETERNAL SourceForge: project discovery + file tree walk + text/archive scan.
|
||
|
||
Discovery: allura_sitemap (13 шардов × ~10k /p/-слагов, lastmod ежедневно, без
|
||
Cloudflare). /directory/?q= за Turnstile-challenge — мёртв для unattended loop.
|
||
Курсор шарда: data/sourceforge_sitemap_cursor.txt; пул: data/sourceforge_projects.txt.
|
||
Liveness: GET https://sourceforge.net/rest/p/<short>/ (trailing slash обязателен;
|
||
status "moved"/404 → skip). Мертвы: /api/*, /rest/search/, /rest/p/ (listing).
|
||
Files: https://sourceforge.net/projects/<p>/files/ (HTML parse of files_list table)
|
||
https://sourceforge.net/projects/<p>/files/<path>/download (mirror redirect)
|
||
Auth: none
|
||
"""
|
||
import os as _os
|
||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||
|
||
|
||
import hashlib
|
||
import io
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
import tarfile
|
||
import threading
|
||
import time
|
||
import zipfile
|
||
|
||
import requests
|
||
|
||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||
LOG = BASE + r"/data/sourceforge_eternal.log"
|
||
CURSOR_FILE = BASE + r"/data/sourceforge_cursor.txt"
|
||
SEEN_FILE = BASE + r"/data/sourceforge_seen.txt"
|
||
PROJECTS_FILE = BASE + r"/data/sourceforge_projects.txt"
|
||
|
||
N_THREADS = 4
|
||
PROJECTS_PER_CYCLE = 5 # 5 projects per cycle → ~80 projects in 16 cycles (~24h)
|
||
MAX_DEPTH = 3 # max subfolder depth to recurse
|
||
MAX_FOLDERS_PER_DEPTH = 10 # max folder listings per depth level (rate-limit guard)
|
||
MAX_ITEMS = 200 # per cycle cap; --once uses min(MAX_ITEMS, 5)
|
||
FILE_CAP = 500_000 # max single text file
|
||
ARCHIVE_CAP = 5 * 1024 * 1024 # max source archive to unpack
|
||
SLEEP = 90
|
||
|
||
from key_patterns import scan_text
|
||
|
||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||
|
||
TEXT_EXT = (".txt", ".md", ".rst", ".cfg", ".ini", ".conf", ".env",
|
||
".xml", ".json", ".yaml", ".yml", ".properties",
|
||
".py", ".sh", ".bat", ".ps1", ".php", ".pl", ".rb",
|
||
".js", ".ts", ".java", ".c", ".h", ".cpp", ".hpp",
|
||
".sql", ".html", ".css", ".dockerfile", ".toml")
|
||
|
||
UA = {"User-Agent": "Wget/1.21.3"}
|
||
|
||
# Curated seed — popular open-source projects likely to bundle text/config files
|
||
SEED_PROJECTS = [
|
||
# IDEs / Editors
|
||
"notepadplusplus", "codeblocks", "geany", "dev-cpp",
|
||
# Web / CMS / Database
|
||
"xampp", "wampserver", "wordpress", "joomla", "drupal",
|
||
"moodle", "phpmyadmin", "adminer", "mysqlcc",
|
||
"postgresql", "sqlitebrowser", "heidisql",
|
||
# Networking / Security
|
||
"wireshark", "nmap", "putty", "openssh", "openvpn",
|
||
"torproject", "truecrypt", "veracrypt", "gpg4win",
|
||
"keepass", "password-safe",
|
||
# File transfer / FTP
|
||
"filezilla", "winscp", "smartftp", "cyberduck",
|
||
# Multimedia
|
||
"audacity", "vlc", "ffmpeg", "handbrake", "shotcut",
|
||
"obsproject", "blender", "gimp", "inkscape",
|
||
# Development tools
|
||
"gitforwindows", "tortoisegit", "tortoisesvn", "mercurial",
|
||
"cmake", "mingw", "cygwin", "nsis", "innosetup",
|
||
"python", "rubyinstaller", "perl",
|
||
# Utilities
|
||
"sevenzip", "peazip", "ventoy", "rufus",
|
||
"crystaldiskinfo", "hwmonitor", "speedfan",
|
||
"sysinternals", "processhacker",
|
||
# System
|
||
"clamwin", "spybot", "ccleaner", "defraggler",
|
||
"wiztree", "everything",
|
||
# Emulators / Games
|
||
"dosbox", "pcsx2", "dolphin-emu", "mame",
|
||
"desmume", "visualboyadvance",
|
||
# Science / Math
|
||
"octave", "scilab", "geogebra",
|
||
# Office
|
||
"libreoffice", "openoffice", "abiword",
|
||
"sumatrapdf", "foxitreader",
|
||
# Chat / Comms
|
||
"pidgin", "irssi", "hexchat", "miranda-ng",
|
||
"thunderbird",
|
||
# Other
|
||
"scrcpy.mirror", "portableapps", "chocolate-doom",
|
||
"golang", "rustdesk",
|
||
]
|
||
|
||
# Filter seed to unique non-empty strings
|
||
SEED_PROJECTS = sorted(set(s for s in SEED_PROJECTS if s and s.strip()))
|
||
|
||
# Discovery: allura_sitemap — полный живой список проектов SF (без Cloudflare)
|
||
SITEMAP_INDEX = "https://sourceforge.net/allura_sitemap/sitemap.xml"
|
||
_SHARD_RE = re.compile(
|
||
r'<loc>(https://sourceforge\.net/allura_sitemap/sitemap-\d+\.xml)</loc>')
|
||
_P_SLUG_RE = re.compile(r'sourceforge\.net/p/([a-z0-9][a-z0-9._-]{0,48})/')
|
||
_PROJ_SLUG_RE = re.compile(
|
||
r'sourceforge\.net/projects/([a-z0-9][a-z0-9._-]{0,48})/')
|
||
SITEMAP_CURSOR_FILE = BASE + r"/data/sourceforge_sitemap_cursor.txt"
|
||
|
||
# Session with trust_env=False to bypass local dashscope proxy
|
||
S = requests.Session()
|
||
S.trust_env = False
|
||
|
||
|
||
def log(msg):
|
||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||
with open(LOG, "a", encoding="utf-8") as f:
|
||
f.write(line)
|
||
|
||
|
||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||
isolation_level=None)
|
||
lock = threading.Lock()
|
||
total = 0
|
||
|
||
|
||
def db_add(key, prov, repo):
|
||
global total
|
||
with lock:
|
||
conn.execute(
|
||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||
"VALUES (?,?,?,?,?,?) "
|
||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||
"found=excluded.found, repo=excluded.repo "
|
||
"WHERE keys.status IN ('DEAD','ERR')",
|
||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||
key, prov, "NEW", "sourceforge", repo))
|
||
total += 1
|
||
|
||
|
||
def load_cursor():
|
||
try:
|
||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||
return int(f.read().strip() or 0)
|
||
except (FileNotFoundError, ValueError):
|
||
return 0
|
||
|
||
|
||
def save_cursor(c):
|
||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||
f.write(str(c))
|
||
|
||
|
||
def load_seen():
|
||
s = set()
|
||
try:
|
||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if line:
|
||
s.add(line)
|
||
except FileNotFoundError:
|
||
pass
|
||
return s
|
||
|
||
|
||
seen = load_seen()
|
||
|
||
|
||
def mark_seen(p):
|
||
seen.add(p)
|
||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||
f.write(p + "\n")
|
||
|
||
|
||
def is_junk(v):
|
||
if any(g in v for g in GARBAGE):
|
||
return True
|
||
if len(set(v)) < 6:
|
||
return True
|
||
return False
|
||
|
||
|
||
def sf_get(url, timeout=20):
|
||
"""Thin wrapper with retry/backoff and 403 handling.
|
||
|
||
Uses module session S (trust_env=False) to bypass local proxy.
|
||
"""
|
||
for attempt in (1, 2, 3):
|
||
try:
|
||
r = S.get(url, headers=UA, timeout=timeout)
|
||
if r.status_code in (429, 503):
|
||
ra = int(r.headers.get("Retry-After", 10 * attempt))
|
||
time.sleep(ra)
|
||
continue
|
||
if r.status_code == 403:
|
||
# Rate-limited; brief wait then retry
|
||
time.sleep(5 * attempt)
|
||
continue
|
||
if r.status_code == 404:
|
||
return None
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"HTTP {r.status_code}")
|
||
return r
|
||
except (requests.ConnectionError, requests.Timeout, RuntimeError):
|
||
if attempt == 3:
|
||
raise
|
||
time.sleep(5 * attempt)
|
||
return None
|
||
|
||
|
||
def list_dir(project, folder=""):
|
||
"""Parse files_list table from project's files page.
|
||
|
||
Yields (is_folder, name, display_name).
|
||
is_folder=True: name is relative URL folder path
|
||
is_folder=False: name is full relative file path (for /download)
|
||
"""
|
||
if folder:
|
||
url = f"https://sourceforge.net/projects/{project}/files/{folder}/"
|
||
else:
|
||
url = f"https://sourceforge.net/projects/{project}/files/"
|
||
try:
|
||
r = sf_get(url, timeout=20)
|
||
except Exception:
|
||
return
|
||
if r is None:
|
||
return
|
||
html = r.text
|
||
# Find tbody
|
||
idx = html.find('<table id="files_list"')
|
||
if idx == -1:
|
||
idx = html.find("<table id='files_list'")
|
||
if idx == -1:
|
||
return
|
||
tbody_start = html.find("<tbody>", idx)
|
||
tbody_end = html.find("</tbody>", tbody_start)
|
||
if tbody_start == -1 or tbody_end == -1:
|
||
return
|
||
tbody = html[tbody_start:tbody_end + 8]
|
||
|
||
# Folders — match both relative and absolute href, attributes may span lines
|
||
esc = re.escape(project)
|
||
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
|
||
for m in re.finditer(pattern, tbody, re.DOTALL):
|
||
name = m.group(1)
|
||
if not name or name.endswith("/stats/timeline"):
|
||
continue
|
||
yield True, name, ""
|
||
|
||
# Files
|
||
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
|
||
for m in re.finditer(pattern, tbody, re.DOTALL):
|
||
yield False, m.group(1), m.group(2)
|
||
|
||
|
||
def get_mirror_url(download_page_url):
|
||
"""Fetch the download landing page and extract direct mirror URL."""
|
||
try:
|
||
r = S.get(download_page_url, headers=UA, timeout=20, allow_redirects=False)
|
||
except Exception:
|
||
return None
|
||
if r is None:
|
||
return None
|
||
# Follow redirect chain to get the final mirror
|
||
location = r.headers.get("Location", "")
|
||
if location:
|
||
return location
|
||
return None
|
||
|
||
|
||
def _read_capped(r, cap):
|
||
"""Stream-read r up to cap bytes; None if oversized. Closes r. Caps BEFORE
|
||
the full body lands in RAM (Content-Length pre-check + stream-and-abort)."""
|
||
cl = r.headers.get("Content-Length")
|
||
if cl and cl.isdigit() and int(cl) > cap:
|
||
r.close()
|
||
return None
|
||
chunks = []
|
||
size = 0
|
||
try:
|
||
for chunk in r.iter_content(65536):
|
||
size += len(chunk)
|
||
if size > cap:
|
||
return None
|
||
chunks.append(chunk)
|
||
finally:
|
||
r.close()
|
||
return b"".join(chunks)
|
||
|
||
|
||
def download_file(download_page_url, cap):
|
||
"""Download file from SourceForge via redirect chain, capped at `cap` bytes.
|
||
|
||
The /download URL redirects (302) through mirrors to the actual file.
|
||
Returns (content, final_url) or (None, None) on failure/oversize.
|
||
"""
|
||
for attempt in (1, 2, 3):
|
||
try:
|
||
r = S.get(download_page_url, headers=UA, timeout=60,
|
||
allow_redirects=True, stream=True)
|
||
if r.status_code == 200:
|
||
data = _read_capped(r, cap)
|
||
return (data, r.url) if data else (None, None)
|
||
if r.status_code in (403, 404):
|
||
r.close()
|
||
return None, None
|
||
if r.status_code == 302:
|
||
loc = r.headers.get("Location", "")
|
||
r.close()
|
||
if loc:
|
||
r2 = S.get(loc, headers=UA, timeout=60, stream=True)
|
||
if r2.status_code == 200:
|
||
data = _read_capped(r2, cap)
|
||
return (data, loc) if data else (None, None)
|
||
r2.close()
|
||
return None, None
|
||
r.close()
|
||
raise RuntimeError(f"HTTP {r.status_code}")
|
||
except (requests.ConnectionError, requests.Timeout, RuntimeError):
|
||
if attempt == 3:
|
||
return None, None
|
||
time.sleep(5 * attempt)
|
||
return None, None
|
||
|
||
|
||
def scan_content(text, repo_label):
|
||
"""Run pattern scan + junk filter, db_add hits."""
|
||
n = 0
|
||
for prov, key in scan_text(text):
|
||
if isinstance(key, tuple):
|
||
key = key[0]
|
||
if is_junk(key) or len(key) < 30:
|
||
continue
|
||
db_add(key, prov, repo_label)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def scan_text_file(content, file_path, project):
|
||
"""Scan text content from a downloaded file."""
|
||
return scan_content(content, f"sf:{project}:{file_path}")
|
||
|
||
|
||
def scan_archive(content, archive_path, project):
|
||
"""Extract and scan text files from a zip/tar archive."""
|
||
n = 0
|
||
repo_label = f"sf:{project}:{archive_path}"
|
||
# Try zip
|
||
try:
|
||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||
for m in zf.infolist():
|
||
if m.is_dir() or m.file_size > FILE_CAP:
|
||
continue
|
||
fn = m.filename.lower()
|
||
if not any(fn.endswith(e) for e in TEXT_EXT):
|
||
continue
|
||
try:
|
||
data = zf.read(m).decode("utf-8", errors="ignore")
|
||
except Exception:
|
||
continue
|
||
n += scan_content(data, repo_label + "/" + m.filename)
|
||
return n
|
||
except (zipfile.BadZipFile, Exception):
|
||
pass
|
||
# Try tar.gz / tar.bz2
|
||
try:
|
||
tf = tarfile.open(fileobj=io.BytesIO(content), mode="r:*")
|
||
for m in tf.getmembers():
|
||
if m.isfile() and m.size <= FILE_CAP:
|
||
fn = m.name.lower()
|
||
if not any(fn.endswith(e) for e in TEXT_EXT):
|
||
continue
|
||
try:
|
||
fobj = tf.extractfile(m)
|
||
if fobj is None:
|
||
continue
|
||
data = fobj.read().decode("utf-8", errors="ignore")
|
||
except Exception:
|
||
continue
|
||
n += scan_content(data, repo_label + "/" + m.name)
|
||
except Exception:
|
||
pass
|
||
return n
|
||
|
||
|
||
def walk_project(project):
|
||
"""Walk a project's file tree and collect candidate files.
|
||
|
||
Returns list of (is_archive, path_info) where:
|
||
is_archive=True: path_info = (archive_url, archive_ext, archive_path)
|
||
is_archive=False: path_info = (text_url, file_path)
|
||
|
||
Capped by MAX_FOLDERS_PER_DEPTH per level.
|
||
"""
|
||
candidates = []
|
||
stack = [""] # folder paths relative to /files/
|
||
depth = 0
|
||
while stack and depth <= MAX_DEPTH:
|
||
next_stack = []
|
||
for folder in stack[:MAX_FOLDERS_PER_DEPTH]:
|
||
for is_folder, name, _ in list_dir(project, folder):
|
||
if is_folder:
|
||
if "/stats/" in name:
|
||
continue
|
||
next_stack.append(name)
|
||
else:
|
||
ext = _os.path.splitext(name)[1].lower()
|
||
if ext in TEXT_EXT:
|
||
candidates.append((False, (name, name)))
|
||
elif ext in (".zip", ".tar.gz", ".tar.bz2", ".tgz", ".tar",
|
||
".gz", ".bz2", ".xz"):
|
||
candidates.append((True, (name, ext, name)))
|
||
stack = next_stack
|
||
depth += 1
|
||
return candidates[:MAX_ITEMS]
|
||
|
||
|
||
def work(project):
|
||
"""Walk project, download candidates, scan. Returns (ok_keys, fail_count)."""
|
||
if not project_alive(project):
|
||
log(f"skip {project}: dead/moved")
|
||
return 0, 0
|
||
ok = 0
|
||
fail = 0
|
||
candidates = walk_project(project)
|
||
for is_archive, info in candidates:
|
||
if is_archive:
|
||
path, ext, archive_path = info
|
||
else:
|
||
path, file_path = info
|
||
|
||
sid = f"{project}:{path}"
|
||
if sid in seen:
|
||
continue
|
||
|
||
# Download via redirect chain
|
||
dl_page = f"https://sourceforge.net/projects/{project}/files/{path}/download"
|
||
content, final_url = download_file(
|
||
dl_page, ARCHIVE_CAP if is_archive else FILE_CAP)
|
||
if content is None:
|
||
fail += 1
|
||
continue
|
||
|
||
if is_archive:
|
||
if len(content) > ARCHIVE_CAP:
|
||
continue
|
||
n = scan_archive(content, archive_path, project)
|
||
else:
|
||
if len(content) > FILE_CAP:
|
||
continue
|
||
text = content.decode("utf-8", errors="ignore")
|
||
n = scan_text_file(text, path, project)
|
||
|
||
if n >= 0:
|
||
ok += n
|
||
mark_seen(sid)
|
||
|
||
return ok, fail
|
||
|
||
|
||
def load_sitemap_cursor():
|
||
try:
|
||
with open(SITEMAP_CURSOR_FILE, encoding="utf-8") as f:
|
||
return int(f.read().strip() or 0)
|
||
except (FileNotFoundError, ValueError):
|
||
return 0
|
||
|
||
|
||
def save_sitemap_cursor(v):
|
||
tmp = SITEMAP_CURSOR_FILE + ".tmp"
|
||
with open(tmp, "w", encoding="utf-8") as f:
|
||
f.write(str(v))
|
||
_os.replace(tmp, SITEMAP_CURSOR_FILE)
|
||
|
||
|
||
def discover_projects():
|
||
"""Один шард allura_sitemap за вызов → слаг-пул (ротация по курсору).
|
||
|
||
Курсор.advance только при успешном фетче шарда; ошибка → повтор того же.
|
||
"""
|
||
try:
|
||
r = sf_get(SITEMAP_INDEX, timeout=30)
|
||
if r is None:
|
||
return []
|
||
shards = _SHARD_RE.findall(r.text)
|
||
if not shards:
|
||
return []
|
||
ci = load_sitemap_cursor() % len(shards)
|
||
r2 = sf_get(shards[ci], timeout=90)
|
||
if r2 is None:
|
||
return []
|
||
except Exception:
|
||
return []
|
||
save_sitemap_cursor(ci + 1)
|
||
out, seen_l = [], set()
|
||
for rx in (_P_SLUG_RE, _PROJ_SLUG_RE):
|
||
for s in rx.findall(r2.text):
|
||
if s not in seen_l:
|
||
seen_l.add(s)
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def load_pool():
|
||
"""Seed + discovered pool, deduped, seed first."""
|
||
pool = list(SEED_PROJECTS)
|
||
known = set(pool)
|
||
try:
|
||
with open(PROJECTS_FILE, encoding="utf-8") as f:
|
||
for line in f:
|
||
s = line.strip()
|
||
if s and s not in known:
|
||
known.add(s)
|
||
pool.append(s)
|
||
except FileNotFoundError:
|
||
pass
|
||
return pool
|
||
|
||
|
||
def append_pool(new_slugs):
|
||
"""Append unseen slugs to PROJECTS_FILE. Returns count added."""
|
||
if not new_slugs:
|
||
return 0
|
||
known = set(load_pool())
|
||
fresh = [s for s in new_slugs if s not in known]
|
||
if fresh:
|
||
with open(PROJECTS_FILE, "a", encoding="utf-8") as f:
|
||
f.write("\n".join(fresh) + "\n")
|
||
return len(fresh)
|
||
|
||
|
||
def project_alive(project):
|
||
"""REST liveness: /rest/p/<short>/ (404 или status=moved → False)."""
|
||
try:
|
||
r = sf_get(f"https://sourceforge.net/rest/p/{project}/", timeout=15)
|
||
except Exception:
|
||
return True # сеть/403 — даём шанс walk'у
|
||
if r is None:
|
||
return False
|
||
try:
|
||
j = r.json()
|
||
except ValueError:
|
||
return True
|
||
return j.get("status") != "moved"
|
||
|
||
|
||
def cycle():
|
||
global total
|
||
total = 0
|
||
idx = load_cursor()
|
||
n_new = append_pool(discover_projects())
|
||
if n_new:
|
||
log(f"discovery sitemap shard: +{n_new} new projects")
|
||
projects = load_pool()
|
||
items = []
|
||
for _ in range(PROJECTS_PER_CYCLE):
|
||
p = projects[idx % len(projects)]
|
||
idx += 1
|
||
items.append(p)
|
||
save_cursor(idx)
|
||
|
||
ok = fail = 0
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||
futs = {ex.submit(work, p): p for p in items}
|
||
for fut in as_completed(futs):
|
||
p = futs[fut]
|
||
try:
|
||
res_ok, res_fail = fut.result()
|
||
except Exception:
|
||
res_ok, res_fail = 0, 1
|
||
ok += res_ok
|
||
fail += res_fail
|
||
|
||
log(f"cycle: idx={idx} projects={len(items)} ok={ok} fail={fail} +{total} keys")
|
||
|
||
|
||
def main():
|
||
log("sourceforge_eternal started")
|
||
if "--once" in sys.argv:
|
||
global MAX_ITEMS, PROJECTS_PER_CYCLE
|
||
MAX_ITEMS = min(MAX_ITEMS, 5)
|
||
PROJECTS_PER_CYCLE = 1
|
||
cycle()
|
||
return
|
||
while True:
|
||
try:
|
||
cycle()
|
||
except Exception as e:
|
||
log(f"cycle-err {type(e).__name__}: {e}")
|
||
time.sleep(SLEEP)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |