KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

sourceforge_eternal.py

@@ -0,0 +1,600 @@
+"""ETERNAL SourceForge: project discovery + file tree walk + text/archive scan.
+
+Discovery: allura_sitemap (13 шардов × ~10k /p/-слагов, lastmod ежедневно, без
+Cloudflare). /directory/?q= за Turnstile-challenge — мёртв для unattended loop.
+Курсор шарда: data/sourceforge_sitemap_cursor.txt; пул: data/sourceforge_projects.txt.
+Liveness: GET https://sourceforge.net/rest/p/<short>/ (trailing slash обязателен;
+status "moved"/404 → skip). Мертвы: /api/*, /rest/search/, /rest/p/ (listing).
+Files: https://sourceforge.net/projects/<p>/files/ (HTML parse of files_list table)
+       https://sourceforge.net/projects/<p>/files/<path>/download (mirror redirect)
+Auth: none
+"""
+import os as _os
+BASE = _os.path.dirname(_os.path.abspath(__file__))
+_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
+
+
+import hashlib
+import io
+import re
+import sqlite3
+import sys
+import tarfile
+import threading
+import time
+import zipfile
+
+import requests
+
+DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
+LOG = BASE + r"/data/sourceforge_eternal.log"
+CURSOR_FILE = BASE + r"/data/sourceforge_cursor.txt"
+SEEN_FILE = BASE + r"/data/sourceforge_seen.txt"
+PROJECTS_FILE = BASE + r"/data/sourceforge_projects.txt"
+
+N_THREADS = 4
+PROJECTS_PER_CYCLE = 5          # 5 projects per cycle → ~80 projects in 16 cycles (~24h)
+MAX_DEPTH = 3                   # max subfolder depth to recurse
+MAX_FOLDERS_PER_DEPTH = 10      # max folder listings per depth level (rate-limit guard)
+MAX_ITEMS = 200                 # per cycle cap; --once uses min(MAX_ITEMS, 5)
+FILE_CAP = 500_000              # max single text file
+ARCHIVE_CAP = 5 * 1024 * 1024  # max source archive to unpack
+SLEEP = 90
+
+from key_patterns import scan_text
+
+GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
+           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
+           "abcdefghijklmnopqrst", "1234567890abcdef")
+
+TEXT_EXT = (".txt", ".md", ".rst", ".cfg", ".ini", ".conf", ".env",
+            ".xml", ".json", ".yaml", ".yml", ".properties",
+            ".py", ".sh", ".bat", ".ps1", ".php", ".pl", ".rb",
+            ".js", ".ts", ".java", ".c", ".h", ".cpp", ".hpp",
+            ".sql", ".html", ".css", ".dockerfile", ".toml")
+
+UA = {"User-Agent": "Wget/1.21.3"}
+
+# Curated seed — popular open-source projects likely to bundle text/config files
+SEED_PROJECTS = [
+    # IDEs / Editors
+    "notepadplusplus", "codeblocks", "geany", "dev-cpp",
+    # Web / CMS / Database
+    "xampp", "wampserver", "wordpress", "joomla", "drupal",
+    "moodle", "phpmyadmin", "adminer", "mysqlcc",
+    "postgresql", "sqlitebrowser", "heidisql",
+    # Networking / Security
+    "wireshark", "nmap", "putty", "openssh", "openvpn",
+    "torproject", "truecrypt", "veracrypt", "gpg4win",
+    "keepass", "password-safe",
+    # File transfer / FTP
+    "filezilla", "winscp", "smartftp", "cyberduck",
+    # Multimedia
+    "audacity", "vlc", "ffmpeg", "handbrake", "shotcut",
+    "obsproject", "blender", "gimp", "inkscape",
+    # Development tools
+    "gitforwindows", "tortoisegit", "tortoisesvn", "mercurial",
+    "cmake", "mingw", "cygwin", "nsis", "innosetup",
+    "python", "rubyinstaller", "perl",
+    # Utilities
+    "sevenzip", "peazip", "ventoy", "rufus",
+    "crystaldiskinfo", "hwmonitor", "speedfan",
+    "sysinternals", "processhacker",
+    # System
+    "clamwin", "spybot", "ccleaner", "defraggler",
+    "wiztree", "everything",
+    # Emulators / Games
+    "dosbox", "pcsx2", "dolphin-emu", "mame",
+    "desmume", "visualboyadvance",
+    # Science / Math
+    "octave", "scilab", "geogebra",
+    # Office
+    "libreoffice", "openoffice", "abiword",
+    "sumatrapdf", "foxitreader",
+    # Chat / Comms
+    "pidgin", "irssi", "hexchat", "miranda-ng",
+    "thunderbird",
+    # Other
+    "scrcpy.mirror", "portableapps", "chocolate-doom",
+    "golang", "rustdesk",
+]
+
+# Filter seed to unique non-empty strings
+SEED_PROJECTS = sorted(set(s for s in SEED_PROJECTS if s and s.strip()))
+
+# Discovery: allura_sitemap — полный живой список проектов SF (без Cloudflare)
+SITEMAP_INDEX = "https://sourceforge.net/allura_sitemap/sitemap.xml"
+_SHARD_RE = re.compile(
+    r'<loc>(https://sourceforge\.net/allura_sitemap/sitemap-\d+\.xml)</loc>')
+_P_SLUG_RE = re.compile(r'sourceforge\.net/p/([a-z0-9][a-z0-9._-]{0,48})/')
+_PROJ_SLUG_RE = re.compile(
+    r'sourceforge\.net/projects/([a-z0-9][a-z0-9._-]{0,48})/')
+SITEMAP_CURSOR_FILE = BASE + r"/data/sourceforge_sitemap_cursor.txt"
+
+# Session with trust_env=False to bypass local dashscope proxy
+S = requests.Session()
+S.trust_env = False
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    with open(LOG, "a", encoding="utf-8") as f:
+        f.write(line)
+
+
+conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
+                       isolation_level=None)
+lock = threading.Lock()
+total = 0
+
+
+def db_add(key, prov, repo):
+    global total
+    with lock:
+        conn.execute(
+            "INSERT INTO keys (hash, val, prov, status, found, repo) "
+            "VALUES (?,?,?,?,?,?) "
+            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
+            "found=excluded.found, repo=excluded.repo "
+            "WHERE keys.status IN ('DEAD','ERR')",
+            (hashlib.sha1(key.encode()).hexdigest()[:16],
+             key, prov, "NEW", "sourceforge", repo))
+        total += 1
+
+
+def load_cursor():
+    try:
+        with open(CURSOR_FILE, encoding="utf-8") as f:
+            return int(f.read().strip() or 0)
+    except (FileNotFoundError, ValueError):
+        return 0
+
+
+def save_cursor(c):
+    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
+        f.write(str(c))
+
+
+def load_seen():
+    s = set()
+    try:
+        with open(SEEN_FILE, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    s.add(line)
+    except FileNotFoundError:
+        pass
+    return s
+
+
+seen = load_seen()
+
+
+def mark_seen(p):
+    seen.add(p)
+    with open(SEEN_FILE, "a", encoding="utf-8") as f:
+        f.write(p + "\n")
+
+
+def is_junk(v):
+    if any(g in v for g in GARBAGE):
+        return True
+    if len(set(v)) < 6:
+        return True
+    return False
+
+
+def sf_get(url, timeout=20):
+    """Thin wrapper with retry/backoff and 403 handling.
+
+    Uses module session S (trust_env=False) to bypass local proxy.
+    """
+    for attempt in (1, 2, 3):
+        try:
+            r = S.get(url, headers=UA, timeout=timeout)
+            if r.status_code in (429, 503):
+                ra = int(r.headers.get("Retry-After", 10 * attempt))
+                time.sleep(ra)
+                continue
+            if r.status_code == 403:
+                # Rate-limited; brief wait then retry
+                time.sleep(5 * attempt)
+                continue
+            if r.status_code == 404:
+                return None
+            if r.status_code != 200:
+                raise RuntimeError(f"HTTP {r.status_code}")
+            return r
+        except (requests.ConnectionError, requests.Timeout, RuntimeError):
+            if attempt == 3:
+                raise
+            time.sleep(5 * attempt)
+    return None
+
+
+def list_dir(project, folder=""):
+    """Parse files_list table from project's files page.
+
+    Yields (is_folder, name, display_name).
+    is_folder=True: name is relative URL folder path
+    is_folder=False: name is full relative file path (for /download)
+    """
+    if folder:
+        url = f"https://sourceforge.net/projects/{project}/files/{folder}/"
+    else:
+        url = f"https://sourceforge.net/projects/{project}/files/"
+    try:
+        r = sf_get(url, timeout=20)
+    except Exception:
+        return
+    if r is None:
+        return
+    html = r.text
+    # Find tbody
+    idx = html.find('<table id="files_list"')
+    if idx == -1:
+        idx = html.find("<table id='files_list'")
+    if idx == -1:
+        return
+    tbody_start = html.find("<tbody>", idx)
+    tbody_end = html.find("</tbody>", tbody_start)
+    if tbody_start == -1 or tbody_end == -1:
+        return
+    tbody = html[tbody_start:tbody_end + 8]
+
+    # Folders — match both relative and absolute href, attributes may span lines
+    esc = re.escape(project)
+    pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
+    for m in re.finditer(pattern, tbody, re.DOTALL):
+        name = m.group(1)
+        if not name or name.endswith("/stats/timeline"):
+            continue
+        yield True, name, ""
+
+    # Files
+    pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
+    for m in re.finditer(pattern, tbody, re.DOTALL):
+        yield False, m.group(1), m.group(2)
+
+
+def get_mirror_url(download_page_url):
+    """Fetch the download landing page and extract direct mirror URL."""
+    try:
+        r = S.get(download_page_url, headers=UA, timeout=20, allow_redirects=False)
+    except Exception:
+        return None
+    if r is None:
+        return None
+    # Follow redirect chain to get the final mirror
+    location = r.headers.get("Location", "")
+    if location:
+        return location
+    return None
+
+
+def _read_capped(r, cap):
+    """Stream-read r up to cap bytes; None if oversized. Closes r. Caps BEFORE
+    the full body lands in RAM (Content-Length pre-check + stream-and-abort)."""
+    cl = r.headers.get("Content-Length")
+    if cl and cl.isdigit() and int(cl) > cap:
+        r.close()
+        return None
+    chunks = []
+    size = 0
+    try:
+        for chunk in r.iter_content(65536):
+            size += len(chunk)
+            if size > cap:
+                return None
+            chunks.append(chunk)
+    finally:
+        r.close()
+    return b"".join(chunks)
+
+
+def download_file(download_page_url, cap):
+    """Download file from SourceForge via redirect chain, capped at `cap` bytes.
+
+    The /download URL redirects (302) through mirrors to the actual file.
+    Returns (content, final_url) or (None, None) on failure/oversize.
+    """
+    for attempt in (1, 2, 3):
+        try:
+            r = S.get(download_page_url, headers=UA, timeout=60,
+                      allow_redirects=True, stream=True)
+            if r.status_code == 200:
+                data = _read_capped(r, cap)
+                return (data, r.url) if data else (None, None)
+            if r.status_code in (403, 404):
+                r.close()
+                return None, None
+            if r.status_code == 302:
+                loc = r.headers.get("Location", "")
+                r.close()
+                if loc:
+                    r2 = S.get(loc, headers=UA, timeout=60, stream=True)
+                    if r2.status_code == 200:
+                        data = _read_capped(r2, cap)
+                        return (data, loc) if data else (None, None)
+                    r2.close()
+                    return None, None
+            r.close()
+            raise RuntimeError(f"HTTP {r.status_code}")
+        except (requests.ConnectionError, requests.Timeout, RuntimeError):
+            if attempt == 3:
+                return None, None
+            time.sleep(5 * attempt)
+    return None, None
+
+
+def scan_content(text, repo_label):
+    """Run pattern scan + junk filter, db_add hits."""
+    n = 0
+    for prov, key in scan_text(text):
+        if isinstance(key, tuple):
+            key = key[0]
+        if is_junk(key) or len(key) < 30:
+            continue
+        db_add(key, prov, repo_label)
+        n += 1
+    return n
+
+
+def scan_text_file(content, file_path, project):
+    """Scan text content from a downloaded file."""
+    return scan_content(content, f"sf:{project}:{file_path}")
+
+
+def scan_archive(content, archive_path, project):
+    """Extract and scan text files from a zip/tar archive."""
+    n = 0
+    repo_label = f"sf:{project}:{archive_path}"
+    # Try zip
+    try:
+        zf = zipfile.ZipFile(io.BytesIO(content))
+        for m in zf.infolist():
+            if m.is_dir() or m.file_size > FILE_CAP:
+                continue
+            fn = m.filename.lower()
+            if not any(fn.endswith(e) for e in TEXT_EXT):
+                continue
+            try:
+                data = zf.read(m).decode("utf-8", errors="ignore")
+            except Exception:
+                continue
+            n += scan_content(data, repo_label + "/" + m.filename)
+        return n
+    except (zipfile.BadZipFile, Exception):
+        pass
+    # Try tar.gz / tar.bz2
+    try:
+        tf = tarfile.open(fileobj=io.BytesIO(content), mode="r:*")
+        for m in tf.getmembers():
+            if m.isfile() and m.size <= FILE_CAP:
+                fn = m.name.lower()
+                if not any(fn.endswith(e) for e in TEXT_EXT):
+                    continue
+                try:
+                    fobj = tf.extractfile(m)
+                    if fobj is None:
+                        continue
+                    data = fobj.read().decode("utf-8", errors="ignore")
+                except Exception:
+                    continue
+                n += scan_content(data, repo_label + "/" + m.name)
+    except Exception:
+        pass
+    return n
+
+
+def walk_project(project):
+    """Walk a project's file tree and collect candidate files.
+
+    Returns list of (is_archive, path_info) where:
+      is_archive=True:  path_info = (archive_url, archive_ext, archive_path)
+      is_archive=False: path_info = (text_url, file_path)
+
+    Capped by MAX_FOLDERS_PER_DEPTH per level.
+    """
+    candidates = []
+    stack = [""]  # folder paths relative to /files/
+    depth = 0
+    while stack and depth <= MAX_DEPTH:
+        next_stack = []
+        for folder in stack[:MAX_FOLDERS_PER_DEPTH]:
+            for is_folder, name, _ in list_dir(project, folder):
+                if is_folder:
+                    if "/stats/" in name:
+                        continue
+                    next_stack.append(name)
+                else:
+                    ext = _os.path.splitext(name)[1].lower()
+                    if ext in TEXT_EXT:
+                        candidates.append((False, (name, name)))
+                    elif ext in (".zip", ".tar.gz", ".tar.bz2", ".tgz", ".tar",
+                                 ".gz", ".bz2", ".xz"):
+                        candidates.append((True, (name, ext, name)))
+        stack = next_stack
+        depth += 1
+    return candidates[:MAX_ITEMS]
+
+
+def work(project):
+    """Walk project, download candidates, scan. Returns (ok_keys, fail_count)."""
+    if not project_alive(project):
+        log(f"skip {project}: dead/moved")
+        return 0, 0
+    ok = 0
+    fail = 0
+    candidates = walk_project(project)
+    for is_archive, info in candidates:
+        if is_archive:
+            path, ext, archive_path = info
+        else:
+            path, file_path = info
+
+        sid = f"{project}:{path}"
+        if sid in seen:
+            continue
+
+        # Download via redirect chain
+        dl_page = f"https://sourceforge.net/projects/{project}/files/{path}/download"
+        content, final_url = download_file(
+            dl_page, ARCHIVE_CAP if is_archive else FILE_CAP)
+        if content is None:
+            fail += 1
+            continue
+
+        if is_archive:
+            if len(content) > ARCHIVE_CAP:
+                continue
+            n = scan_archive(content, archive_path, project)
+        else:
+            if len(content) > FILE_CAP:
+                continue
+            text = content.decode("utf-8", errors="ignore")
+            n = scan_text_file(text, path, project)
+
+        if n >= 0:
+            ok += n
+            mark_seen(sid)
+
+    return ok, fail
+
+
+def load_sitemap_cursor():
+    try:
+        with open(SITEMAP_CURSOR_FILE, encoding="utf-8") as f:
+            return int(f.read().strip() or 0)
+    except (FileNotFoundError, ValueError):
+        return 0
+
+
+def save_sitemap_cursor(v):
+    tmp = SITEMAP_CURSOR_FILE + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        f.write(str(v))
+    _os.replace(tmp, SITEMAP_CURSOR_FILE)
+
+
+def discover_projects():
+    """Один шард allura_sitemap за вызов → слаг-пул (ротация по курсору).
+
+    Курсор.advance только при успешном фетче шарда; ошибка → повтор того же.
+    """
+    try:
+        r = sf_get(SITEMAP_INDEX, timeout=30)
+        if r is None:
+            return []
+        shards = _SHARD_RE.findall(r.text)
+        if not shards:
+            return []
+        ci = load_sitemap_cursor() % len(shards)
+        r2 = sf_get(shards[ci], timeout=90)
+        if r2 is None:
+            return []
+    except Exception:
+        return []
+    save_sitemap_cursor(ci + 1)
+    out, seen_l = [], set()
+    for rx in (_P_SLUG_RE, _PROJ_SLUG_RE):
+        for s in rx.findall(r2.text):
+            if s not in seen_l:
+                seen_l.add(s)
+                out.append(s)
+    return out
+
+
+def load_pool():
+    """Seed + discovered pool, deduped, seed first."""
+    pool = list(SEED_PROJECTS)
+    known = set(pool)
+    try:
+        with open(PROJECTS_FILE, encoding="utf-8") as f:
+            for line in f:
+                s = line.strip()
+                if s and s not in known:
+                    known.add(s)
+                    pool.append(s)
+    except FileNotFoundError:
+        pass
+    return pool
+
+
+def append_pool(new_slugs):
+    """Append unseen slugs to PROJECTS_FILE. Returns count added."""
+    if not new_slugs:
+        return 0
+    known = set(load_pool())
+    fresh = [s for s in new_slugs if s not in known]
+    if fresh:
+        with open(PROJECTS_FILE, "a", encoding="utf-8") as f:
+            f.write("\n".join(fresh) + "\n")
+    return len(fresh)
+
+
+def project_alive(project):
+    """REST liveness: /rest/p/<short>/ (404 или status=moved → False)."""
+    try:
+        r = sf_get(f"https://sourceforge.net/rest/p/{project}/", timeout=15)
+    except Exception:
+        return True  # сеть/403 — даём шанс walk'у
+    if r is None:
+        return False
+    try:
+        j = r.json()
+    except ValueError:
+        return True
+    return j.get("status") != "moved"
+
+
+def cycle():
+    global total
+    total = 0
+    idx = load_cursor()
+    n_new = append_pool(discover_projects())
+    if n_new:
+        log(f"discovery sitemap shard: +{n_new} new projects")
+    projects = load_pool()
+    items = []
+    for _ in range(PROJECTS_PER_CYCLE):
+        p = projects[idx % len(projects)]
+        idx += 1
+        items.append(p)
+    save_cursor(idx)
+
+    ok = fail = 0
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
+        futs = {ex.submit(work, p): p for p in items}
+        for fut in as_completed(futs):
+            p = futs[fut]
+            try:
+                res_ok, res_fail = fut.result()
+            except Exception:
+                res_ok, res_fail = 0, 1
+            ok += res_ok
+            fail += res_fail
+
+    log(f"cycle: idx={idx} projects={len(items)} ok={ok} fail={fail} +{total} keys")
+
+
+def main():
+    log("sourceforge_eternal started")
+    if "--once" in sys.argv:
+        global MAX_ITEMS, PROJECTS_PER_CYCLE
+        MAX_ITEMS = min(MAX_ITEMS, 5)
+        PROJECTS_PER_CYCLE = 1
+        cycle()
+        return
+    while True:
+        try:
+            cycle()
+        except Exception as e:
+            log(f"cycle-err {type(e).__name__}: {e}")
+        time.sleep(SLEEP)
+
+
+if __name__ == "__main__":
+    main()