KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

cran_eternal.py

@@ -0,0 +1,228 @@
+"""ETERNAL CRAN firehose: PACKAGES.gz index + source tarball scan.
+
+Anonymous. Index = src/contrib/PACKAGES.gz (DCF, ~25k packages). Per cycle
+parse index, process BATCH=120 unseen packages (alphabetical, cursor file
+wraps). Each: tar.gz (cap 8MB) -> tarfile scan. seen=Package (versions
+churn skipped). mark_seen ONLY on success.
+"""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import gzip
+import hashlib
+import io
+import sqlite3
+import sys
+import tarfile
+import threading
+import time
+
+import requests
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.path.join(KF_BASE, "data/cran_eternal.log")
+CURSOR_FILE = os.path.join(KF_BASE, "data/cran_cursor.txt")
+SEEN_FILE = os.path.join(KF_BASE, "data/cran_seen.txt")
+N_THREADS = 6
+BATCH = 120
+TAR_CAP = 8 * 1024 * 1024
+FILE_CAP = 500_000
+SLEEP = 120
+
+from key_patterns import scan_text
+
+GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
+           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
+           "abcdefghijklmnopqrst", "1234567890abcdef")
+
+SCAN_EXT = (".R", ".Rmd", ".Rnw", ".py", ".js", ".json", ".yaml", ".yml",
+            ".env", ".Renviron", ".csv", ".xml", ".toml", ".cfg", ".ini")
+
+UA = {"User-Agent": "keyfarm-cran/1.0"}
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    with open(LOG, "a", encoding="utf-8") as f:
+        f.write(line)
+
+
+conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
+                       isolation_level=None)
+lock = threading.Lock()
+total = 0
+
+
+def db_add(key, prov, repo):
+    global total
+    with lock:
+        conn.execute(
+            "INSERT INTO keys (hash, val, prov, status, found, repo) "
+            "VALUES (?,?,?,?,?,?) "
+            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
+            "found=excluded.found, repo=excluded.repo "
+            "WHERE keys.status IN ('DEAD','ERR')",
+            (hashlib.sha1(key.encode()).hexdigest()[:16],
+             key, prov, "NEW", "cran", repo))
+        total += 1
+
+
+def load_cursor():
+    try:
+        with open(CURSOR_FILE, encoding="utf-8") as f:
+            return int(f.read().strip() or 0)
+    except (FileNotFoundError, ValueError):
+        return 0
+
+
+def save_cursor(c):
+    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
+        f.write(str(c))
+
+
+def load_seen():
+    s = set()
+    try:
+        with open(SEEN_FILE, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    s.add(line)
+    except FileNotFoundError:
+        pass
+    return s
+
+
+seen = load_seen()
+
+
+def mark_seen(p):
+    seen.add(p)
+    with open(SEEN_FILE, "a", encoding="utf-8") as f:
+        f.write(p + "\n")
+
+
+def work(pkg, ver):
+    """Fetch source tarball, scan. -1 on final fail."""
+    n = 0
+    url = (f"https://cran.r-project.org/src/contrib/"
+           f"{pkg}_{ver}.tar.gz")
+    for attempt in (1, 2, 3):
+        try:
+            r = requests.get(url, headers=UA, timeout=40)
+            if r.status_code == 404:
+                return 0
+            if r.status_code != 200:
+                raise RuntimeError(f"http {r.status_code}")
+            if len(r.content) > TAR_CAP:
+                return 0
+            tf = tarfile.open(fileobj=io.BytesIO(r.content), mode="r:gz")
+            for m in tf.getmembers():
+                if not m.isfile() or m.size > FILE_CAP:
+                    continue
+                fn = m.name.lower()
+                if not any(fn.endswith(e.lower()) for e in SCAN_EXT):
+                    continue
+                ef = tf.extractfile(m)
+                if ef is None:
+                    continue
+                data = ef.read()
+                if b"\x00" in data[:8192]:
+                    continue  # binary archive member — no real text key has NUL
+                text = data.decode("utf-8", errors="ignore")
+                for prov, key in scan_text(text):
+                    if isinstance(key, tuple):
+                        key = key[0]
+                    if any(g in key for g in GARBAGE) or len(key) < 30:
+                        continue
+                    db_add(key, prov, f"cran:{pkg}")
+                    n += 1
+            return n
+        except Exception:
+            if attempt == 3:
+                log(f"  pkg-err {pkg}")
+                return -1
+            time.sleep(8)
+    return -1
+
+
+def cycle():
+    global total
+    total = 0
+    # index (re-download each cycle: 1.9MB, versions bump in PACKAGES)
+    for attempt in (1, 2, 3):
+        try:
+            r = requests.get("https://cran.r-project.org/src/contrib/"
+                             "PACKAGES.gz", headers=UA, timeout=60)
+            if r.status_code != 200:
+                raise RuntimeError(f"http {r.status_code}")
+            break
+        except Exception as e:
+            if attempt == 3:
+                log(f"index-err {type(e).__name__}")
+                return
+            time.sleep(8)
+    txt = gzip.decompress(r.content).decode("utf-8", "replace")
+    pkgs = []
+    vers = {}
+    for block in txt.split("\n\n"):
+        pkg = ver = None
+        for line in block.splitlines():
+            if line.startswith("Package: "):
+                pkg = line[9:].strip()
+            elif line.startswith("Version: "):
+                ver = line[9:].strip()
+            if pkg and ver:
+                break
+        if pkg and ver:
+            pkgs.append(pkg)
+            vers[pkg] = ver
+    pkgs.sort()
+
+    idx = load_cursor() % len(pkgs)
+    items = []
+    for _ in range(len(pkgs)):
+        pkg = pkgs[idx]
+        if pkg not in seen:
+            items.append((pkg, vers[pkg]))
+            if len(items) >= BATCH:
+                idx = (idx + 1) % len(pkgs)
+                break
+        idx = (idx + 1) % len(pkgs)
+    save_cursor(idx)
+
+    ok = fail = 0
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
+        futs = {ex.submit(work, p, v): p for p, v in items}
+        for fut in as_completed(futs):
+            p = futs[fut]
+            try:
+                res = fut.result()
+            except Exception:
+                res = -1
+            if res == -1:
+                fail += 1          # NOT marked seen: retried next cycle
+            else:
+                ok += 1
+                mark_seen(p)
+
+    log(f"cycle: total={len(pkgs)} idx={idx} items={len(items)} ok={ok} "
+        f"fail={fail} +{total} keys")
+
+
+def main():
+    log("cran_eternal started")
+    if "--once" in sys.argv:
+        cycle()
+        return
+    while True:
+        try:
+            cycle()
+        except Exception as e:
+            log(f"cycle-err {type(e).__name__}: {e}")
+        time.sleep(SLEEP)
+
+
+if __name__ == "__main__":
+    main()