KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

2026-10-20 07:50:07 +00:00
commit a5008d837b
178 changed files with 36909 additions and 0 deletions
--- a/retro_scan.py
+++ b/retro_scan.py
@@ -0,0 +1,110 @@
+"""Retro full-file scan: all (repo, path) hit-pairs already in DB.
+Cycle parser fetches full files only for current-cycle hits (runtime dedup);
+this one-shot re-harvests every historical hit with the complete file body."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import hashlib
+import json
+import re
+import sqlite3
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+sys.path.insert(0, KF_BASE)
+from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+N_THREADS = 16
+
+
+def log(msg):
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+
+def load_targets():
+    db = sqlite3.connect(DB, timeout=60)
+    c = db.cursor()
+    c.execute("SELECT repo FROM keys WHERE repo LIKE '%/%' "
+              "AND found IN ('gh-eternal','gh-dork2','legacy-waves','wiki')")
+    pairs = set()
+    for (r,) in c.fetchall():
+        if r.count("/") >= 1 and len(r) < 200 and "…" not in r:
+            parts = r.split("/")
+            repo = "/".join(parts[:2])
+            path = "/".join(parts[2:])
+            if repo and path and not path.startswith("commit"):
+                pairs.add((repo, path))
+    db.close()
+    return list(pairs)
+
+
+def fetch_and_scan(pair, tok):
+    repo, path = pair
+    try:
+        import requests
+        r = requests.get(
+            f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
+            headers={"Authorization": f"Bearer {tok}",
+                     "User-Agent": "keyfarm"},
+            timeout=20)
+        if r.status_code != 200 or len(r.text) > 2_000_000:
+            return pair, None
+        found = []
+        for prov, rx in PATTERNS:
+            for key in rx.findall(r.text):
+                if isinstance(key, tuple):
+                    key = key[0]
+                if any(g in key for g in GARBAGE) or len(key) < 30:
+                    continue
+                found.append((prov, key))
+        return pair, found
+    except Exception:
+        return pair, None
+
+
+def main():
+    pairs = load_targets()
+    log(f"targets: {len(pairs)} files")
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
+    cur = db.cursor()
+    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
+    new_total = 0
+    lock = __import__("threading").Lock()
+
+    def work(pair):
+        nonlocal new_total
+        tok = POOL.tokens[hash(pair[0]) % len(POOL.tokens)]
+        pair, found = fetch_and_scan(pair, tok)
+        n = 0
+        if found:
+            for prov, key in found:
+                if key in seen:
+                    continue
+                with lock:
+                    seen.add(key)
+                    cur.execute(
+                        "INSERT OR IGNORE INTO keys "
+                        "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
+                        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
+                         "NEW", "retro-scan", f"{pair[0]}/{pair[1]}"))
+                    new_total += 1
+                n += 1
+        return n
+
+    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
+        futs = {ex.submit(work, p): p for p in pairs}
+        done = 0
+        for fut in as_completed(futs):
+            done += 1
+            if done % 200 == 0:
+                db.commit()
+                log(f"{done}/{len(pairs)} scanned, +{new_total} new")
+    db.commit()
+    log(f"done: +{new_total} new keys")
+    db.close()
+
+
+if __name__ == "__main__":
+    main()