KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

wiki_scan.py

@@ -0,0 +1,119 @@
+"""Scan GitHub wikis of repo-hits: code search does NOT index wiki pages.
+Wiki = separate git repo {owner}/{repo}.wiki.git — clone, grep PATTERNS."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+sys.path.insert(0, KF_BASE)
+from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.path.join(KF_BASE, "wiki_scan.log")
+
+
+def log(msg):
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+
+def wiki_urls_from_db(limit):
+    db = sqlite3.connect(DB, timeout=60)
+    c = db.cursor()
+    c.execute("SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != '' "
+              "GROUP BY repo ORDER BY count(*) DESC LIMIT ?", (limit,))
+    repos = []
+    for (r,) in c.fetchall():
+        repo = r.split("/")[0] + "/" + r.split("/")[1] if "/" in r else ""
+        if repo and repo not in repos:
+            repos.append(repo)
+    db.close()
+    return repos
+
+
+def clone_and_scan(repo):
+    """Clone wiki repo shallow, grep PATTERNS over all files."""
+    tok = POOL.tokens[0] if POOL.tokens else ""
+    url = f"https://x-access-token:{tok}@github.com/{repo}.wiki.git"
+    tmp = tempfile.mkdtemp(prefix="wiki_")
+    found = []
+    try:
+        r = subprocess.run(
+            ["git", "clone", "--depth", "1", "-q", url, tmp],
+            capture_output=True, timeout=30)
+        if r.returncode != 0:
+            return repo, "no-wiki", []
+        import re
+        texts = []
+        import pathlib
+        for f in pathlib.Path(tmp).rglob("*"):
+            if f.is_file() and f.stat().st_size < 1_000_000:
+                try:
+                    texts.append(f.read_text(errors="ignore"))
+                except Exception:
+                    pass
+        blob = "\n".join(texts)
+        for prov, rx in PATTERNS:
+            for key in rx.findall(blob):
+                if isinstance(key, tuple):
+                    key = key[0]
+                if any(g in key for g in GARBAGE) or len(key) < 30:
+                    continue
+                found.append((prov, key))
+        return repo, f"scanned ({len(texts)} files)", found
+    except subprocess.TimeoutExpired:
+        return repo, "timeout", []
+    except Exception as e:
+        return repo, f"error {e!r}", []
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def main(limit, threads):
+    repos = wiki_urls_from_db(limit)
+    log(f"start: {len(repos)} repos to wiki-scan, {threads} threads")
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
+    cur = db.cursor()
+    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
+    new_total = 0
+    lock_cur = __import__("threading").Lock()
+
+    def scan(repo):
+        nonlocal new_total
+        repo, status, found = clone_and_scan(repo)
+        n = 0
+        for prov, key in found:
+            if key in seen:
+                continue
+            with lock_cur:
+                seen.add(key)
+                import hashlib
+                cur.execute(
+                    "INSERT OR IGNORE INTO keys "
+                    "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
+                    (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
+                     "NEW", "wiki-scan", f"{repo}/wiki"))
+                db.commit()
+                new_total += 1
+            n += 1
+        if n or "scanned" in status:
+            log(f"{repo}: {status}, +{n} new")
+        return n
+
+    with ThreadPoolExecutor(max_workers=threads) as ex:
+        futs = [ex.submit(scan, r) for r in repos]
+        for _ in as_completed(futs):
+            pass
+    log(f"done: +{new_total} new keys")
+    db.close()
+
+
+if __name__ == "__main__":
+    limit = int(sys.argv[1]) if len(sys.argv) > 1 else 300
+    threads = int(sys.argv[2]) if len(sys.argv) > 2 else 8
+    main(limit, threads)