KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

deep_scan.py

@@ -0,0 +1,146 @@
+"""Deep repo scan: for each repo with hit history, pull full git tree,
+fetch config-ish files, grep PATTERNS. Catches files beyond search paging."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import hashlib
+import sqlite3
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+sys.path.insert(0, KF_BASE)
+import requests  # noqa: E402
+from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+N_THREADS = 16
+S = requests.Session()
+S.headers.update({"User-Agent": "keyfarm"})
+
+# file-name filters: high-yield config carriers
+HOT_NAMES = (".env", "config", "secret", "token", "apikey", "api_key",
+             "credential", "mcp.json", "settings", "keys.json", "auth",
+             "docker-compose", "dockerfile", ".npmrc", "application.yml",
+             "application.properties", "claude", "vercel", "netlify",
+             "readme", "example", "sample", "demo", "backup", "notes",
+             ".bak", "history", "scratch", "tmp", "old", "copy")
+HOT_EXT = (".env", ".json", ".yml", ".yaml", ".toml", ".ini", ".cfg",
+           ".properties", ".ipynb", ".sh", ".md", ".txt", ".pem.key")
+MAX_FILES_PER_REPO = 60
+
+
+def log(msg):
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+
+def load_repos():
+    db = sqlite3.connect(DB, timeout=60)
+    c = db.cursor()
+    c.execute("SELECT repo FROM keys WHERE repo LIKE '_%/_%' "
+              "AND found IN ('gh-eternal','gh-dork2','retro-scan','legacy-waves')")
+    repos = set()
+    for (r,) in c.fetchall():
+        if r.count("/") >= 1 and len(r) < 150 and "…" not in r:
+            repo = "/".join(r.split("/")[:2])
+            if repo.isascii() and not repo.endswith(".git"):
+                repos.add(repo)
+    db.close()
+    return sorted(repos)
+
+
+def repo_tree_files(repo, tok):
+    """Full recursive tree of repo via API. Returns list of candidate paths."""
+    try:
+        r = S.get(
+            f"https://api.github.com/repos/{repo}/git/trees/HEAD?recursive=1",
+            headers={"Authorization": f"Bearer {tok}"},
+            timeout=20)
+        if r.status_code == 409:  # empty repo
+            return []
+        if r.status_code != 200:
+            return None
+        paths = []
+        for t in r.json().get("tree", []):
+            if t.get("type") != "blob":
+                continue
+            p = t.get("path", "")
+            low = p.lower()
+            base = low.rsplit("/", 1)[-1]
+            if any(h in low for h in HOT_NAMES) or base.endswith(HOT_EXT):
+                if not any(x in low for x in ("node_modules/", "vendor/",
+                                              "dist/", "build/", ".min.")):
+                    paths.append(p)
+        return paths[:MAX_FILES_PER_REPO]
+    except Exception:
+        return None
+
+
+def fetch_and_scan(repo, path, tok):
+    try:
+        r = S.get(f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
+                  headers={"Authorization": f"Bearer {tok}"},
+                  timeout=20)
+        if r.status_code != 200 or len(r.text) > 2_000_000:
+            return []
+        found = []
+        for prov, rx in PATTERNS:
+            for key in rx.findall(r.text):
+                if isinstance(key, tuple):
+                    key = key[0]
+                if any(g in key for g in GARBAGE) or len(key) < 30:
+                    continue
+                found.append((prov, key))
+        return found
+    except Exception:
+        return []
+
+
+def main(threads=N_THREADS):
+    repos = load_repos()
+    log(f"repos: {len(repos)} to deep-scan")
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
+    cur = db.cursor()
+    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
+    new_total = 0
+    lock = __import__("threading").Lock()
+
+    def work(repo):
+        nonlocal new_total
+        tok = POOL.tokens[hash(repo) % len(POOL.tokens)]
+        paths = repo_tree_files(repo, tok)
+        if not paths:
+            return 0
+        n = 0
+        for p in paths:
+            for prov, key in fetch_and_scan(repo, p, tok):
+                if key in seen:
+                    continue
+                with lock:
+                    seen.add(key)
+                    cur.execute(
+                        "INSERT OR IGNORE INTO keys "
+                        "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
+                        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
+                         "NEW", "deep-scan", f"{repo}/{p}"))
+                    new_total += 1
+                n += 1
+        return n
+
+    done = 0
+    with ThreadPoolExecutor(max_workers=threads) as ex:
+        futs = {ex.submit(work, r): r for r in repos}
+        for fut in as_completed(futs):
+            done += 1
+            if done % 50 == 0:
+                with lock:
+                    db.commit()
+                log(f"{done}/{len(repos)} repos, +{new_total} new")
+    with lock:
+        db.commit()
+    log(f"done: +{new_total} new keys")
+    db.close()
+
+
+if __name__ == "__main__":
+    main()