KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

2026-10-20 07:50:07 +00:00
commit a5008d837b
178 changed files with 36909 additions and 0 deletions
--- a/apply_ds_audit.py
+++ b/apply_ds_audit.py
@@ -0,0 +1,142 @@
+"""apply_ds_audit.py — write multi-region probe verdicts back into keys.db.
+
+audit_ds_multiregion.py proved the stored statuses are wrong in both directions:
+8 rows marked DEAD authenticate fine (region-locked keys probed only on Beijing
+before), and 31 rows marked WORKING now reject everywhere. This applies the
+verdicts, using the closed status vocabulary the rest of the farm consumes
+(NEW/WORKING/DEAD/LIMITED/FREE/ERR) and putting the fine class in `plan`.
+
+Never marks DEAD from a transport/endpoint verdict: net and endpoint_error leave
+the row untouched for the next pass.
+
+Run: python apply_ds_audit.py [--audit PATH] [--dry]
+"""
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import shutil
+import sqlite3
+import sys
+import time
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DB = os.path.join(HERE, "data", "keys.db")
+
+# probe verdict -> (status, plan prefix); None status means "do not touch"
+VERDICT = {
+    "WORKING": ("WORKING", "auth-ok"),
+    "arrearage": ("LIMITED", "arrearage"),
+    "free_tier": ("LIMITED", "free-tier-exhausted"),
+    "reject": ("DEAD", "rejected-all-regions"),
+    "throttled": (None, "throttled"),        # caller/IP side: retry later
+    "net": (None, "net"),
+    "endpoint_error": (None, "endpoint"),
+    "error": (None, "probe-error"),
+}
+
+
+def latest_audit(explicit: str | None) -> str:
+    if explicit:
+        return explicit
+    cands = sorted(glob.glob(os.path.join(os.path.dirname(HERE),
+                                          "ds_multiregion_audit_*.json")))
+    cands += sorted(glob.glob(os.path.join(HERE, "ds_multiregion_audit_*.json")))
+    if not cands:
+        sys.exit("no ds_multiregion_audit_*.json found — run audit_ds_multiregion.py first")
+    return cands[-1]
+
+
+# a positive probe proves the real provider; the regex guess in `prov` cannot
+# separate DASHSCOPE from DEEPSEEK (identical sk-<32hex> shape)
+PROOF = (("dashscope", "DASHSCOPE"), ("deepseek", "DEEPSEEK"), ("zai", "ZAI"),
+         ("zhipu", "ZAI"), ("bigmodel", "ZAI"), ("moonshot", "MOONSHOT"),
+         ("kimi", "KIMI"), ("siliconflow", "SILICONFLOW"))
+
+
+def proved_provider(info):
+    head = str(info or "").strip().lower()
+    for needle, prov in PROOF:
+        if head.startswith(needle):
+            return prov
+    return None
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--audit", default=None)
+    ap.add_argument("--dry", action="store_true")
+    args = ap.parse_args()
+
+    path = latest_audit(args.audit)
+    data = json.load(open(path, encoding="utf-8"))
+    rows = data.get("results") or []
+    print(f"audit: {path} ({len(rows)} verdicts, generated {data.get('generated')})")
+
+    conn = sqlite3.connect(DB, timeout=60)
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=30000")
+    before = dict(conn.execute("SELECT status, COUNT(*) FROM keys GROUP BY status"))
+    changes: dict[str, int] = {}
+    skipped = 0
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    for r in rows:
+        h = r.get("hash")
+        verdict = r.get("probe_status")
+        mapping = VERDICT.get(verdict)
+        if not h or mapping is None:
+            skipped += 1
+            continue
+        status, prefix = mapping
+        if status == "WORKING" and r.get("usable") is False:
+            # authenticated but cannot run inference (arrears / free tier spent):
+            # LIMITED keeps it out of rotators without calling the key dead
+            status = "LIMITED"
+            prefix = str(r.get("usable_info") or "unusable")[:40]
+        if status is None:
+            skipped += 1                     # transient: leave for the next pass
+            continue
+        cur = conn.execute("SELECT status FROM keys WHERE hash=?", (h,)).fetchone()
+        if not cur:
+            skipped += 1
+            continue
+        if cur[0] == status:
+            continue
+        plan = f"{prefix}: {str(r.get('info') or '')[:150]}"
+        proved = proved_provider(r.get("info")) if status == "WORKING" else None
+        if args.dry:
+            changes[f"{cur[0]}->{status}"] = changes.get(f"{cur[0]}->{status}", 0) + 1
+            continue
+        if proved:
+            conn.execute("UPDATE keys SET status=?, prov=?, plan=?, remaining=? "
+                         "WHERE hash=?", (status, proved, plan,
+                                          "OK" if status == "WORKING" else None, h))
+        else:
+            conn.execute("UPDATE keys SET status=?, plan=?, remaining=? WHERE hash=?",
+                         (status, plan, "OK" if status == "WORKING" else None, h))
+        changes[f"{cur[0]}->{status}"] = changes.get(f"{cur[0]}->{status}", 0) + 1
+
+    if not args.dry and sum(changes.values()):
+        # snapshot BEFORE committing — this copy is the rollback point
+        bak = DB + ".bak-apply-" + time.strftime("%Y%m%d_%H%M%S")
+        shutil.copy2(DB, bak)
+        print("backup:", bak)
+        conn.commit()
+    elif not args.dry:
+        conn.commit()
+
+    after = dict(conn.execute("SELECT status, COUNT(*) FROM keys GROUP BY status"))
+    conn.close()
+    print("skipped (transient/absent):", skipped)
+    print("transitions:", json.dumps(changes, ensure_ascii=False) or "{}")
+    print("status counts before:", json.dumps(before, ensure_ascii=False))
+    print("status counts after :", json.dumps(after, ensure_ascii=False))
+    print("DRY RUN — nothing written" if args.dry else "APPLIED")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())