"""audit_ds_multiregion.py — re-validate DashScope-family rows with the real probes. Why this exists: the stored statuses came from a single-region chat probe, so region-bound keys read as 401 and free-tier-exhausted keys read as dead. Tier 1 re-runs the multi-region zero-token /v1/models probe; tier 2 (--chat-verify) proves the key can actually serve inference, because authentication alone put 8 arrears keys into a live rotator once already. Tier-2 semantics are provider-specific and were measured, not assumed: DashScope 1-token qwen-turbo completion; 400 Arrearage / 403 FreeTierOnly DeepSeek no extra call — probe_deepseek already returns the real balance (`deepseek $-0.06`); zero or negative means authenticated but empty ZAI/GLM api.z.ai answers with vendor codes: 1113 insufficient balance, 1308 five-hour usage window exhausted (key is fine, reset is scheduled), 1211 unknown model (our probe's fault, retry another model), 5xx (server fault, never a verdict on the key) Read-only on keys.db; the report lands outside any git repo. Run: python audit_ds_multiregion.py [--limit N] [--providers DASHSCOPE,ZAI] [--statuses WORKING,FREE,LIMITED,DEAD] [--workers 4] [--chat-verify] [--out PATH] """ from __future__ import annotations import argparse import importlib.util import json import os import re import sqlite3 import sys import tempfile import time from concurrent.futures import ThreadPoolExecutor HERE = os.path.dirname(os.path.abspath(__file__)) DB = os.path.join(HERE, "data", "keys.db") HUNT = os.path.join(HERE, "dashscope_hunt.py") ZAI_CHAT = ( ("https://api.z.ai/api/coding/paas/v4/chat/completions", "glm-5.2"), ("https://api.z.ai/api/paas/v4/chat/completions", "glm-4.5"), ("https://api.z.ai/api/paas/v4/chat/completions", "glm-4.5-air"), # free-tier model: a key can be funded for glm-4-flash while 1113-ing on the # paid ones, so this must stay in the ladder — dropping it hid 72 keys ("https://api.z.ai/api/paas/v4/chat/completions", "glm-4-flash"), ) DS_FAMILY = ("DASHSCOPE", "DEEPSEEK", "QWEN", "TONGYI", "ALIBABA", "BAILIAN") ZAI_FAMILY = ("ZAI", "ZHIPU", "GLM", "BIGMODEL", "ZHIPUAI") def load_hunt(): # keep the audit off the production DB: the module connects at import time os.environ["KF_DB"] = os.path.join(tempfile.gettempdir(), "kf_audit_probe.db") if HERE not in sys.path: sys.path.insert(0, HERE) spec = importlib.util.spec_from_file_location("dsh_audit", HUNT) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def candidates(providers, statuses, limit): conn = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) q = ("SELECT hash, val, prov, status, repo FROM keys " f"WHERE prov IN ({','.join('?' * len(providers))}) " f"AND status IN ({','.join('?' * len(statuses))}) " "ORDER BY CASE status WHEN 'WORKING' THEN 0 WHEN 'FREE' THEN 1 " "WHEN 'LIMITED' THEN 2 ELSE 3 END, id DESC LIMIT ?") rows = list(conn.execute(q, (*providers, *statuses, limit))) conn.close() return rows def probe_family(mod, prov, val): """Tier 1: pick the probe that actually speaks this provider's API. The regex cannot separate DASHSCOPE from DEEPSEEK (identical sk-<32hex>), so for that shape both probes run and the first positive wins; a billing/quota verdict beats a bare reject because it is more informative. """ p = (prov or "").upper() if p in ZAI_FAMILY or ("." in val and len(val.split(".")[0]) == 32): st, info = ("reject", None) for fn in (mod.probe_zai, mod.probe_bigmodel): st, info = fn(val) if st == "WORKING": return st, info return st, info if p in ("KIMI", "MOONSHOT"): return mod.probe_moonshot(val) if p == "SILICONFLOW": return mod.probe_siliconflow(val) best = ("reject", None) for fn in (mod.probe_dashscope, mod.probe_deepseek): st, info = fn(val) if st == "WORKING": return st, info if st in ("arrearage", "free_tier") and best[0] == "reject": best = (st, info) return best def probe_zai_chat(mod, val): """Tier 2 for ZAI/GLM, classified by the vendor's own error codes.""" last = ("net", None) for url, model in ZAI_CHAT: try: r = mod.requests.post( url, headers={**mod.UA, "Authorization": f"Bearer {val}", "Content-Type": "application/json"}, json={"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 1, "stream": False}, timeout=25) except Exception: last = ("net", None) continue body = (r.text or "")[:400] low = body.lower() if r.status_code == 200: return ("WORKING", f"chat-verified {model}") if '"1211"' in body or "unknown model" in low or "模型不存在" in body: last = ("endpoint_error", f"{model}:1211") # our model choice, retry continue if '"1308"' in body or "usage limit" in low: # five-hour window exhausted: the key is funded and works after reset return ("WORKING", f"window-exhausted {model}") if '"1113"' in body or "insufficient balance" in low or "余额不足" in body: return ("arrearage", f"{model}:1113") if r.status_code >= 500: last = ("net", r.status_code) # server fault, not a verdict continue if r.status_code in (401, 403): return ("reject", r.status_code) last = ("reject", r.status_code) return last def deepseek_usable(info): """Balance is already in the tier-1 info string; no second call needed.""" m = re.search(r"\$(-?[\d.]+)", str(info or "")) if not m: return None, "balance-unparsed" # unknown, never a demotion bal = float(m.group(1)) return (bal > 0), f"balance={bal}" def usable_verdict(u_st): if u_st is None or u_st in ("net", "endpoint_error", "error"): return None # unknown: leave the row alone return u_st == "WORKING" def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=60) ap.add_argument("--providers", default="DASHSCOPE") ap.add_argument("--statuses", default="WORKING,FREE,LIMITED,DEAD") ap.add_argument("--workers", type=int, default=4) ap.add_argument("--chat-verify", action="store_true", help="tier 2: prove the key can serve inference, not just auth") ap.add_argument("--out", default=None) args = ap.parse_args() mod = load_hunt() providers = tuple(p.strip().upper() for p in args.providers.split(",") if p.strip()) statuses = tuple(s.strip().upper() for s in args.statuses.split(",") if s.strip()) rows = candidates(providers, statuses, args.limit) print(f"tier1=multi-region /v1/models | candidates={len(rows)} " f"providers={providers} statuses={statuses} workers={args.workers} " f"chat_verify={args.chat_verify}") if not rows: print("nothing to audit") return 1 def one(row): h, val, prov, st, repo = row t0 = time.time() try: status, info = probe_family(mod, prov, val) except Exception as exc: # never let one key kill the audit status, info = "error", f"{type(exc).__name__}: {exc}" usable, usable_info = None, None if args.chat_verify and status == "WORKING": fam = (prov or "").upper() try: if fam == "DEEPSEEK" or str(info).startswith("deepseek $"): usable, usable_info = deepseek_usable(info) elif fam in ZAI_FAMILY: u_st, u_info = probe_zai_chat(mod, val) usable, usable_info = usable_verdict(u_st), f"{u_st}:{u_info}" elif fam in DS_FAMILY: u_st, u_info = mod.probe_dashscope_chat(val) usable, usable_info = usable_verdict(u_st), f"{u_st}:{u_info}" except Exception as exc: usable, usable_info = None, f"error:{type(exc).__name__}" return {"hash": h, "db_prov": prov, "db_status": st, "probe_status": status, "info": str(info)[:120], "usable": usable, "usable_info": str(usable_info)[:120], "masked": val[:8] + "…" + val[-4:], "repo": (repo or "")[:60], "secs": round(time.time() - t0, 1)} t0 = time.time() with ThreadPoolExecutor(max_workers=args.workers) as ex: out = list(ex.map(one, rows)) counts: dict[str, int] = {} for o in out: counts[o["probe_status"]] = counts.get(o["probe_status"], 0) + 1 revived = [o for o in out if o["probe_status"] == "WORKING" and o["db_status"] != "WORKING"] false_working = [o for o in out if o["probe_status"] != "WORKING" and o["db_status"] == "WORKING"] print(f"\nprobed {len(out)} in {time.time() - t0:.0f}s -> {counts}") if args.chat_verify: u = [o for o in out if o.get("usable") is True] nu = [o for o in out if o.get("usable") is False] unk = [o for o in out if o.get("usable") is None and o["probe_status"] == "WORKING"] print(f"tier-2: usable={len(u)} NOT-usable={len(nu)} unknown={len(unk)}") reasons: dict[str, int] = {} for o in nu: reasons[(o.get("usable_info") or "")[:34]] = reasons.get((o.get("usable_info") or "")[:34], 0) + 1 print(" not-usable reasons:", dict(sorted(reasons.items(), key=lambda x: -x[1])[:8])) byprov: dict[str, int] = {} for o in u: byprov[o["db_prov"]] = byprov.get(o["db_prov"], 0) + 1 print(" usable by provider:", byprov) print(f"revived (DB said not-WORKING, probe says WORKING): {len(revived)}") for o in revived[:8]: print(f" {o['masked']} db={o['db_status']} -> {o['info']}") print(f"stale WORKING (probe disagrees): {len(false_working)}") live = [o for o in out if o["probe_status"] == "WORKING"] print(f"\nLIVE now: {len(live)}") for o in live[:10]: print(f" {o['masked']} db={o['db_status']} {o['info']} " f"usable={o.get('usable')} ({o['secs']}s)") dest = args.out or os.path.join( os.path.dirname(HERE), "ds_multiregion_audit_" + time.strftime("%Y%m%d_%H%M%S") + ".json") with open(dest, "w", encoding="utf-8") as fh: json.dump({"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ"), "probe": "tier1 multi-region /v1/models (zero token) + tier2 provider-specific", "providers": list(providers), "statuses": list(statuses), "counts": counts, "revived": len(revived), "stale_working": len(false_working), "results": out}, fh, ensure_ascii=False, indent=1) print("report:", dest) return 0 if __name__ == "__main__": raise SystemExit(main())