254 lines
11 KiB
Python
254 lines
11 KiB
Python
"""audit_ds_multiregion.py — re-validate DashScope-family rows with the real probes.
|
|
|
|
Why this exists: the stored statuses came from a single-region chat probe, so
|
|
region-bound keys read as 401 and free-tier-exhausted keys read as dead. Tier 1
|
|
re-runs the multi-region zero-token /v1/models probe; tier 2 (--chat-verify)
|
|
proves the key can actually serve inference, because authentication alone put
|
|
8 arrears keys into a live rotator once already.
|
|
|
|
Tier-2 semantics are provider-specific and were measured, not assumed:
|
|
DashScope 1-token qwen-turbo completion; 400 Arrearage / 403 FreeTierOnly
|
|
DeepSeek no extra call — probe_deepseek already returns the real balance
|
|
(`deepseek $-0.06`); zero or negative means authenticated but empty
|
|
ZAI/GLM api.z.ai answers with vendor codes: 1113 insufficient balance,
|
|
1308 five-hour usage window exhausted (key is fine, reset is
|
|
scheduled), 1211 unknown model (our probe's fault, retry another
|
|
model), 5xx (server fault, never a verdict on the key)
|
|
|
|
Read-only on keys.db; the report lands outside any git repo.
|
|
|
|
Run: python audit_ds_multiregion.py [--limit N] [--providers DASHSCOPE,ZAI]
|
|
[--statuses WORKING,FREE,LIMITED,DEAD]
|
|
[--workers 4] [--chat-verify] [--out PATH]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
DB = os.path.join(HERE, "data", "keys.db")
|
|
HUNT = os.path.join(HERE, "dashscope_hunt.py")
|
|
ZAI_CHAT = (
|
|
("https://api.z.ai/api/coding/paas/v4/chat/completions", "glm-5.2"),
|
|
("https://api.z.ai/api/paas/v4/chat/completions", "glm-4.5"),
|
|
("https://api.z.ai/api/paas/v4/chat/completions", "glm-4.5-air"),
|
|
# free-tier model: a key can be funded for glm-4-flash while 1113-ing on the
|
|
# paid ones, so this must stay in the ladder — dropping it hid 72 keys
|
|
("https://api.z.ai/api/paas/v4/chat/completions", "glm-4-flash"),
|
|
)
|
|
|
|
DS_FAMILY = ("DASHSCOPE", "DEEPSEEK", "QWEN", "TONGYI", "ALIBABA", "BAILIAN")
|
|
ZAI_FAMILY = ("ZAI", "ZHIPU", "GLM", "BIGMODEL", "ZHIPUAI")
|
|
|
|
def load_hunt():
|
|
# keep the audit off the production DB: the module connects at import time
|
|
os.environ["KF_DB"] = os.path.join(tempfile.gettempdir(), "kf_audit_probe.db")
|
|
if HERE not in sys.path:
|
|
sys.path.insert(0, HERE)
|
|
spec = importlib.util.spec_from_file_location("dsh_audit", HUNT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
def candidates(providers, statuses, limit):
|
|
conn = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
|
|
q = ("SELECT hash, val, prov, status, repo FROM keys "
|
|
f"WHERE prov IN ({','.join('?' * len(providers))}) "
|
|
f"AND status IN ({','.join('?' * len(statuses))}) "
|
|
"ORDER BY CASE status WHEN 'WORKING' THEN 0 WHEN 'FREE' THEN 1 "
|
|
"WHEN 'LIMITED' THEN 2 ELSE 3 END, id DESC LIMIT ?")
|
|
rows = list(conn.execute(q, (*providers, *statuses, limit)))
|
|
conn.close()
|
|
return rows
|
|
|
|
|
|
def probe_family(mod, prov, val):
|
|
"""Tier 1: pick the probe that actually speaks this provider's API.
|
|
|
|
The regex cannot separate DASHSCOPE from DEEPSEEK (identical sk-<32hex>), so
|
|
for that shape both probes run and the first positive wins; a billing/quota
|
|
verdict beats a bare reject because it is more informative.
|
|
"""
|
|
p = (prov or "").upper()
|
|
if p in ZAI_FAMILY or ("." in val and len(val.split(".")[0]) == 32):
|
|
st, info = ("reject", None)
|
|
for fn in (mod.probe_zai, mod.probe_bigmodel):
|
|
st, info = fn(val)
|
|
if st == "WORKING":
|
|
return st, info
|
|
return st, info
|
|
if p in ("KIMI", "MOONSHOT"):
|
|
return mod.probe_moonshot(val)
|
|
if p == "SILICONFLOW":
|
|
return mod.probe_siliconflow(val)
|
|
best = ("reject", None)
|
|
for fn in (mod.probe_dashscope, mod.probe_deepseek):
|
|
st, info = fn(val)
|
|
if st == "WORKING":
|
|
return st, info
|
|
if st in ("arrearage", "free_tier") and best[0] == "reject":
|
|
best = (st, info)
|
|
return best
|
|
|
|
|
|
def probe_zai_chat(mod, val):
|
|
"""Tier 2 for ZAI/GLM, classified by the vendor's own error codes."""
|
|
last = ("net", None)
|
|
for url, model in ZAI_CHAT:
|
|
try:
|
|
r = mod.requests.post(
|
|
url,
|
|
headers={**mod.UA, "Authorization": f"Bearer {val}",
|
|
"Content-Type": "application/json"},
|
|
json={"model": model,
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"max_tokens": 1, "stream": False},
|
|
timeout=25)
|
|
except Exception:
|
|
last = ("net", None)
|
|
continue
|
|
body = (r.text or "")[:400]
|
|
low = body.lower()
|
|
if r.status_code == 200:
|
|
return ("WORKING", f"chat-verified {model}")
|
|
if '"1211"' in body or "unknown model" in low or "模型不存在" in body:
|
|
last = ("endpoint_error", f"{model}:1211") # our model choice, retry
|
|
continue
|
|
if '"1308"' in body or "usage limit" in low:
|
|
# five-hour window exhausted: the key is funded and works after reset
|
|
return ("WORKING", f"window-exhausted {model}")
|
|
if '"1113"' in body or "insufficient balance" in low or "余额不足" in body:
|
|
return ("arrearage", f"{model}:1113")
|
|
if r.status_code >= 500:
|
|
last = ("net", r.status_code) # server fault, not a verdict
|
|
continue
|
|
if r.status_code in (401, 403):
|
|
return ("reject", r.status_code)
|
|
last = ("reject", r.status_code)
|
|
return last
|
|
|
|
|
|
def deepseek_usable(info):
|
|
"""Balance is already in the tier-1 info string; no second call needed."""
|
|
m = re.search(r"\$(-?[\d.]+)", str(info or ""))
|
|
if not m:
|
|
return None, "balance-unparsed" # unknown, never a demotion
|
|
bal = float(m.group(1))
|
|
return (bal > 0), f"balance={bal}"
|
|
|
|
|
|
def usable_verdict(u_st):
|
|
if u_st is None or u_st in ("net", "endpoint_error", "error"):
|
|
return None # unknown: leave the row alone
|
|
return u_st == "WORKING"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=60)
|
|
ap.add_argument("--providers", default="DASHSCOPE")
|
|
ap.add_argument("--statuses", default="WORKING,FREE,LIMITED,DEAD")
|
|
ap.add_argument("--workers", type=int, default=4)
|
|
ap.add_argument("--chat-verify", action="store_true",
|
|
help="tier 2: prove the key can serve inference, not just auth")
|
|
ap.add_argument("--out", default=None)
|
|
args = ap.parse_args()
|
|
|
|
mod = load_hunt()
|
|
providers = tuple(p.strip().upper() for p in args.providers.split(",") if p.strip())
|
|
statuses = tuple(s.strip().upper() for s in args.statuses.split(",") if s.strip())
|
|
rows = candidates(providers, statuses, args.limit)
|
|
print(f"tier1=multi-region /v1/models | candidates={len(rows)} "
|
|
f"providers={providers} statuses={statuses} workers={args.workers} "
|
|
f"chat_verify={args.chat_verify}")
|
|
if not rows:
|
|
print("nothing to audit")
|
|
return 1
|
|
|
|
def one(row):
|
|
h, val, prov, st, repo = row
|
|
t0 = time.time()
|
|
try:
|
|
status, info = probe_family(mod, prov, val)
|
|
except Exception as exc: # never let one key kill the audit
|
|
status, info = "error", f"{type(exc).__name__}: {exc}"
|
|
usable, usable_info = None, None
|
|
if args.chat_verify and status == "WORKING":
|
|
fam = (prov or "").upper()
|
|
try:
|
|
if fam == "DEEPSEEK" or str(info).startswith("deepseek $"):
|
|
usable, usable_info = deepseek_usable(info)
|
|
elif fam in ZAI_FAMILY:
|
|
u_st, u_info = probe_zai_chat(mod, val)
|
|
usable, usable_info = usable_verdict(u_st), f"{u_st}:{u_info}"
|
|
elif fam in DS_FAMILY:
|
|
u_st, u_info = mod.probe_dashscope_chat(val)
|
|
usable, usable_info = usable_verdict(u_st), f"{u_st}:{u_info}"
|
|
except Exception as exc:
|
|
usable, usable_info = None, f"error:{type(exc).__name__}"
|
|
return {"hash": h, "db_prov": prov, "db_status": st,
|
|
"probe_status": status, "info": str(info)[:120],
|
|
"usable": usable, "usable_info": str(usable_info)[:120],
|
|
"masked": val[:8] + "…" + val[-4:], "repo": (repo or "")[:60],
|
|
"secs": round(time.time() - t0, 1)}
|
|
|
|
t0 = time.time()
|
|
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
|
out = list(ex.map(one, rows))
|
|
|
|
counts: dict[str, int] = {}
|
|
for o in out:
|
|
counts[o["probe_status"]] = counts.get(o["probe_status"], 0) + 1
|
|
revived = [o for o in out if o["probe_status"] == "WORKING" and o["db_status"] != "WORKING"]
|
|
false_working = [o for o in out if o["probe_status"] != "WORKING" and o["db_status"] == "WORKING"]
|
|
print(f"\nprobed {len(out)} in {time.time() - t0:.0f}s -> {counts}")
|
|
if args.chat_verify:
|
|
u = [o for o in out if o.get("usable") is True]
|
|
nu = [o for o in out if o.get("usable") is False]
|
|
unk = [o for o in out if o.get("usable") is None and o["probe_status"] == "WORKING"]
|
|
print(f"tier-2: usable={len(u)} NOT-usable={len(nu)} unknown={len(unk)}")
|
|
reasons: dict[str, int] = {}
|
|
for o in nu:
|
|
reasons[(o.get("usable_info") or "")[:34]] = reasons.get((o.get("usable_info") or "")[:34], 0) + 1
|
|
print(" not-usable reasons:", dict(sorted(reasons.items(), key=lambda x: -x[1])[:8]))
|
|
byprov: dict[str, int] = {}
|
|
for o in u:
|
|
byprov[o["db_prov"]] = byprov.get(o["db_prov"], 0) + 1
|
|
print(" usable by provider:", byprov)
|
|
print(f"revived (DB said not-WORKING, probe says WORKING): {len(revived)}")
|
|
for o in revived[:8]:
|
|
print(f" {o['masked']} db={o['db_status']} -> {o['info']}")
|
|
print(f"stale WORKING (probe disagrees): {len(false_working)}")
|
|
live = [o for o in out if o["probe_status"] == "WORKING"]
|
|
print(f"\nLIVE now: {len(live)}")
|
|
for o in live[:10]:
|
|
print(f" {o['masked']} db={o['db_status']} {o['info']} "
|
|
f"usable={o.get('usable')} ({o['secs']}s)")
|
|
|
|
dest = args.out or os.path.join(
|
|
os.path.dirname(HERE), "ds_multiregion_audit_" + time.strftime("%Y%m%d_%H%M%S") + ".json")
|
|
with open(dest, "w", encoding="utf-8") as fh:
|
|
json.dump({"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"probe": "tier1 multi-region /v1/models (zero token) + tier2 provider-specific",
|
|
"providers": list(providers), "statuses": list(statuses),
|
|
"counts": counts, "revived": len(revived),
|
|
"stale_working": len(false_working), "results": out},
|
|
fh, ensure_ascii=False, indent=1)
|
|
print("report:", dest)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|