vault ключей + мост abuse-radar + китайские источники: всё в одном реестре

- omni/vault.py: пул ключей парсеров (env-first, data/vault.json, ротация, cooldown 401/402/403=1ч, 429=10м), только маски в выводе
- tools/build_vault.py: сборка пула из MASTER_KEYS.csv и локальных дампов (значения не в git)
- tools/vault_check.py: живая проверка с маскированием
- searchers: firecrawl/tavily/exa/serper/jina/apify берут ключ из vault, если нет env
- omni radar: домены+каналы из radar_server.db -> реестр, --import-posts заливает посты каналов
- radar-источники помечены opt_in: общий обход не тормозят (869 против 3337 всего)
- omni/sources_cn.py + tools/wave_cn.py: 9 живых китайских источников (52pojie, nodeseek, kanxue, anquanke, 4hou, xz.aliyun, ithome, cnbeta, v2ex)
- net.get(proxy=...): принудительный прокси для CF-площадок
- /api/keys + строка ключей в панели
- tests/test_vault_radar.py: 7 тестов, всего 29 passed

omni/cli.py

@@ -298,6 +298,122 @@ def cmd_pump(a):
     return 0
 
 
+def cmd_keys(a):
+    """Сводка vault: сколько ключей на сервис, сколько готово, что остывает."""
+    from . import vault
+    st = vault.stats()
+    if a.service:
+        st = [x for x in st if x["service"] == a.service]
+    if not st:
+        print("vault пуст — собери: python tools/build_vault.py")
+        return 1
+    w = max(len(x["service"]) for x in st)
+    print("%-*s %8s %8s %9s  %s" % (w, "сервис", "ключей", "готово", "остывают", "env"))
+    for x in st:
+        print("%-*s %8d %8d %9d  %s" % (w, x["service"], x["keys"], x["ready"],
+                                        x["cooling"], "*" if x["env"] else ""))
+    if a.verbose:
+        print()
+        for x in st:
+            for r in (vault._load()[0].get(x["service"]) or []):
+                print("  %-*s %s %-7s %s" % (w, x["service"], vault.mask(r["key"]),
+                                             r.get("status") or "?", (r.get("note") or "")[:30]))
+    return 0
+
+
+def cmd_radar(a):
+    """Мост из abuse-radar: домены и Telegram-каналы радара -> реестр omni."""
+    import json as _json
+    import sqlite3 as _sq
+    from pathlib import Path as _P
+    from . import sources as _src
+
+    cand = [a.db] if a.db else [
+        str(_P("C:/Users/User/tmp/abuse_radar/radar_server.db")),
+        str(_P("C:/Users/User/tmp/abuse_radar/radar.db")),
+        "data/radar.db",
+    ]
+    db = next((c for c in cand if _P(c).exists()), "")
+    if not db:
+        print("radar.db не найден, укажи --db")
+        return 2
+
+    c = _sq.connect(db)
+    doms = c.execute("SELECT domain, score, info FROM domains ORDER BY score DESC").fetchall()
+    chans = c.execute("SELECT name, ok, fails FROM channels WHERE ok=1").fetchall()
+    posts = c.execute("SELECT chan, mid, date, text FROM messages").fetchall() if a.import_posts else []
+    c.close()
+
+    if a.import_posts:
+        def head_line(txt):
+            for ln in (txt or "").splitlines():
+                ln = ln.strip()
+                if len(ln) >= 20 and sum(ch.isalpha() for ch in ln) >= 10:
+                    return ln[:150]
+            return ((txt or "").strip().splitlines() or [""])[0][:150]
+
+        items, seen_u = [], set()
+        for chan, mid, date, text in posts:
+            text = (text or "").strip()
+            if len(text) < 30:
+                continue
+            url_u = "https://t.me/%s/%s" % (chan, mid)
+            if url_u in seen_u:
+                continue
+            seen_u.add(url_u)
+            items.append(Item("rad_" + chan, "post", url=url_u, title=head_line(text), text=text,
+                              author="@" + chan, published=(date or "")[:19],
+                              source="rad_" + chan, extra={"via": "radar"}))
+        if a.limit:
+            items = items[:a.limit]
+        n_new = n_dup = 0
+        for i in range(0, len(items), 500):
+            n1, n2 = store.add(items[i:i + 500])
+            n_new += n1
+            n_dup += n2
+        print("постов радара: %d | новых %d, дублей %d" % (len(items), n_new, n_dup))
+
+    have = {s.url.rstrip('/').lower() for s in _src.all_sources()}
+    have_names = {s.name for s in _src.all_sources()}
+
+    new_dom, new_chan = [], []
+    for dom, score, info in doms:
+        if a.min_score and (score or 0) < a.min_score:
+            continue
+        url = "https://" + dom
+        if url.lower() in have or ("rad_" + dom.replace('.', '_').replace('-', '_')) in have_names:
+            continue
+        title = ""
+        try:
+            title = (_json.loads(info or "{}") or {}).get("title") or ""
+        except Exception:
+            pass
+        new_dom.append((dom, score, title))
+    for name, ok, fails in chans:
+        if ("tg_" + name) in have_names:
+            continue
+        new_chan.append(name)
+
+    if a.limit:
+        new_dom, new_chan = new_dom[:a.limit], new_chan[:a.limit]
+    print("радар: %s" % db)
+    print("доменов всего %d, новых для omni %d | каналов живых %d, новых %d"
+          % (len(doms), len(new_dom), len(chans), len(new_chan)))
+    for dom, score, title in new_dom[:5]:
+        print("  %-28s score %-4s %s" % (dom, score, (title or "")[:44]))
+    if new_chan[:5]:
+        print("  каналы:", ", ".join(new_chan[:8]))
+    if not a.apply:
+        print("dry-run: добавь --apply, чтобы записать в data/radar_sources.json")
+        return 0
+
+    out = _P("data") / "radar_sources.json"
+    out.write_text(_json.dumps({"domains": new_dom, "channels": new_chan},
+                               ensure_ascii=False, indent=1), encoding="utf-8")
+    print("записано: %s (доменов %d, каналов %d)" % (out, len(new_dom), len(new_chan)))
+    return 0
+
+
 def main(argv=None):
     try:
         sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -405,6 +521,19 @@ def main(argv=None):
     sp.add_argument("--apply", action="store_true")
     sp.set_defaults(fn=cmd_pump)
 
+    sp = sub.add_parser("keys", help="ключи парсеров из vault: сервисы, счётчики, остывание")
+    sp.add_argument("--service", default="", help="только один сервис")
+    sp.add_argument("-v", "--verbose", action="store_true", help="по ключам, масками")
+    sp.set_defaults(fn=cmd_keys)
+
+    sp = sub.add_parser("radar", help="забрать источники из abuse-radar (домены + каналы)")
+    sp.add_argument("--db", default="", help="путь к radar.db (по умолчанию ищет рядом)")
+    sp.add_argument("--apply", action="store_true", help="записать в реестр omni")
+    sp.add_argument("--min-score", type=int, default=20, help="минимальный score домена")
+    sp.add_argument("--import-posts", action="store_true", help="залить посты каналов радара в базу")
+    sp.add_argument("--limit", type=int, default=0)
+    sp.set_defaults(fn=cmd_radar)
+
     args = p.parse_args(argv)
     return args.fn(args)
 

omni/net.py

@@ -117,8 +117,11 @@ def post_json(url, payload, headers=None, timeout=None):
         return Resp(url, 0, "", "curl-post", f"{type(e).__name__}: {e}"[:160])
 
 
-def get(url, headers=None, timeout=None, json_mode=False):
-    """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ."""
+def get(url, headers=None, timeout=None, json_mode=False, proxy=None):
+    """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.
+
+    proxy=... — принудительный прокси для конкретного источника (напр. CF-форумы через DE).
+    """
     timeout = timeout or _cfg.get("timeout", 25)
     host = urllib.parse.urlsplit(url).netloc
     _throttle(host)
@@ -132,6 +135,13 @@ def get(url, headers=None, timeout=None, json_mode=False):
     if ck:
         h["Cookie"] = ck
 
+    if proxy:
+        forced = _try_curl(url, h, timeout, proxy)
+        if forced.ok and not forced.blocked:
+            return forced
+        alt = _try_httpx(url, h, timeout, proxy)
+        return alt if alt.status == 200 else forced
+
     first = _try_httpx(url, h, timeout, None)
     if first.ok and not first.blocked:
         return first

omni/providers/searchers.py

@@ -14,8 +14,18 @@ def _mk(provider, title, url, text="", score=0.0, published="", extra=None):
                 published=published, extra=extra or {})
 
 
+ENV2SVC = {"FIRECRAWL_API_KEY": "firecrawl", "TAVILY_API_KEY": "tavily", "EXA_API_KEY": "exa",
+           "SERPER_API_KEY": "serper", "JINA_API_KEY": "jina", "APIFY_TOKEN": "apify"}
+
+
 def _key(name):
-    return config.KEYS.get(name)
+    """env первым, иначе ключ из vault (data/vault.json), иначе пусто."""
+    v = config.KEYS.get(name)
+    if v:
+        return v
+    from .. import vault
+    svc = ENV2SVC.get(name)
+    return vault.get(svc) if svc else ""
 
 
 def available(name):

omni/serve.py

@@ -236,6 +236,15 @@ class Handler(BaseHTTPRequestHandler):
             items = [{"key": k, "label": taxonomy.LABELS.get(k, k), "n": v} for k, v in cnt.items()]
             return self._json({"ok": True, "types": items, "total": sum(cnt.values())})
 
+        if path == "/api/keys":
+            from . import vault as _v
+            rows = _v.stats()
+            if q.get("service"):
+                rows = [x for x in rows if x["service"] == q["service"][0]]
+            return self._json({"ok": True, "services": rows,
+                               "total_keys": sum(x["keys"] for x in rows),
+                               "total_ready": sum(x["ready"] for x in rows)})
+
         if path == "/api/runs":
             try:
                 limit = _intv(q, "limit", 80, 1, 500)

omni/sources.py

@@ -207,6 +207,58 @@ SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for
 BY_NAME = {s.name: s for s in SOURCES}
 CATEGORIES = sorted({s.category for s in SOURCES})
 
+_RADAR = {"at": 0.0, "rows": []}
+
+
+def radar_sources():
+    """Источники из abuse-radar: data/radar_sources.json (см. omni radar --apply).
+
+    Домены идут html_listing (root страница с маркерами), каналы — tg_web.
+    Файл перечитывается при изменении mtime, реестр не разрастается дублями.
+    """
+    import json
+    from pathlib import Path
+    p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
+    try:
+        mtime = p.stat().st_mtime
+    except OSError:
+        return []
+    if _RADAR["rows"] and _RADAR["at"] == mtime:
+        return _RADAR["rows"]
+    rows = []
+    try:
+        d = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return []
+    for item in d.get("domains") or []:
+        dom, score, title = (list(item) + ["", 0, ""])[:3] if isinstance(item, list) else (item.get("domain"), item.get("score"), item.get("title"))
+        name = "rad_" + str(dom).replace(".", "_").replace("-", "_")
+        rows.append(Source(name=name, category="abuse", kind="site", url="https://" + str(dom),
+                           adapter="html_listing", tags=("radar", "abuse"),
+                           limit=12, detail=False, extra={"score": score, "title": (title or "")[:120], "opt_in": True}))
+    for ch in d.get("channels") or []:
+        name = "tg_" + str(ch).lstrip("@")
+        rows.append(Source(name=name, category="abuse", kind="tg", url="https://t.me/" + str(ch).lstrip("@"),
+                           adapter="tg_web", tags=("tg", "abuse", "radar"), limit=25,
+                           detail=False, extra={"opt_in": True}))
+    _RADAR.update(at=mtime, rows=rows)
+    return rows
+
+
+def _cn_sources():
+    """Китайские/азиатские источники из сгенерированного omni/sources_cn.py."""
+    try:
+        from .sources_cn import CN
+    except Exception:
+        return []
+    out = []
+    for d in CN:
+        try:
+            out.append(Source(**d))
+        except Exception:
+            continue
+    return out
+
 
 def all_sources():
     """Реестр + пользовательские источники из панели, без дублей по url."""
@@ -219,7 +271,7 @@ def all_sources():
              ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
         extra.append(Source(**d))
     seen, out = set(), []
-    for s in SOURCES + extra:
+    for s in SOURCES + _cn_sources() + extra + radar_sources():
         key = (s.url or "").rstrip("/").lower()
         if key and key in seen:
             continue
@@ -227,15 +279,27 @@ def all_sources():
         out.append(s)
     return out
 
-def enabled_sources():
-    """Только включённые: выключенные в панели источники не дёргаем."""
+def enabled_sources(include_opt_in=False):
+    """Только включённые: выключенные в панели источники не дёргаем.
+
+    Источники из abuse-radar помечены extra["opt_in"] — их 2000+, в общий обход
+    они не лезут; берутся только когда явно просят (--tags radar / --only / category abuse).
+    """
     from . import source_state
     fl = source_state.flags()
-    return [s for s in all_sources() if fl.get(s.name, {}).get("enabled", True)]
+    out = []
+    for s in all_sources():
+        if not fl.get(s.name, {}).get("enabled", True):
+            continue
+        if not include_opt_in and (s.extra or {}).get("opt_in"):
+            continue
+        out.append(s)
+    return out
 
 
 def pick(category=None, only=None, tags=None, respect_enabled=True):
-    out = enabled_sources() if respect_enabled else all_sources()
+    explicit = bool(only or tags or category)
+    out = enabled_sources(include_opt_in=explicit) if respect_enabled else all_sources()
     if category:
         cats = {c.strip() for c in category.split(",") if c.strip()}
         out = [s for s in out if s.category in cats]

omni/sources_cn.py

@@ -0,0 +1,17 @@
+"""Китайские и азиатские источники (проверены живыми запросами tools/wave_cn.py).
+
+Форумы реверса/хостинга, security-новости, tech-ленты. Все — публичные списки,
+без ключей и кук. Подключаются в реестр omni как категории forum/security/news.
+"""
+
+CN = [
+    {"name": "cn_52pojie", "category": "forum", "kind": "site", "url": "https://www.52pojie.cn/forum.php?mod=guide&view=new", "adapter": "html_listing", "tags": ("cn", "reverse", "abuse"), "limit": 20},
+    {"name": "cn_nodeseek", "category": "forum", "kind": "site", "url": "https://www.nodeseek.com/", "adapter": "html_listing", "tags": ("cn", "hosting"), "limit": 20},
+    {"name": "cn_anquanke", "category": "security", "kind": "site", "url": "https://www.anquanke.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 20},
+    {"name": "cn_4hou", "category": "security", "kind": "site", "url": "https://www.4hou.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 15},
+    {"name": "cn_kanxue", "category": "forum", "kind": "site", "url": "https://bbs.kanxue.com/", "adapter": "html_listing", "tags": ("cn", "reverse"), "limit": 20},
+    {"name": "cn_ithome", "category": "news", "kind": "site", "url": "https://www.ithome.com/rss/", "adapter": "rss", "tags": ("cn", "tech"), "limit": 20},
+    {"name": "cn_cnbeta", "category": "news", "kind": "site", "url": "https://www.cnbeta.com.tw/backend.php", "adapter": "rss", "tags": ("cn", "tech"), "limit": 20},
+    {"name": "cn_v2ex_rss", "category": "forum", "kind": "site", "url": "https://www.v2ex.com/index.xml", "adapter": "rss", "tags": ("cn", "community"), "limit": 20},
+    {"name": "cn_baidu_sec", "category": "security", "kind": "site", "url": "https://xz.aliyun.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 15},
+]

omni/vault.py

@@ -0,0 +1,128 @@
+"""Vault: ключи сервисов с ротацией и остыванием.
+
+Источник — data/vault.json (собирается tools/build_vault.py из дампов ключей).
+Значения живут только на диске и в памяти: в логах и панели печатается маска.
+
+Правила:
+  * env важнее файла (FIRECRAWL_API_KEY и т.п.) — ручное управление не ломается;
+  * ключ, получивший 401/402/403, уходит в остывание на час; 429 — на 10 минут;
+  * round-robin, чтобы не долбить один ключ и не выжигать квоту.
+
+Использование:
+  from . import vault
+  key = vault.get("firecrawl")           # или "" если ничего нет
+  vault.mark("firecrawl", key, "dead")   # 401/402/403
+  vault.mark("firecrawl", key, "rate")   # 429
+"""
+import json
+import os
+import random
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+VAULT = ROOT / "data" / "vault.json"
+STATE = ROOT / "data" / "vault_state.json"
+
+ENV = {
+    "firecrawl": "FIRECRAWL_API_KEY",
+    "tavily": "TAVILY_API_KEY",
+    "exa": "EXA_API_KEY",
+    "serper": "SERPER_API_KEY",
+    "jina": "JINA_API_KEY",
+    "apify": "APIFY_TOKEN",
+    "2captcha": "TWOCAPTCHA_KEY",
+    "capmonster": "CAPMONSTER_KEY",
+    "telegram": "TG_BOT_TOKEN",
+}
+
+COOLDOWN = {"dead": 3600, "rate": 600, "empty": 1800}
+
+_cache = {"data": None, "state": None, "at": 0.0}
+
+
+def _load():
+    now = time.time()
+    if _cache["data"] is not None and now - _cache["at"] < 30:
+        return _cache["data"], _cache["state"]
+    data, state = {}, {}
+    try:
+        data = json.loads(VAULT.read_text(encoding="utf-8"))
+    except Exception:
+        data = {}
+    try:
+        state = json.loads(STATE.read_text(encoding="utf-8"))
+    except Exception:
+        state = {}
+    _cache.update(data=data, state=state, at=now)
+    return data, state
+
+
+def _save_state(state):
+    try:
+        STATE.parent.mkdir(parents=True, exist_ok=True)
+        STATE.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
+    except Exception:
+        pass
+    _cache["state"] = state
+
+
+def services():
+    data, _ = _load()
+    return sorted(data)
+
+
+def keys(service):
+    data, _ = _load()
+    return [r["key"] for r in (data.get(service) or []) if r.get("key")]
+
+
+def mask(k):
+    k = k or ""
+    return (k[:4] + "…" + k[-2:]) if len(k) > 8 else "…"
+
+
+def get(service, offset=0):
+    """Живой ключ сервиса: env, иначе ротация по пулу с учётом остывания."""
+    env_name = ENV.get(service)
+    if env_name and os.environ.get(env_name):
+        return os.environ[env_name].strip()
+    pool = keys(service)
+    if not pool:
+        return ""
+    _, state = _load()
+    st = state.get(service, {})
+    now = time.time()
+    avail = []
+    for k in pool:
+        until = st.get(k, 0)
+        if until <= now:
+            avail.append(k)
+    if not avail:
+        # всё остывает — берём тот, у кого срок истекает раньше
+        k = min(pool, key=lambda x: st.get(x, 0))
+        return k
+    return avail[(offset + int(now)) % len(avail)] if len(avail) > 1 else avail[0]
+
+
+def mark(service, key, kind="dead"):
+    if not key:
+        return
+    _, state = _load()
+    st = state.setdefault(service, {})
+    st[key] = time.time() + COOLDOWN.get(kind, 600)
+    _save_state(state)
+
+
+def stats():
+    data, state = _load()
+    out = []
+    now = time.time()
+    for svc in sorted(data):
+        rows = data[svc]
+        st = state.get(svc, {})
+        cooling = sum(1 for r in rows if st.get(r["key"], 0) > now)
+        out.append({"service": svc, "keys": len(rows), "cooling": cooling,
+                    "ready": len(rows) - cooling,
+                    "env": bool(ENV.get(svc) and os.environ.get(ENV[svc]))})
+    return out

tests/test_vault_radar.py

@@ -0,0 +1,61 @@
+"""Тесты на «одно место»: vault ключей, opt-in радара, китайские источники."""
+import json
+
+from omni import sources, vault
+
+
+def test_vault_env_first(monkeypatch):
+    monkeypatch.setenv("FIRECRAWL_API_KEY", "fc-env-key")
+    assert vault.get("firecrawl") == "fc-env-key"
+
+
+def test_vault_mask_never_shows_value():
+    m = vault.mask("fc-1234567890abcdef")
+    assert "1234567890" not in m
+    assert m.endswith("ef")
+    assert vault.mask("short") == "…"
+
+
+def test_vault_roundrobin_and_cooldown(monkeypatch):
+    data = {"zzsvc": [{"key": "k1"}, {"key": "k2"}]}
+    monkeypatch.setattr(vault, "_load", lambda: (data, {}))
+    monkeypatch.setattr(vault, "ENV", {})
+    got = {vault.get("zzsvc", offset=i) for i in range(6)}
+    assert got == {"k1", "k2"}, "пул должен ротироваться по обоим ключам"
+    monkeypatch.setattr(vault, "_load", lambda: (data, {"zzsvc": {"k1": 9e12, "k2": 9e12}}))
+    assert vault.get("zzsvc") in {"k1", "k2"}, "при остывании всё равно отдаём ключ"
+
+
+def test_radar_sources_opt_in():
+    r = sources.radar_sources()
+    if not r:
+        return
+    assert all((s.extra or {}).get("opt_in") for s in r)
+    plain = sources.enabled_sources()
+    assert not any((s.extra or {}).get("opt_in") for s in plain), "радар не должен лезть в общий обход"
+    assert sources.pick(tags="radar"), "по тегу radar источники должны находиться"
+
+
+def test_cn_sources_registered():
+    cn = [s for s in sources.all_sources() if s.name.startswith("cn_")]
+    assert cn, "китайские источники должны быть в реестре"
+    assert all(s.url.startswith("http") for s in cn)
+    assert all(s.adapter in ("html_listing", "rss") for s in cn)
+
+
+def test_all_sources_no_dup_url():
+    seen = {}
+    for s in sources.all_sources():
+        u = (s.url or "").rstrip("/").lower()
+        assert u not in seen, "дубль url: %s" % s.name
+        seen[u] = s.name
+
+
+def test_radar_json_shape():
+    from pathlib import Path
+    p = Path("data/radar_sources.json")
+    if not p.exists():
+        return
+    d = json.loads(p.read_text(encoding="utf-8"))
+    assert isinstance(d.get("domains"), list) and isinstance(d.get("channels"), list)
+    assert all(len(x) >= 2 for x in d["domains"])

tools/build_vault.py

@@ -0,0 +1,139 @@
+"""Единый инвентарь ключей для парсеров: MASTER_KEYS.csv -> data/vault.json.
+
+Собирает всё, что лежит по диску (MASTER_KEYS.csv, пулы keyhunter, fl_bot), и
+раскладывает по сервисам. Файл data/vault.json в .gitignore — ключи не уезжают в репо.
+
+Запуск:
+  python tools/build_vault.py            # собрать
+  python tools/build_vault.py --stats    # только показать, что получилось
+"""
+import argparse
+import csv
+import json
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "data" / "vault.json"
+
+SOURCES = [
+    Path("C:/Users/User/tmp/keys_inventory/MASTER_KEYS.csv"),
+    Path("C:/Users/User/Desktop/_DOCS/ALL_WORKING_KEYS_WITH_BALANCE.txt"),
+]
+
+# service в MASTER_KEYS -> имя сервиса в vault
+ALIAS = {
+    "firecrawl.dev": "firecrawl",
+    "tavily": "tavily",
+    "exa": "exa",
+    "serper": "serper",
+    "jina": "jina",
+    "apify": "apify",
+    "scrapingant": "scrapingant",
+    "brightdata": "brightdata",
+    "scraperapi": "scraperapi",
+    "zenrows": "zenrows",
+    "twocaptcha": "2captcha",
+    "2captcha": "2captcha",
+    "capmonster": "capmonster",
+    "telegram": "telegram",
+    "proxy": "proxy",
+}
+
+KEY_RE = re.compile(r"^[A-Za-z0-9_\-\.:]{16,200}$")
+
+
+def norm_service(s):
+    s = (s or "").strip().lower()
+    for k, v in ALIAS.items():
+        if s == k or s.startswith(k):
+            return v
+    return s
+
+
+def good_key(k):
+    k = (k or "").strip().strip('"').strip("'")
+    if not k or " " in k or len(k) < 16:
+        return ""
+    return k if KEY_RE.match(k) else ""
+
+
+def from_csv(path, bag):
+    for r in csv.DictReader(path.open(encoding="utf-8", errors="ignore")):
+        svc = norm_service(r.get("service"))
+        key = good_key(r.get("api_key"))
+        if svc and key:
+            bag[svc].append({"key": key, "status": (r.get("status") or "").strip(),
+                             "quota": (r.get("quota_balance") or "").strip(),
+                             "src": Path(r.get("source_file") or path.name).name})
+    return True
+
+
+def from_txt(path, bag):
+    """Строки вида TAG|key или TAG:key или key — берём только пары с тегом."""
+    pat = re.compile(r"^\s*([A-Za-z0-9_\-]{2,20})\s*[|:]\s*([A-Za-z0-9_\-\.]{16,200})\s*$")
+    for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
+        m = pat.match(line)
+        if m:
+            svc = norm_service(m.group(1))
+            bag[svc].append({"key": m.group(2), "status": "unknown", "quota": "", "src": path.name})
+    return True
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--stats", action="store_true", help="только показать текущий vault")
+    a = ap.parse_args()
+
+    if a.stats:
+        if not OUT.exists():
+            print("vault.json нет — запусти без --stats")
+            return 1
+        d = json.loads(OUT.read_text(encoding="utf-8"))
+        total = sum(len(v) for v in d.values())
+        print("сервисов %d, ключей %d" % (len(d), total))
+        for s, rows in sorted(d.items(), key=lambda x: -len(x[1]))[:25]:
+            alive = sum(1 for r in rows if r.get("status") in ("live", "alive", "ok", "live-verified-2026-10-24"))
+            print("  %-22s %5d %s" % (s, len(rows), ("(живых %d)" % alive) if alive else ""))
+        return 0
+
+    bag = defaultdict(list)
+    used = 0
+    for p in SOURCES:
+        if not p.exists():
+            continue
+        try:
+            if p.suffix == ".csv":
+                from_csv(p, bag)
+            else:
+                from_txt(p, bag)
+            used += 1
+        except Exception as e:
+            print("skip %s: %s" % (p.name, type(e).__name__))
+
+    # дедуп по значению ключа внутри сервиса
+    out = {}
+    for svc, rows in bag.items():
+        seen, clean = set(), []
+        for r in rows:
+            if r["key"] in seen:
+                continue
+            seen.add(r["key"])
+            clean.append(r)
+        out[svc] = clean
+
+    OUT.parent.mkdir(parents=True, exist_ok=True)
+    OUT.write_text(json.dumps(out, ensure_ascii=False, indent=0), encoding="utf-8")
+    total = sum(len(v) for v in out.values())
+    print("файлов прочитано %d | сервисов %d | ключей %d | записано в %s"
+          % (used, len(out), total, OUT))
+    for s in ("firecrawl", "tavily", "exa", "serper", "jina", "apify", "2captcha", "telegram", "proxy"):
+        if s in out:
+            print("  %-12s %d" % (s, len(out[s])))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/vault_check.py

@@ -0,0 +1,112 @@
+"""Проверка ключей vault живыми запросами. Значения не печатаются — только маски.
+
+  python tools/vault_check.py firecrawl --limit 20
+  python tools/vault_check.py 2captcha
+  python tools/vault_check.py all
+
+Статусы в data/vault.json: live / dead / rate / nofund.
+firecrawl: GET /v2/team/credit-usage (200 = живой, 402 = без кредитов, 401 = мёртвый).
+2captcha:  GET res.php?action=getbalance (status:1 = живой, баланс в request).
+"""
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+import requests
+
+ROOT = Path(__file__).resolve().parents[1]
+VAULT = ROOT / "data" / "vault.json"
+
+
+def load():
+    return json.loads(VAULT.read_text(encoding="utf-8"))
+
+
+def save(d):
+    VAULT.write_text(json.dumps(d, ensure_ascii=False, indent=0), encoding="utf-8")
+
+
+def mask(k):
+    return k[:4] + "…" + k[-2:] if len(k) > 8 else "…"
+
+
+def check_firecrawl(key):
+    for url in ("https://api.firecrawl.dev/v2/team/credit-usage",
+                "https://api.firecrawl.dev/v1/credits"):
+        try:
+            r = requests.get(url, headers={"Authorization": "Bearer " + key}, timeout=20)
+        except Exception as e:
+            return "err", type(e).__name__
+        if r.status_code == 200:
+            try:
+                j = r.json()
+            except Exception:
+                j = {}
+            rem = j.get("remainingCredits") or j.get("remaining_credits") or \
+                  (j.get("data") or {}).get("remainingCredits")
+            return "live", ("credits %s" % rem) if rem is not None else "ok"
+        if r.status_code == 401:
+            return "dead", "401"
+        if r.status_code == 402:
+            return "nofund", "402"
+        if r.status_code == 429:
+            return "rate", "429"
+        if r.status_code == 403:
+            return "dead", "403"
+    return "dead", "no-endpoint"
+
+
+def check_2captcha(key):
+    try:
+        r = requests.get("https://2captcha.com/res.php",
+                         params={"key": key, "action": "getbalance", "json": 1}, timeout=20)
+        j = r.json()
+    except Exception as e:
+        return "err", type(e).__name__
+    if j.get("status") == 1:
+        return "live", "баланс %s" % j.get("request")
+    err = str(j.get("request") or "")
+    if "ERROR_WRONG_USER_KEY" in err or "ERROR_KEY_DOES_NOT_EXIST" in err:
+        return "dead", err[:30]
+    return "rate", err[:30]
+
+
+CHECKS = {"firecrawl": check_firecrawl, "2captcha": check_2captcha}
+
+
+def run(service, limit):
+    d = load()
+    rows = d.get(service) or []
+    if not rows:
+        print("в vault нет сервиса", service)
+        return
+    fn = CHECKS[service]
+    rows = rows[:limit] if limit else rows
+    print("%s: проверяю %d ключей" % (service, len(rows)))
+    stat = {}
+    for r in rows:
+        st, note = fn(r["key"])
+        r["status"] = st
+        r["note"] = note
+        stat[st] = stat.get(st, 0) + 1
+        mark = {"live": "+", "dead": "-", "nofund": "$", "rate": "~", "err": "!"}.get(st, "?")
+        print("  %s %s %-7s %s" % (mark, mask(r["key"]), st, note))
+        time.sleep(0.35)
+    save(d)
+    print("итог %s: %s" % (service, stat))
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("service")
+    ap.add_argument("--limit", type=int, default=0)
+    a = ap.parse_args()
+    if a.service == "all":
+        for s in CHECKS:
+            run(s, a.limit)
+    elif a.service in CHECKS:
+        run(a.service, a.limit)
+    else:
+        print("поддерживается:", ", ".join(CHECKS))

tools/wave_cn.py

@@ -0,0 +1,86 @@
+"""Волна по китайским/азиатским источникам: форумы, реверс, security-новости.
+
+Проверяет кандидатов живыми запросами (лента / список тем), пишет живые в
+omni/sources_cn.py. Запуск:  python tools/wave_cn.py [--apply]
+"""
+import sys
+import time
+from pathlib import Path
+
+import requests
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "omni" / "sources_cn.py"
+
+UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
+
+# name, url, adapter, category, tags, limit
+CAND = [
+    ("cn_52pojie", "https://www.52pojie.cn/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "reverse", "abuse"), 20),
+    ("cn_nodeseek", "https://www.nodeseek.com/", "html_listing", "forum", ("cn", "hosting"), 20),
+    ("cn_freebuf", "https://www.freebuf.com/", "html_listing", "security", ("cn", "security"), 20),
+    ("cn_seebug", "https://paper.seebug.org/", "html_listing", "security", ("cn", "papers", "security"), 15),
+    ("cn_anquanke", "https://www.anquanke.com/", "html_listing", "security", ("cn", "security"), 20),
+    ("cn_4hou", "https://www.4hou.com/", "html_listing", "security", ("cn", "security"), 15),
+    ("cn_kanxue", "https://bbs.kanxue.com/", "html_listing", "forum", ("cn", "reverse"), 20),
+    ("cn_ithome", "https://www.ithome.com/rss/", "rss", "news", ("cn", "tech"), 20),
+    ("cn_cnbeta", "https://www.cnbeta.com.tw/backend.php", "rss", "news", ("cn", "tech"), 20),
+    ("cn_sspai", "https://sspai.com/feed", "rss", "news", ("cn", "tech"), 15),
+    ("cn_chiphell", "https://www.chiphell.com/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "hardware"), 15),
+    ("cn_52pojie_rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", "forum", ("cn", "reverse"), 15),
+    ("cn_v2ex_rss", "https://www.v2ex.com/index.xml", "rss", "forum", ("cn", "community"), 20),
+    ("cn_linuxdo_rss", "https://linux.do/latest.rss", "rss", "forum", ("cn", "community", "ai"), 20),
+    ("cn_hostloc_rss", "https://hostloc.com/forum.php?mod=rss", "rss", "forum", ("cn", "hosting"), 15),
+    ("cn_baidu_sec", "https://xz.aliyun.com/", "html_listing", "security", ("cn", "security"), 15),
+    ("cn_secfree", "https://www.secfree.com/", "html_listing", "security", ("cn", "security"), 10),
+    ("cn_tttang", "https://tttang.com/", "html_listing", "security", ("cn", "security"), 10),
+]
+
+
+def probe(url, adapter="html_listing"):
+    try:
+        r = requests.get(url, headers={"User-Agent": UA}, timeout=18, allow_redirects=True)
+    except Exception as e:
+        return 0, type(e).__name__, 0
+    if adapter == "rss":
+        n_links = r.text.count("<item") + r.text.count("<entry")
+    else:
+        n_links = r.text.count("<a ")
+    return r.status_code, "", n_links
+
+
+def main():
+    apply = "--apply" in sys.argv
+    live, dead = [], []
+    for name, url, adapter, cat, tags, lim in CAND:
+        code, err, links = probe(url, adapter)
+        ok = code == 200 and links >= 30
+        print("%-18s %-3s links %-5d %s" % (name, code or err, links, "LIVE" if ok else "—"))
+        if ok:
+            live.append((name, url, adapter, cat, tags, lim))
+        else:
+            dead.append(name)
+        time.sleep(0.8)
+
+    print("\nживых %d, мёртвых %d" % (len(live), len(dead)))
+    if not apply:
+        print("dry-run: --apply чтобы записать omni/sources_cn.py")
+        return 0
+
+    body = ['"""Китайские и азиатские источники (проверены живыми запросами tools/wave_cn.py).',
+            '', 'Форумы реверса/хостинга, security-новости, tech-ленты. Все — публичные списки,',
+            'без ключей и кук. Подключаются в реестр omni как категории forum/security/news.',
+            '"""', '', 'CN = [']
+    for name, url, adapter, cat, tags, lim in live:
+        t = ", ".join('"%s"' % x for x in tags)
+        body.append('    {"name": "%s", "category": "%s", "kind": "site", "url": "%s",'
+                    ' "adapter": "%s", "tags": (%s), "limit": %d},' % (name, cat, url, adapter, t, lim))
+    body.append(']')
+    OUT.write_text("\n".join(body) + "\n", encoding="utf-8", newline="\n")
+    print("записано", OUT, "| источников", len(live))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

web/index.html

@@ -524,6 +524,21 @@ async function loadTypes() {
   if (r.ok) buildTypes(r.types);
 }
 
+async function loadKeys() {
+  try {
+    const k = await api('/api/keys');
+    if (!k || k.error) return;
+    const top = (k.services || []).slice(0, 6)
+      .map(x => `${x.service} ${fmt(x.keys)}`).join(' · ');
+    const f = $('#aside-foot');
+    if (f && !f.dataset.keys) {
+      f.dataset.keys = '1';
+      f.innerHTML += `<br><b>${fmt(k.total_keys)}</b> ключей · <b>${(k.services||[]).length}</b> сервисов` +
+        (top ? `<br><span style="opacity:.65">${top}</span>` : '');
+    }
+  } catch (e) {}
+}
+
 async function loadStats() {
   const s = await api('/api/stats');
   if (s.error) return toast(s.error, 'err');
@@ -536,8 +551,10 @@ async function loadStats() {
     `<b>${fmt(s.sources)}</b> источников · <b>${fmt(s.sources_enabled)}</b> вкл<br>` +
     `<b>${fmt(s.alive)}</b> живых · <b>${fmt(s.tested)}</b> проверено<br>` +
     `<b>${fmt(s.runs)}</b> прогонов · <b>${fmt(s.priced)}</b> с ценой`;
+  $('#aside-foot').dataset.keys = '';
   renderCats(s.categories);
   buildChips(s.categories);
+  loadKeys();
   loadTypes();
   const ks = [
     ['записей', s.items, 'в sqlite + fts5', ''],