diff --git a/omni/cli.py b/omni/cli.py index 1b8346f..0f592f2 100644 --- a/omni/cli.py +++ b/omni/cli.py @@ -298,6 +298,122 @@ def cmd_pump(a): return 0 +def cmd_keys(a): + """Сводка vault: сколько ключей на сервис, сколько готово, что остывает.""" + from . import vault + st = vault.stats() + if a.service: + st = [x for x in st if x["service"] == a.service] + if not st: + print("vault пуст — собери: python tools/build_vault.py") + return 1 + w = max(len(x["service"]) for x in st) + print("%-*s %8s %8s %9s %s" % (w, "сервис", "ключей", "готово", "остывают", "env")) + for x in st: + print("%-*s %8d %8d %9d %s" % (w, x["service"], x["keys"], x["ready"], + x["cooling"], "*" if x["env"] else "")) + if a.verbose: + print() + for x in st: + for r in (vault._load()[0].get(x["service"]) or []): + print(" %-*s %s %-7s %s" % (w, x["service"], vault.mask(r["key"]), + r.get("status") or "?", (r.get("note") or "")[:30])) + return 0 + + +def cmd_radar(a): + """Мост из abuse-radar: домены и Telegram-каналы радара -> реестр omni.""" + import json as _json + import sqlite3 as _sq + from pathlib import Path as _P + from . import sources as _src + + cand = [a.db] if a.db else [ + str(_P("C:/Users/User/tmp/abuse_radar/radar_server.db")), + str(_P("C:/Users/User/tmp/abuse_radar/radar.db")), + "data/radar.db", + ] + db = next((c for c in cand if _P(c).exists()), "") + if not db: + print("radar.db не найден, укажи --db") + return 2 + + c = _sq.connect(db) + doms = c.execute("SELECT domain, score, info FROM domains ORDER BY score DESC").fetchall() + chans = c.execute("SELECT name, ok, fails FROM channels WHERE ok=1").fetchall() + posts = c.execute("SELECT chan, mid, date, text FROM messages").fetchall() if a.import_posts else [] + c.close() + + if a.import_posts: + def head_line(txt): + for ln in (txt or "").splitlines(): + ln = ln.strip() + if len(ln) >= 20 and sum(ch.isalpha() for ch in ln) >= 10: + return ln[:150] + return ((txt or "").strip().splitlines() or [""])[0][:150] + + items, seen_u = [], set() + for chan, mid, date, text in posts: + text = (text or "").strip() + if len(text) < 30: + continue + url_u = "https://t.me/%s/%s" % (chan, mid) + if url_u in seen_u: + continue + seen_u.add(url_u) + items.append(Item("rad_" + chan, "post", url=url_u, title=head_line(text), text=text, + author="@" + chan, published=(date or "")[:19], + source="rad_" + chan, extra={"via": "radar"})) + if a.limit: + items = items[:a.limit] + n_new = n_dup = 0 + for i in range(0, len(items), 500): + n1, n2 = store.add(items[i:i + 500]) + n_new += n1 + n_dup += n2 + print("постов радара: %d | новых %d, дублей %d" % (len(items), n_new, n_dup)) + + have = {s.url.rstrip('/').lower() for s in _src.all_sources()} + have_names = {s.name for s in _src.all_sources()} + + new_dom, new_chan = [], [] + for dom, score, info in doms: + if a.min_score and (score or 0) < a.min_score: + continue + url = "https://" + dom + if url.lower() in have or ("rad_" + dom.replace('.', '_').replace('-', '_')) in have_names: + continue + title = "" + try: + title = (_json.loads(info or "{}") or {}).get("title") or "" + except Exception: + pass + new_dom.append((dom, score, title)) + for name, ok, fails in chans: + if ("tg_" + name) in have_names: + continue + new_chan.append(name) + + if a.limit: + new_dom, new_chan = new_dom[:a.limit], new_chan[:a.limit] + print("радар: %s" % db) + print("доменов всего %d, новых для omni %d | каналов живых %d, новых %d" + % (len(doms), len(new_dom), len(chans), len(new_chan))) + for dom, score, title in new_dom[:5]: + print(" %-28s score %-4s %s" % (dom, score, (title or "")[:44])) + if new_chan[:5]: + print(" каналы:", ", ".join(new_chan[:8])) + if not a.apply: + print("dry-run: добавь --apply, чтобы записать в data/radar_sources.json") + return 0 + + out = _P("data") / "radar_sources.json" + out.write_text(_json.dumps({"domains": new_dom, "channels": new_chan}, + ensure_ascii=False, indent=1), encoding="utf-8") + print("записано: %s (доменов %d, каналов %d)" % (out, len(new_dom), len(new_chan))) + return 0 + + def main(argv=None): try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") @@ -405,6 +521,19 @@ def main(argv=None): sp.add_argument("--apply", action="store_true") sp.set_defaults(fn=cmd_pump) + sp = sub.add_parser("keys", help="ключи парсеров из vault: сервисы, счётчики, остывание") + sp.add_argument("--service", default="", help="только один сервис") + sp.add_argument("-v", "--verbose", action="store_true", help="по ключам, масками") + sp.set_defaults(fn=cmd_keys) + + sp = sub.add_parser("radar", help="забрать источники из abuse-radar (домены + каналы)") + sp.add_argument("--db", default="", help="путь к radar.db (по умолчанию ищет рядом)") + sp.add_argument("--apply", action="store_true", help="записать в реестр omni") + sp.add_argument("--min-score", type=int, default=20, help="минимальный score домена") + sp.add_argument("--import-posts", action="store_true", help="залить посты каналов радара в базу") + sp.add_argument("--limit", type=int, default=0) + sp.set_defaults(fn=cmd_radar) + args = p.parse_args(argv) return args.fn(args) diff --git a/omni/net.py b/omni/net.py index 3644e93..4d92cd8 100644 --- a/omni/net.py +++ b/omni/net.py @@ -117,8 +117,11 @@ def post_json(url, payload, headers=None, timeout=None): return Resp(url, 0, "", "curl-post", f"{type(e).__name__}: {e}"[:160]) -def get(url, headers=None, timeout=None, json_mode=False): - """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.""" +def get(url, headers=None, timeout=None, json_mode=False, proxy=None): + """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ. + + proxy=... — принудительный прокси для конкретного источника (напр. CF-форумы через DE). + """ timeout = timeout or _cfg.get("timeout", 25) host = urllib.parse.urlsplit(url).netloc _throttle(host) @@ -132,6 +135,13 @@ def get(url, headers=None, timeout=None, json_mode=False): if ck: h["Cookie"] = ck + if proxy: + forced = _try_curl(url, h, timeout, proxy) + if forced.ok and not forced.blocked: + return forced + alt = _try_httpx(url, h, timeout, proxy) + return alt if alt.status == 200 else forced + first = _try_httpx(url, h, timeout, None) if first.ok and not first.blocked: return first diff --git a/omni/providers/searchers.py b/omni/providers/searchers.py index 64694bb..875bc84 100644 --- a/omni/providers/searchers.py +++ b/omni/providers/searchers.py @@ -14,8 +14,18 @@ def _mk(provider, title, url, text="", score=0.0, published="", extra=None): published=published, extra=extra or {}) +ENV2SVC = {"FIRECRAWL_API_KEY": "firecrawl", "TAVILY_API_KEY": "tavily", "EXA_API_KEY": "exa", + "SERPER_API_KEY": "serper", "JINA_API_KEY": "jina", "APIFY_TOKEN": "apify"} + + def _key(name): - return config.KEYS.get(name) + """env первым, иначе ключ из vault (data/vault.json), иначе пусто.""" + v = config.KEYS.get(name) + if v: + return v + from .. import vault + svc = ENV2SVC.get(name) + return vault.get(svc) if svc else "" def available(name): diff --git a/omni/serve.py b/omni/serve.py index e48a62f..51ffdca 100644 --- a/omni/serve.py +++ b/omni/serve.py @@ -236,6 +236,15 @@ class Handler(BaseHTTPRequestHandler): items = [{"key": k, "label": taxonomy.LABELS.get(k, k), "n": v} for k, v in cnt.items()] return self._json({"ok": True, "types": items, "total": sum(cnt.values())}) + if path == "/api/keys": + from . import vault as _v + rows = _v.stats() + if q.get("service"): + rows = [x for x in rows if x["service"] == q["service"][0]] + return self._json({"ok": True, "services": rows, + "total_keys": sum(x["keys"] for x in rows), + "total_ready": sum(x["ready"] for x in rows)}) + if path == "/api/runs": try: limit = _intv(q, "limit", 80, 1, 500) diff --git a/omni/sources.py b/omni/sources.py index 790167e..f99ef0b 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -207,6 +207,58 @@ SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for BY_NAME = {s.name: s for s in SOURCES} CATEGORIES = sorted({s.category for s in SOURCES}) +_RADAR = {"at": 0.0, "rows": []} + + +def radar_sources(): + """Источники из abuse-radar: data/radar_sources.json (см. omni radar --apply). + + Домены идут html_listing (root страница с маркерами), каналы — tg_web. + Файл перечитывается при изменении mtime, реестр не разрастается дублями. + """ + import json + from pathlib import Path + p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json" + try: + mtime = p.stat().st_mtime + except OSError: + return [] + if _RADAR["rows"] and _RADAR["at"] == mtime: + return _RADAR["rows"] + rows = [] + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + return [] + for item in d.get("domains") or []: + dom, score, title = (list(item) + ["", 0, ""])[:3] if isinstance(item, list) else (item.get("domain"), item.get("score"), item.get("title")) + name = "rad_" + str(dom).replace(".", "_").replace("-", "_") + rows.append(Source(name=name, category="abuse", kind="site", url="https://" + str(dom), + adapter="html_listing", tags=("radar", "abuse"), + limit=12, detail=False, extra={"score": score, "title": (title or "")[:120], "opt_in": True})) + for ch in d.get("channels") or []: + name = "tg_" + str(ch).lstrip("@") + rows.append(Source(name=name, category="abuse", kind="tg", url="https://t.me/" + str(ch).lstrip("@"), + adapter="tg_web", tags=("tg", "abuse", "radar"), limit=25, + detail=False, extra={"opt_in": True})) + _RADAR.update(at=mtime, rows=rows) + return rows + + +def _cn_sources(): + """Китайские/азиатские источники из сгенерированного omni/sources_cn.py.""" + try: + from .sources_cn import CN + except Exception: + return [] + out = [] + for d in CN: + try: + out.append(Source(**d)) + except Exception: + continue + return out + def all_sources(): """Реестр + пользовательские источники из панели, без дублей по url.""" @@ -219,7 +271,7 @@ def all_sources(): ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")} extra.append(Source(**d)) seen, out = set(), [] - for s in SOURCES + extra: + for s in SOURCES + _cn_sources() + extra + radar_sources(): key = (s.url or "").rstrip("/").lower() if key and key in seen: continue @@ -227,15 +279,27 @@ def all_sources(): out.append(s) return out -def enabled_sources(): - """Только включённые: выключенные в панели источники не дёргаем.""" +def enabled_sources(include_opt_in=False): + """Только включённые: выключенные в панели источники не дёргаем. + + Источники из abuse-radar помечены extra["opt_in"] — их 2000+, в общий обход + они не лезут; берутся только когда явно просят (--tags radar / --only / category abuse). + """ from . import source_state fl = source_state.flags() - return [s for s in all_sources() if fl.get(s.name, {}).get("enabled", True)] + out = [] + for s in all_sources(): + if not fl.get(s.name, {}).get("enabled", True): + continue + if not include_opt_in and (s.extra or {}).get("opt_in"): + continue + out.append(s) + return out def pick(category=None, only=None, tags=None, respect_enabled=True): - out = enabled_sources() if respect_enabled else all_sources() + explicit = bool(only or tags or category) + out = enabled_sources(include_opt_in=explicit) if respect_enabled else all_sources() if category: cats = {c.strip() for c in category.split(",") if c.strip()} out = [s for s in out if s.category in cats] diff --git a/omni/sources_cn.py b/omni/sources_cn.py new file mode 100644 index 0000000..75f837f --- /dev/null +++ b/omni/sources_cn.py @@ -0,0 +1,17 @@ +"""Китайские и азиатские источники (проверены живыми запросами tools/wave_cn.py). + +Форумы реверса/хостинга, security-новости, tech-ленты. Все — публичные списки, +без ключей и кук. Подключаются в реестр omni как категории forum/security/news. +""" + +CN = [ + {"name": "cn_52pojie", "category": "forum", "kind": "site", "url": "https://www.52pojie.cn/forum.php?mod=guide&view=new", "adapter": "html_listing", "tags": ("cn", "reverse", "abuse"), "limit": 20}, + {"name": "cn_nodeseek", "category": "forum", "kind": "site", "url": "https://www.nodeseek.com/", "adapter": "html_listing", "tags": ("cn", "hosting"), "limit": 20}, + {"name": "cn_anquanke", "category": "security", "kind": "site", "url": "https://www.anquanke.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 20}, + {"name": "cn_4hou", "category": "security", "kind": "site", "url": "https://www.4hou.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 15}, + {"name": "cn_kanxue", "category": "forum", "kind": "site", "url": "https://bbs.kanxue.com/", "adapter": "html_listing", "tags": ("cn", "reverse"), "limit": 20}, + {"name": "cn_ithome", "category": "news", "kind": "site", "url": "https://www.ithome.com/rss/", "adapter": "rss", "tags": ("cn", "tech"), "limit": 20}, + {"name": "cn_cnbeta", "category": "news", "kind": "site", "url": "https://www.cnbeta.com.tw/backend.php", "adapter": "rss", "tags": ("cn", "tech"), "limit": 20}, + {"name": "cn_v2ex_rss", "category": "forum", "kind": "site", "url": "https://www.v2ex.com/index.xml", "adapter": "rss", "tags": ("cn", "community"), "limit": 20}, + {"name": "cn_baidu_sec", "category": "security", "kind": "site", "url": "https://xz.aliyun.com/", "adapter": "html_listing", "tags": ("cn", "security"), "limit": 15}, +] diff --git a/omni/vault.py b/omni/vault.py new file mode 100644 index 0000000..e50c767 --- /dev/null +++ b/omni/vault.py @@ -0,0 +1,128 @@ +"""Vault: ключи сервисов с ротацией и остыванием. + +Источник — data/vault.json (собирается tools/build_vault.py из дампов ключей). +Значения живут только на диске и в памяти: в логах и панели печатается маска. + +Правила: + * env важнее файла (FIRECRAWL_API_KEY и т.п.) — ручное управление не ломается; + * ключ, получивший 401/402/403, уходит в остывание на час; 429 — на 10 минут; + * round-robin, чтобы не долбить один ключ и не выжигать квоту. + +Использование: + from . import vault + key = vault.get("firecrawl") # или "" если ничего нет + vault.mark("firecrawl", key, "dead") # 401/402/403 + vault.mark("firecrawl", key, "rate") # 429 +""" +import json +import os +import random +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VAULT = ROOT / "data" / "vault.json" +STATE = ROOT / "data" / "vault_state.json" + +ENV = { + "firecrawl": "FIRECRAWL_API_KEY", + "tavily": "TAVILY_API_KEY", + "exa": "EXA_API_KEY", + "serper": "SERPER_API_KEY", + "jina": "JINA_API_KEY", + "apify": "APIFY_TOKEN", + "2captcha": "TWOCAPTCHA_KEY", + "capmonster": "CAPMONSTER_KEY", + "telegram": "TG_BOT_TOKEN", +} + +COOLDOWN = {"dead": 3600, "rate": 600, "empty": 1800} + +_cache = {"data": None, "state": None, "at": 0.0} + + +def _load(): + now = time.time() + if _cache["data"] is not None and now - _cache["at"] < 30: + return _cache["data"], _cache["state"] + data, state = {}, {} + try: + data = json.loads(VAULT.read_text(encoding="utf-8")) + except Exception: + data = {} + try: + state = json.loads(STATE.read_text(encoding="utf-8")) + except Exception: + state = {} + _cache.update(data=data, state=state, at=now) + return data, state + + +def _save_state(state): + try: + STATE.parent.mkdir(parents=True, exist_ok=True) + STATE.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8") + except Exception: + pass + _cache["state"] = state + + +def services(): + data, _ = _load() + return sorted(data) + + +def keys(service): + data, _ = _load() + return [r["key"] for r in (data.get(service) or []) if r.get("key")] + + +def mask(k): + k = k or "" + return (k[:4] + "…" + k[-2:]) if len(k) > 8 else "…" + + +def get(service, offset=0): + """Живой ключ сервиса: env, иначе ротация по пулу с учётом остывания.""" + env_name = ENV.get(service) + if env_name and os.environ.get(env_name): + return os.environ[env_name].strip() + pool = keys(service) + if not pool: + return "" + _, state = _load() + st = state.get(service, {}) + now = time.time() + avail = [] + for k in pool: + until = st.get(k, 0) + if until <= now: + avail.append(k) + if not avail: + # всё остывает — берём тот, у кого срок истекает раньше + k = min(pool, key=lambda x: st.get(x, 0)) + return k + return avail[(offset + int(now)) % len(avail)] if len(avail) > 1 else avail[0] + + +def mark(service, key, kind="dead"): + if not key: + return + _, state = _load() + st = state.setdefault(service, {}) + st[key] = time.time() + COOLDOWN.get(kind, 600) + _save_state(state) + + +def stats(): + data, state = _load() + out = [] + now = time.time() + for svc in sorted(data): + rows = data[svc] + st = state.get(svc, {}) + cooling = sum(1 for r in rows if st.get(r["key"], 0) > now) + out.append({"service": svc, "keys": len(rows), "cooling": cooling, + "ready": len(rows) - cooling, + "env": bool(ENV.get(svc) and os.environ.get(ENV[svc]))}) + return out \ No newline at end of file diff --git a/tests/test_vault_radar.py b/tests/test_vault_radar.py new file mode 100644 index 0000000..369d90b --- /dev/null +++ b/tests/test_vault_radar.py @@ -0,0 +1,61 @@ +"""Тесты на «одно место»: vault ключей, opt-in радара, китайские источники.""" +import json + +from omni import sources, vault + + +def test_vault_env_first(monkeypatch): + monkeypatch.setenv("FIRECRAWL_API_KEY", "fc-env-key") + assert vault.get("firecrawl") == "fc-env-key" + + +def test_vault_mask_never_shows_value(): + m = vault.mask("fc-1234567890abcdef") + assert "1234567890" not in m + assert m.endswith("ef") + assert vault.mask("short") == "…" + + +def test_vault_roundrobin_and_cooldown(monkeypatch): + data = {"zzsvc": [{"key": "k1"}, {"key": "k2"}]} + monkeypatch.setattr(vault, "_load", lambda: (data, {})) + monkeypatch.setattr(vault, "ENV", {}) + got = {vault.get("zzsvc", offset=i) for i in range(6)} + assert got == {"k1", "k2"}, "пул должен ротироваться по обоим ключам" + monkeypatch.setattr(vault, "_load", lambda: (data, {"zzsvc": {"k1": 9e12, "k2": 9e12}})) + assert vault.get("zzsvc") in {"k1", "k2"}, "при остывании всё равно отдаём ключ" + + +def test_radar_sources_opt_in(): + r = sources.radar_sources() + if not r: + return + assert all((s.extra or {}).get("opt_in") for s in r) + plain = sources.enabled_sources() + assert not any((s.extra or {}).get("opt_in") for s in plain), "радар не должен лезть в общий обход" + assert sources.pick(tags="radar"), "по тегу radar источники должны находиться" + + +def test_cn_sources_registered(): + cn = [s for s in sources.all_sources() if s.name.startswith("cn_")] + assert cn, "китайские источники должны быть в реестре" + assert all(s.url.startswith("http") for s in cn) + assert all(s.adapter in ("html_listing", "rss") for s in cn) + + +def test_all_sources_no_dup_url(): + seen = {} + for s in sources.all_sources(): + u = (s.url or "").rstrip("/").lower() + assert u not in seen, "дубль url: %s" % s.name + seen[u] = s.name + + +def test_radar_json_shape(): + from pathlib import Path + p = Path("data/radar_sources.json") + if not p.exists(): + return + d = json.loads(p.read_text(encoding="utf-8")) + assert isinstance(d.get("domains"), list) and isinstance(d.get("channels"), list) + assert all(len(x) >= 2 for x in d["domains"]) \ No newline at end of file diff --git a/tools/build_vault.py b/tools/build_vault.py new file mode 100644 index 0000000..9d935bd --- /dev/null +++ b/tools/build_vault.py @@ -0,0 +1,139 @@ +"""Единый инвентарь ключей для парсеров: MASTER_KEYS.csv -> data/vault.json. + +Собирает всё, что лежит по диску (MASTER_KEYS.csv, пулы keyhunter, fl_bot), и +раскладывает по сервисам. Файл data/vault.json в .gitignore — ключи не уезжают в репо. + +Запуск: + python tools/build_vault.py # собрать + python tools/build_vault.py --stats # только показать, что получилось +""" +import argparse +import csv +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "data" / "vault.json" + +SOURCES = [ + Path("C:/Users/User/tmp/keys_inventory/MASTER_KEYS.csv"), + Path("C:/Users/User/Desktop/_DOCS/ALL_WORKING_KEYS_WITH_BALANCE.txt"), +] + +# service в MASTER_KEYS -> имя сервиса в vault +ALIAS = { + "firecrawl.dev": "firecrawl", + "tavily": "tavily", + "exa": "exa", + "serper": "serper", + "jina": "jina", + "apify": "apify", + "scrapingant": "scrapingant", + "brightdata": "brightdata", + "scraperapi": "scraperapi", + "zenrows": "zenrows", + "twocaptcha": "2captcha", + "2captcha": "2captcha", + "capmonster": "capmonster", + "telegram": "telegram", + "proxy": "proxy", +} + +KEY_RE = re.compile(r"^[A-Za-z0-9_\-\.:]{16,200}$") + + +def norm_service(s): + s = (s or "").strip().lower() + for k, v in ALIAS.items(): + if s == k or s.startswith(k): + return v + return s + + +def good_key(k): + k = (k or "").strip().strip('"').strip("'") + if not k or " " in k or len(k) < 16: + return "" + return k if KEY_RE.match(k) else "" + + +def from_csv(path, bag): + for r in csv.DictReader(path.open(encoding="utf-8", errors="ignore")): + svc = norm_service(r.get("service")) + key = good_key(r.get("api_key")) + if svc and key: + bag[svc].append({"key": key, "status": (r.get("status") or "").strip(), + "quota": (r.get("quota_balance") or "").strip(), + "src": Path(r.get("source_file") or path.name).name}) + return True + + +def from_txt(path, bag): + """Строки вида TAG|key или TAG:key или key — берём только пары с тегом.""" + pat = re.compile(r"^\s*([A-Za-z0-9_\-]{2,20})\s*[|:]\s*([A-Za-z0-9_\-\.]{16,200})\s*$") + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + m = pat.match(line) + if m: + svc = norm_service(m.group(1)) + bag[svc].append({"key": m.group(2), "status": "unknown", "quota": "", "src": path.name}) + return True + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--stats", action="store_true", help="только показать текущий vault") + a = ap.parse_args() + + if a.stats: + if not OUT.exists(): + print("vault.json нет — запусти без --stats") + return 1 + d = json.loads(OUT.read_text(encoding="utf-8")) + total = sum(len(v) for v in d.values()) + print("сервисов %d, ключей %d" % (len(d), total)) + for s, rows in sorted(d.items(), key=lambda x: -len(x[1]))[:25]: + alive = sum(1 for r in rows if r.get("status") in ("live", "alive", "ok", "live-verified-2026-10-24")) + print(" %-22s %5d %s" % (s, len(rows), ("(живых %d)" % alive) if alive else "")) + return 0 + + bag = defaultdict(list) + used = 0 + for p in SOURCES: + if not p.exists(): + continue + try: + if p.suffix == ".csv": + from_csv(p, bag) + else: + from_txt(p, bag) + used += 1 + except Exception as e: + print("skip %s: %s" % (p.name, type(e).__name__)) + + # дедуп по значению ключа внутри сервиса + out = {} + for svc, rows in bag.items(): + seen, clean = set(), [] + for r in rows: + if r["key"] in seen: + continue + seen.add(r["key"]) + clean.append(r) + out[svc] = clean + + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(out, ensure_ascii=False, indent=0), encoding="utf-8") + total = sum(len(v) for v in out.values()) + print("файлов прочитано %d | сервисов %d | ключей %d | записано в %s" + % (used, len(out), total, OUT)) + for s in ("firecrawl", "tavily", "exa", "serper", "jina", "apify", "2captcha", "telegram", "proxy"): + if s in out: + print(" %-12s %d" % (s, len(out[s]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tools/vault_check.py b/tools/vault_check.py new file mode 100644 index 0000000..4abc6ea --- /dev/null +++ b/tools/vault_check.py @@ -0,0 +1,112 @@ +"""Проверка ключей vault живыми запросами. Значения не печатаются — только маски. + + python tools/vault_check.py firecrawl --limit 20 + python tools/vault_check.py 2captcha + python tools/vault_check.py all + +Статусы в data/vault.json: live / dead / rate / nofund. +firecrawl: GET /v2/team/credit-usage (200 = живой, 402 = без кредитов, 401 = мёртвый). +2captcha: GET res.php?action=getbalance (status:1 = живой, баланс в request). +""" +import argparse +import json +import sys +import time +from pathlib import Path + +import requests + +ROOT = Path(__file__).resolve().parents[1] +VAULT = ROOT / "data" / "vault.json" + + +def load(): + return json.loads(VAULT.read_text(encoding="utf-8")) + + +def save(d): + VAULT.write_text(json.dumps(d, ensure_ascii=False, indent=0), encoding="utf-8") + + +def mask(k): + return k[:4] + "…" + k[-2:] if len(k) > 8 else "…" + + +def check_firecrawl(key): + for url in ("https://api.firecrawl.dev/v2/team/credit-usage", + "https://api.firecrawl.dev/v1/credits"): + try: + r = requests.get(url, headers={"Authorization": "Bearer " + key}, timeout=20) + except Exception as e: + return "err", type(e).__name__ + if r.status_code == 200: + try: + j = r.json() + except Exception: + j = {} + rem = j.get("remainingCredits") or j.get("remaining_credits") or \ + (j.get("data") or {}).get("remainingCredits") + return "live", ("credits %s" % rem) if rem is not None else "ok" + if r.status_code == 401: + return "dead", "401" + if r.status_code == 402: + return "nofund", "402" + if r.status_code == 429: + return "rate", "429" + if r.status_code == 403: + return "dead", "403" + return "dead", "no-endpoint" + + +def check_2captcha(key): + try: + r = requests.get("https://2captcha.com/res.php", + params={"key": key, "action": "getbalance", "json": 1}, timeout=20) + j = r.json() + except Exception as e: + return "err", type(e).__name__ + if j.get("status") == 1: + return "live", "баланс %s" % j.get("request") + err = str(j.get("request") or "") + if "ERROR_WRONG_USER_KEY" in err or "ERROR_KEY_DOES_NOT_EXIST" in err: + return "dead", err[:30] + return "rate", err[:30] + + +CHECKS = {"firecrawl": check_firecrawl, "2captcha": check_2captcha} + + +def run(service, limit): + d = load() + rows = d.get(service) or [] + if not rows: + print("в vault нет сервиса", service) + return + fn = CHECKS[service] + rows = rows[:limit] if limit else rows + print("%s: проверяю %d ключей" % (service, len(rows))) + stat = {} + for r in rows: + st, note = fn(r["key"]) + r["status"] = st + r["note"] = note + stat[st] = stat.get(st, 0) + 1 + mark = {"live": "+", "dead": "-", "nofund": "$", "rate": "~", "err": "!"}.get(st, "?") + print(" %s %s %-7s %s" % (mark, mask(r["key"]), st, note)) + time.sleep(0.35) + save(d) + print("итог %s: %s" % (service, stat)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("service") + ap.add_argument("--limit", type=int, default=0) + a = ap.parse_args() + if a.service == "all": + for s in CHECKS: + run(s, a.limit) + elif a.service in CHECKS: + run(a.service, a.limit) + else: + print("поддерживается:", ", ".join(CHECKS)) \ No newline at end of file diff --git a/tools/wave_cn.py b/tools/wave_cn.py new file mode 100644 index 0000000..859b550 --- /dev/null +++ b/tools/wave_cn.py @@ -0,0 +1,86 @@ +"""Волна по китайским/азиатским источникам: форумы, реверс, security-новости. + +Проверяет кандидатов живыми запросами (лента / список тем), пишет живые в +omni/sources_cn.py. Запуск: python tools/wave_cn.py [--apply] +""" +import sys +import time +from pathlib import Path + +import requests + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "omni" / "sources_cn.py" + +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0 Safari/537.36") + +# name, url, adapter, category, tags, limit +CAND = [ + ("cn_52pojie", "https://www.52pojie.cn/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "reverse", "abuse"), 20), + ("cn_nodeseek", "https://www.nodeseek.com/", "html_listing", "forum", ("cn", "hosting"), 20), + ("cn_freebuf", "https://www.freebuf.com/", "html_listing", "security", ("cn", "security"), 20), + ("cn_seebug", "https://paper.seebug.org/", "html_listing", "security", ("cn", "papers", "security"), 15), + ("cn_anquanke", "https://www.anquanke.com/", "html_listing", "security", ("cn", "security"), 20), + ("cn_4hou", "https://www.4hou.com/", "html_listing", "security", ("cn", "security"), 15), + ("cn_kanxue", "https://bbs.kanxue.com/", "html_listing", "forum", ("cn", "reverse"), 20), + ("cn_ithome", "https://www.ithome.com/rss/", "rss", "news", ("cn", "tech"), 20), + ("cn_cnbeta", "https://www.cnbeta.com.tw/backend.php", "rss", "news", ("cn", "tech"), 20), + ("cn_sspai", "https://sspai.com/feed", "rss", "news", ("cn", "tech"), 15), + ("cn_chiphell", "https://www.chiphell.com/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "hardware"), 15), + ("cn_52pojie_rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", "forum", ("cn", "reverse"), 15), + ("cn_v2ex_rss", "https://www.v2ex.com/index.xml", "rss", "forum", ("cn", "community"), 20), + ("cn_linuxdo_rss", "https://linux.do/latest.rss", "rss", "forum", ("cn", "community", "ai"), 20), + ("cn_hostloc_rss", "https://hostloc.com/forum.php?mod=rss", "rss", "forum", ("cn", "hosting"), 15), + ("cn_baidu_sec", "https://xz.aliyun.com/", "html_listing", "security", ("cn", "security"), 15), + ("cn_secfree", "https://www.secfree.com/", "html_listing", "security", ("cn", "security"), 10), + ("cn_tttang", "https://tttang.com/", "html_listing", "security", ("cn", "security"), 10), +] + + +def probe(url, adapter="html_listing"): + try: + r = requests.get(url, headers={"User-Agent": UA}, timeout=18, allow_redirects=True) + except Exception as e: + return 0, type(e).__name__, 0 + if adapter == "rss": + n_links = r.text.count("= 30 + print("%-18s %-3s links %-5d %s" % (name, code or err, links, "LIVE" if ok else "—")) + if ok: + live.append((name, url, adapter, cat, tags, lim)) + else: + dead.append(name) + time.sleep(0.8) + + print("\nживых %d, мёртвых %d" % (len(live), len(dead))) + if not apply: + print("dry-run: --apply чтобы записать omni/sources_cn.py") + return 0 + + body = ['"""Китайские и азиатские источники (проверены живыми запросами tools/wave_cn.py).', + '', 'Форумы реверса/хостинга, security-новости, tech-ленты. Все — публичные списки,', + 'без ключей и кук. Подключаются в реестр omni как категории forum/security/news.', + '"""', '', 'CN = ['] + for name, url, adapter, cat, tags, lim in live: + t = ", ".join('"%s"' % x for x in tags) + body.append(' {"name": "%s", "category": "%s", "kind": "site", "url": "%s",' + ' "adapter": "%s", "tags": (%s), "limit": %d},' % (name, cat, url, adapter, t, lim)) + body.append(']') + OUT.write_text("\n".join(body) + "\n", encoding="utf-8", newline="\n") + print("записано", OUT, "| источников", len(live)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/web/index.html b/web/index.html index 1a524ba..edae49e 100644 --- a/web/index.html +++ b/web/index.html @@ -524,6 +524,21 @@ async function loadTypes() { if (r.ok) buildTypes(r.types); } +async function loadKeys() { + try { + const k = await api('/api/keys'); + if (!k || k.error) return; + const top = (k.services || []).slice(0, 6) + .map(x => `${x.service} ${fmt(x.keys)}`).join(' · '); + const f = $('#aside-foot'); + if (f && !f.dataset.keys) { + f.dataset.keys = '1'; + f.innerHTML += `
${fmt(k.total_keys)} ключей · ${(k.services||[]).length} сервисов` + + (top ? `
${top}` : ''); + } + } catch (e) {} +} + async function loadStats() { const s = await api('/api/stats'); if (s.error) return toast(s.error, 'err'); @@ -536,8 +551,10 @@ async function loadStats() { `${fmt(s.sources)} источников · ${fmt(s.sources_enabled)} вкл
` + `${fmt(s.alive)} живых · ${fmt(s.tested)} проверено
` + `${fmt(s.runs)} прогонов · ${fmt(s.priced)} с ценой`; + $('#aside-foot').dataset.keys = ''; renderCats(s.categories); buildChips(s.categories); + loadKeys(); loadTypes(); const ks = [ ['записей', s.items, 'в sqlite + fts5', ''],