vault ключей + мост abuse-radar + китайские источники: всё в одном реестре
- omni/vault.py: пул ключей парсеров (env-first, data/vault.json, ротация, cooldown 401/402/403=1ч, 429=10м), только маски в выводе - tools/build_vault.py: сборка пула из MASTER_KEYS.csv и локальных дампов (значения не в git) - tools/vault_check.py: живая проверка с маскированием - searchers: firecrawl/tavily/exa/serper/jina/apify берут ключ из vault, если нет env - omni radar: домены+каналы из radar_server.db -> реестр, --import-posts заливает посты каналов - radar-источники помечены opt_in: общий обход не тормозят (869 против 3337 всего) - omni/sources_cn.py + tools/wave_cn.py: 9 живых китайских источников (52pojie, nodeseek, kanxue, anquanke, 4hou, xz.aliyun, ithome, cnbeta, v2ex) - net.get(proxy=...): принудительный прокси для CF-площадок - /api/keys + строка ключей в панели - tests/test_vault_radar.py: 7 тестов, всего 29 passed
This commit is contained in:
ENI
139
tools/build_vault.py
Normal file
139
tools/build_vault.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""Единый инвентарь ключей для парсеров: MASTER_KEYS.csv -> data/vault.json.
|
||||
|
||||
Собирает всё, что лежит по диску (MASTER_KEYS.csv, пулы keyhunter, fl_bot), и
|
||||
раскладывает по сервисам. Файл data/vault.json в .gitignore — ключи не уезжают в репо.
|
||||
|
||||
Запуск:
|
||||
python tools/build_vault.py # собрать
|
||||
python tools/build_vault.py --stats # только показать, что получилось
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "data" / "vault.json"
|
||||
|
||||
SOURCES = [
|
||||
Path("C:/Users/User/tmp/keys_inventory/MASTER_KEYS.csv"),
|
||||
Path("C:/Users/User/Desktop/_DOCS/ALL_WORKING_KEYS_WITH_BALANCE.txt"),
|
||||
]
|
||||
|
||||
# service в MASTER_KEYS -> имя сервиса в vault
|
||||
ALIAS = {
|
||||
"firecrawl.dev": "firecrawl",
|
||||
"tavily": "tavily",
|
||||
"exa": "exa",
|
||||
"serper": "serper",
|
||||
"jina": "jina",
|
||||
"apify": "apify",
|
||||
"scrapingant": "scrapingant",
|
||||
"brightdata": "brightdata",
|
||||
"scraperapi": "scraperapi",
|
||||
"zenrows": "zenrows",
|
||||
"twocaptcha": "2captcha",
|
||||
"2captcha": "2captcha",
|
||||
"capmonster": "capmonster",
|
||||
"telegram": "telegram",
|
||||
"proxy": "proxy",
|
||||
}
|
||||
|
||||
KEY_RE = re.compile(r"^[A-Za-z0-9_\-\.:]{16,200}$")
|
||||
|
||||
|
||||
def norm_service(s):
|
||||
s = (s or "").strip().lower()
|
||||
for k, v in ALIAS.items():
|
||||
if s == k or s.startswith(k):
|
||||
return v
|
||||
return s
|
||||
|
||||
|
||||
def good_key(k):
|
||||
k = (k or "").strip().strip('"').strip("'")
|
||||
if not k or " " in k or len(k) < 16:
|
||||
return ""
|
||||
return k if KEY_RE.match(k) else ""
|
||||
|
||||
|
||||
def from_csv(path, bag):
|
||||
for r in csv.DictReader(path.open(encoding="utf-8", errors="ignore")):
|
||||
svc = norm_service(r.get("service"))
|
||||
key = good_key(r.get("api_key"))
|
||||
if svc and key:
|
||||
bag[svc].append({"key": key, "status": (r.get("status") or "").strip(),
|
||||
"quota": (r.get("quota_balance") or "").strip(),
|
||||
"src": Path(r.get("source_file") or path.name).name})
|
||||
return True
|
||||
|
||||
|
||||
def from_txt(path, bag):
|
||||
"""Строки вида TAG|key или TAG:key или key — берём только пары с тегом."""
|
||||
pat = re.compile(r"^\s*([A-Za-z0-9_\-]{2,20})\s*[|:]\s*([A-Za-z0-9_\-\.]{16,200})\s*$")
|
||||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
m = pat.match(line)
|
||||
if m:
|
||||
svc = norm_service(m.group(1))
|
||||
bag[svc].append({"key": m.group(2), "status": "unknown", "quota": "", "src": path.name})
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--stats", action="store_true", help="только показать текущий vault")
|
||||
a = ap.parse_args()
|
||||
|
||||
if a.stats:
|
||||
if not OUT.exists():
|
||||
print("vault.json нет — запусти без --stats")
|
||||
return 1
|
||||
d = json.loads(OUT.read_text(encoding="utf-8"))
|
||||
total = sum(len(v) for v in d.values())
|
||||
print("сервисов %d, ключей %d" % (len(d), total))
|
||||
for s, rows in sorted(d.items(), key=lambda x: -len(x[1]))[:25]:
|
||||
alive = sum(1 for r in rows if r.get("status") in ("live", "alive", "ok", "live-verified-2026-10-24"))
|
||||
print(" %-22s %5d %s" % (s, len(rows), ("(живых %d)" % alive) if alive else ""))
|
||||
return 0
|
||||
|
||||
bag = defaultdict(list)
|
||||
used = 0
|
||||
for p in SOURCES:
|
||||
if not p.exists():
|
||||
continue
|
||||
try:
|
||||
if p.suffix == ".csv":
|
||||
from_csv(p, bag)
|
||||
else:
|
||||
from_txt(p, bag)
|
||||
used += 1
|
||||
except Exception as e:
|
||||
print("skip %s: %s" % (p.name, type(e).__name__))
|
||||
|
||||
# дедуп по значению ключа внутри сервиса
|
||||
out = {}
|
||||
for svc, rows in bag.items():
|
||||
seen, clean = set(), []
|
||||
for r in rows:
|
||||
if r["key"] in seen:
|
||||
continue
|
||||
seen.add(r["key"])
|
||||
clean.append(r)
|
||||
out[svc] = clean
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps(out, ensure_ascii=False, indent=0), encoding="utf-8")
|
||||
total = sum(len(v) for v in out.values())
|
||||
print("файлов прочитано %d | сервисов %d | ключей %d | записано в %s"
|
||||
% (used, len(out), total, OUT))
|
||||
for s in ("firecrawl", "tavily", "exa", "serper", "jina", "apify", "2captcha", "telegram", "proxy"):
|
||||
if s in out:
|
||||
print(" %-12s %d" % (s, len(out[s])))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
112
tools/vault_check.py
Normal file
112
tools/vault_check.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Проверка ключей vault живыми запросами. Значения не печатаются — только маски.
|
||||
|
||||
python tools/vault_check.py firecrawl --limit 20
|
||||
python tools/vault_check.py 2captcha
|
||||
python tools/vault_check.py all
|
||||
|
||||
Статусы в data/vault.json: live / dead / rate / nofund.
|
||||
firecrawl: GET /v2/team/credit-usage (200 = живой, 402 = без кредитов, 401 = мёртвый).
|
||||
2captcha: GET res.php?action=getbalance (status:1 = живой, баланс в request).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VAULT = ROOT / "data" / "vault.json"
|
||||
|
||||
|
||||
def load():
|
||||
return json.loads(VAULT.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def save(d):
|
||||
VAULT.write_text(json.dumps(d, ensure_ascii=False, indent=0), encoding="utf-8")
|
||||
|
||||
|
||||
def mask(k):
|
||||
return k[:4] + "…" + k[-2:] if len(k) > 8 else "…"
|
||||
|
||||
|
||||
def check_firecrawl(key):
|
||||
for url in ("https://api.firecrawl.dev/v2/team/credit-usage",
|
||||
"https://api.firecrawl.dev/v1/credits"):
|
||||
try:
|
||||
r = requests.get(url, headers={"Authorization": "Bearer " + key}, timeout=20)
|
||||
except Exception as e:
|
||||
return "err", type(e).__name__
|
||||
if r.status_code == 200:
|
||||
try:
|
||||
j = r.json()
|
||||
except Exception:
|
||||
j = {}
|
||||
rem = j.get("remainingCredits") or j.get("remaining_credits") or \
|
||||
(j.get("data") or {}).get("remainingCredits")
|
||||
return "live", ("credits %s" % rem) if rem is not None else "ok"
|
||||
if r.status_code == 401:
|
||||
return "dead", "401"
|
||||
if r.status_code == 402:
|
||||
return "nofund", "402"
|
||||
if r.status_code == 429:
|
||||
return "rate", "429"
|
||||
if r.status_code == 403:
|
||||
return "dead", "403"
|
||||
return "dead", "no-endpoint"
|
||||
|
||||
|
||||
def check_2captcha(key):
|
||||
try:
|
||||
r = requests.get("https://2captcha.com/res.php",
|
||||
params={"key": key, "action": "getbalance", "json": 1}, timeout=20)
|
||||
j = r.json()
|
||||
except Exception as e:
|
||||
return "err", type(e).__name__
|
||||
if j.get("status") == 1:
|
||||
return "live", "баланс %s" % j.get("request")
|
||||
err = str(j.get("request") or "")
|
||||
if "ERROR_WRONG_USER_KEY" in err or "ERROR_KEY_DOES_NOT_EXIST" in err:
|
||||
return "dead", err[:30]
|
||||
return "rate", err[:30]
|
||||
|
||||
|
||||
CHECKS = {"firecrawl": check_firecrawl, "2captcha": check_2captcha}
|
||||
|
||||
|
||||
def run(service, limit):
|
||||
d = load()
|
||||
rows = d.get(service) or []
|
||||
if not rows:
|
||||
print("в vault нет сервиса", service)
|
||||
return
|
||||
fn = CHECKS[service]
|
||||
rows = rows[:limit] if limit else rows
|
||||
print("%s: проверяю %d ключей" % (service, len(rows)))
|
||||
stat = {}
|
||||
for r in rows:
|
||||
st, note = fn(r["key"])
|
||||
r["status"] = st
|
||||
r["note"] = note
|
||||
stat[st] = stat.get(st, 0) + 1
|
||||
mark = {"live": "+", "dead": "-", "nofund": "$", "rate": "~", "err": "!"}.get(st, "?")
|
||||
print(" %s %s %-7s %s" % (mark, mask(r["key"]), st, note))
|
||||
time.sleep(0.35)
|
||||
save(d)
|
||||
print("итог %s: %s" % (service, stat))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("service")
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
a = ap.parse_args()
|
||||
if a.service == "all":
|
||||
for s in CHECKS:
|
||||
run(s, a.limit)
|
||||
elif a.service in CHECKS:
|
||||
run(a.service, a.limit)
|
||||
else:
|
||||
print("поддерживается:", ", ".join(CHECKS))
|
||||
86
tools/wave_cn.py
Normal file
86
tools/wave_cn.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""Волна по китайским/азиатским источникам: форумы, реверс, security-новости.
|
||||
|
||||
Проверяет кандидатов живыми запросами (лента / список тем), пишет живые в
|
||||
omni/sources_cn.py. Запуск: python tools/wave_cn.py [--apply]
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "omni" / "sources_cn.py"
|
||||
|
||||
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
|
||||
|
||||
# name, url, adapter, category, tags, limit
|
||||
CAND = [
|
||||
("cn_52pojie", "https://www.52pojie.cn/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "reverse", "abuse"), 20),
|
||||
("cn_nodeseek", "https://www.nodeseek.com/", "html_listing", "forum", ("cn", "hosting"), 20),
|
||||
("cn_freebuf", "https://www.freebuf.com/", "html_listing", "security", ("cn", "security"), 20),
|
||||
("cn_seebug", "https://paper.seebug.org/", "html_listing", "security", ("cn", "papers", "security"), 15),
|
||||
("cn_anquanke", "https://www.anquanke.com/", "html_listing", "security", ("cn", "security"), 20),
|
||||
("cn_4hou", "https://www.4hou.com/", "html_listing", "security", ("cn", "security"), 15),
|
||||
("cn_kanxue", "https://bbs.kanxue.com/", "html_listing", "forum", ("cn", "reverse"), 20),
|
||||
("cn_ithome", "https://www.ithome.com/rss/", "rss", "news", ("cn", "tech"), 20),
|
||||
("cn_cnbeta", "https://www.cnbeta.com.tw/backend.php", "rss", "news", ("cn", "tech"), 20),
|
||||
("cn_sspai", "https://sspai.com/feed", "rss", "news", ("cn", "tech"), 15),
|
||||
("cn_chiphell", "https://www.chiphell.com/forum.php?mod=guide&view=new", "html_listing", "forum", ("cn", "hardware"), 15),
|
||||
("cn_52pojie_rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", "forum", ("cn", "reverse"), 15),
|
||||
("cn_v2ex_rss", "https://www.v2ex.com/index.xml", "rss", "forum", ("cn", "community"), 20),
|
||||
("cn_linuxdo_rss", "https://linux.do/latest.rss", "rss", "forum", ("cn", "community", "ai"), 20),
|
||||
("cn_hostloc_rss", "https://hostloc.com/forum.php?mod=rss", "rss", "forum", ("cn", "hosting"), 15),
|
||||
("cn_baidu_sec", "https://xz.aliyun.com/", "html_listing", "security", ("cn", "security"), 15),
|
||||
("cn_secfree", "https://www.secfree.com/", "html_listing", "security", ("cn", "security"), 10),
|
||||
("cn_tttang", "https://tttang.com/", "html_listing", "security", ("cn", "security"), 10),
|
||||
]
|
||||
|
||||
|
||||
def probe(url, adapter="html_listing"):
|
||||
try:
|
||||
r = requests.get(url, headers={"User-Agent": UA}, timeout=18, allow_redirects=True)
|
||||
except Exception as e:
|
||||
return 0, type(e).__name__, 0
|
||||
if adapter == "rss":
|
||||
n_links = r.text.count("<item") + r.text.count("<entry")
|
||||
else:
|
||||
n_links = r.text.count("<a ")
|
||||
return r.status_code, "", n_links
|
||||
|
||||
|
||||
def main():
|
||||
apply = "--apply" in sys.argv
|
||||
live, dead = [], []
|
||||
for name, url, adapter, cat, tags, lim in CAND:
|
||||
code, err, links = probe(url, adapter)
|
||||
ok = code == 200 and links >= 30
|
||||
print("%-18s %-3s links %-5d %s" % (name, code or err, links, "LIVE" if ok else "—"))
|
||||
if ok:
|
||||
live.append((name, url, adapter, cat, tags, lim))
|
||||
else:
|
||||
dead.append(name)
|
||||
time.sleep(0.8)
|
||||
|
||||
print("\nживых %d, мёртвых %d" % (len(live), len(dead)))
|
||||
if not apply:
|
||||
print("dry-run: --apply чтобы записать omni/sources_cn.py")
|
||||
return 0
|
||||
|
||||
body = ['"""Китайские и азиатские источники (проверены живыми запросами tools/wave_cn.py).',
|
||||
'', 'Форумы реверса/хостинга, security-новости, tech-ленты. Все — публичные списки,',
|
||||
'без ключей и кук. Подключаются в реестр omni как категории forum/security/news.',
|
||||
'"""', '', 'CN = [']
|
||||
for name, url, adapter, cat, tags, lim in live:
|
||||
t = ", ".join('"%s"' % x for x in tags)
|
||||
body.append(' {"name": "%s", "category": "%s", "kind": "site", "url": "%s",'
|
||||
' "adapter": "%s", "tags": (%s), "limit": %d},' % (name, cat, url, adapter, t, lim))
|
||||
body.append(']')
|
||||
OUT.write_text("\n".join(body) + "\n", encoding="utf-8", newline="\n")
|
||||
print("записано", OUT, "| источников", len(live))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user