omni-parse/tools/build_vault.py

"""Единый инвентарь ключей для парсеров: MASTER_KEYS.csv -> data/vault.json.

Собирает всё, что лежит по диску (MASTER_KEYS.csv, пулы keyhunter, fl_bot), и
раскладывает по сервисам. Файл data/vault.json в .gitignore — ключи не уезжают в репо.

Запуск:
  python tools/build_vault.py            # собрать
  python tools/build_vault.py --stats    # только показать, что получилось
"""
import argparse
import csv
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "data" / "vault.json"

SOURCES = [
    Path("C:/Users/User/tmp/keys_inventory/MASTER_KEYS.csv"),
    Path("C:/Users/User/Desktop/_DOCS/ALL_WORKING_KEYS_WITH_BALANCE.txt"),
]

# service в MASTER_KEYS -> имя сервиса в vault
ALIAS = {
    "firecrawl.dev": "firecrawl",
    "tavily": "tavily",
    "exa": "exa",
    "serper": "serper",
    "jina": "jina",
    "apify": "apify",
    "scrapingant": "scrapingant",
    "brightdata": "brightdata",
    "scraperapi": "scraperapi",
    "zenrows": "zenrows",
    "twocaptcha": "2captcha",
    "2captcha": "2captcha",
    "capmonster": "capmonster",
    "telegram": "telegram",
    "proxy": "proxy",
}

KEY_RE = re.compile(r"^[A-Za-z0-9_\-\.:]{16,200}$")


def norm_service(s):
    s = (s or "").strip().lower()
    for k, v in ALIAS.items():
        if s == k or s.startswith(k):
            return v
    return s


def good_key(k):
    k = (k or "").strip().strip('"').strip("'")
    if not k or " " in k or len(k) < 16:
        return ""
    return k if KEY_RE.match(k) else ""


def from_csv(path, bag):
    for r in csv.DictReader(path.open(encoding="utf-8", errors="ignore")):
        svc = norm_service(r.get("service"))
        key = good_key(r.get("api_key"))
        if svc and key:
            bag[svc].append({"key": key, "status": (r.get("status") or "").strip(),
                             "quota": (r.get("quota_balance") or "").strip(),
                             "src": Path(r.get("source_file") or path.name).name})
    return True


def from_txt(path, bag):
    """Строки вида TAG|key или TAG:key или key — берём только пары с тегом."""
    pat = re.compile(r"^\s*([A-Za-z0-9_\-]{2,20})\s*[|:]\s*([A-Za-z0-9_\-\.]{16,200})\s*$")
    for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
        m = pat.match(line)
        if m:
            svc = norm_service(m.group(1))
            bag[svc].append({"key": m.group(2), "status": "unknown", "quota": "", "src": path.name})
    return True


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--stats", action="store_true", help="только показать текущий vault")
    a = ap.parse_args()

    if a.stats:
        if not OUT.exists():
            print("vault.json нет — запусти без --stats")
            return 1
        d = json.loads(OUT.read_text(encoding="utf-8"))
        total = sum(len(v) for v in d.values())
        print("сервисов %d, ключей %d" % (len(d), total))
        for s, rows in sorted(d.items(), key=lambda x: -len(x[1]))[:25]:
            alive = sum(1 for r in rows if r.get("status") in ("live", "alive", "ok", "live-verified-2026-10-24"))
            print("  %-22s %5d %s" % (s, len(rows), ("(живых %d)" % alive) if alive else ""))
        return 0

    bag = defaultdict(list)
    used = 0
    for p in SOURCES:
        if not p.exists():
            continue
        try:
            if p.suffix == ".csv":
                from_csv(p, bag)
            else:
                from_txt(p, bag)
            used += 1
        except Exception as e:
            print("skip %s: %s" % (p.name, type(e).__name__))

    # дедуп по значению ключа внутри сервиса
    out = {}
    for svc, rows in bag.items():
        seen, clean = set(), []
        for r in rows:
            if r["key"] in seen:
                continue
            seen.add(r["key"])
            clean.append(r)
        out[svc] = clean

    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(json.dumps(out, ensure_ascii=False, indent=0), encoding="utf-8")
    total = sum(len(v) for v in out.values())
    print("файлов прочитано %d | сервисов %d | ключей %d | записано в %s"
          % (used, len(out), total, OUT))
    for s in ("firecrawl", "tavily", "exa", "serper", "jina", "apify", "2captcha", "telegram", "proxy"):
        if s in out:
            print("  %-12s %d" % (s, len(out[s])))
    return 0


if __name__ == "__main__":
    sys.exit(main())