"""Авто-расширение источников: тест кандидатов → генерация omni/sources_auto.py.

    python tools/auto_expand.py --probe         # только проверка, без записи
    python tools/auto_expand.py --write         # живые кандидаты → sources_auto.py
    python tools/auto_expand.py --discover      # добрать кандидатов из public-apis
    python tools/auto_expand.py --add NAME URL --adapter rss

Кандидаты: data/source_candidates.json (list of {name,url,adapter,kind,extra,tags,limit}).
Здоровье: data/source_health.json — сколько раз источник отдал items и когда.
Мёртвым считается кандидат с 0 items в 3 прогонах подряд → исключается из генерации.
"""
import argparse
import json
import sys
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from omni import feeds  # noqa: E402
from omni.sources import Source  # noqa: E402

CAND = ROOT / "data" / "source_candidates.json"
HEALTH = ROOT / "data" / "source_health.json"
OUT = ROOT / "omni" / "sources_auto.py"
PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md"
RE_URL = __import__("re").compile(r"\((https?://[^)\s]+)\)")
RE_LINK = __import__("re").compile(r"\[([^\]]+)\]\([^)]+\)")

HEADER = '''"""Авто-расширение: источники на публичных API без ключа.

Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.

Обновлено: {ts} · живых {live} из {total}
"""


def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None):
    return {{"name": name, "category": "abuse", "kind": kind, "url": url,
            "adapter": adapter, "tags": tuple(tags), "limit": limit,
            "detail": False, "extra": dict(extra or {{}})}}


AUTO = [
{body}]
'''

GROUP_RU = {
    "json": "# ---------- открытые JSON API без ключа ----------",
    "reddit": "# ---------- Reddit через pullpush.io (без ключа) ----------",
    "social": "# ---------- Bluesky / Mastodon ----------",
    "discourse": "# ---------- Discourse-сообщества ----------",
    "rss": "# ---------- RSS/Atom ----------",
    "tg": "# ---------- Telegram-превью ----------",
}


def load(path, default):
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception:
        return default


def save(path, data):
    path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")


def to_source(c):
    return Source(name=c["name"], category=c.get("category", "abuse"), kind=c.get("kind", "json"),
                  url=c["url"], adapter=c["adapter"], tags=tuple(c.get("tags") or ("abuse",)),
                  limit=int(c.get("limit", 20)), detail=False, extra=dict(c.get("extra") or {}))


def probe(cands, pause=2.0):
    health = load(HEALTH, {})
    res = []
    last_host = None
    for c in cands:
        host = c["url"].split("/")[2] if "//" in c["url"] else ""
        if host == last_host and pause:
            time.sleep(pause)
        last_host = host
        try:
            items = feeds.fetch(to_source(c), limit=min(20, int(c.get("limit", 20))))
            n = len(items)
            sample = [i.title[:70] for i in items[:2]]
            err = ""
        except Exception as exc:
            n, sample, err = 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:90])
        h = health.setdefault(c["name"], {"ok": 0, "fail": 0, "items": 0})
        h["items"] = n
        h["ok"] = h.get("ok", 0) + 1 if n else 0
        h["fail"] = 0 if n else h.get("fail", 0) + 1
        h["last"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M")
        h["url"] = c["url"]
        if err:
            h["error"] = err
        else:
            h.pop("error", None)
        res.append((c, n, sample, err))
        print("%-26s %-4s %3d  %s" % (c["name"], "LIVE" if n else "DEAD", n, err or " | ".join(sample)))
    save(HEALTH, health)
    return res


def write_module(cands, results):
    health = load(HEALTH, {})
    order = ["json", "reddit", "social", "discourse", "rss", "tg"]
    buckets = {k: [] for k in order}
    for c, n, _, _ in results:
        if not n:
            continue
        if (health.get(c["name"]) or {}).get("fail", 0) >= 3:
            print("skip (3 fails подряд):", c["name"])
            continue
        g = c.get("group") or ("tg" if c["adapter"] == "tg_web" else
                               "discourse" if c["adapter"] == "discourse" else
                               "reddit" if "pullpush" in c["url"] or "reddit" in c["url"] else
                               "social" if c["adapter"] in ("bsky_search", "mastodon_tag") else
                               "rss" if c["adapter"] == "rss" else "json")
        buckets.setdefault(g, []).append(c)
    body = []
    live = 0
    for g in order + [k for k in buckets if k not in order]:
        rows = buckets.get(g) or []
        if not rows:
            continue
        body.append("    " + GROUP_RU.get(g, "# ---------- прочее ----------"))
        for c in rows:
            live += 1
            extra = ""
            if c.get("extra"):
                extra = ", extra=%s" % json.dumps(c["extra"], ensure_ascii=False)
            tags = ", ".join('"%s"' % t for t in (c.get("tags") or ("abuse",)))
            body.append('    A("%s", "%s", "%s", "%s", (%s)%s),' % (
                c["name"], c.get("kind", "json"), c["url"], c["adapter"], tags, extra))
        body.append("")
    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    OUT.write_text(HEADER.format(ts=ts, live=live, total=len(cands), body="\n".join(body) + "\n"),
                   encoding="utf-8")
    print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands)))


CAT_OK = ("Artificial Intelligence", "Machine Learning", "Security", "News", "Open Data",
          "Development", "Text Analysis", "Data Validation", "Sports", "Cryptocurrency")
KEEP_ANY_CAT = ("llm", "gpt", "openai", "threat", "cve", "vuln", "leak", "hack", "malware",
                "phish", "proxy", "temp mail", "sms", "telegram", "scrap", "dns", "whois")
BAD = ("waifu", "anime", "cat fact", "dog", "joke", "meme", "pokemon", "quote", "trivia",
       "recipe", "cocktail", "game", "3d", "texture", "insurance", "crypto price")
KW_OK = ("ai", "llm", "gpt", "model", "secur", "threat", "cve", "vuln", "leak", "hack", "news",
         "feed", "telegram", "proxy", "mail", "sms", "text", "translate", "scrap", "search",
         "crypto", "token", "domain", "dns", "ip", "geo")


def discover(limit=40):
    """Добрать бесплатные (Auth=No) API из public-apis: только релевантные категории/слова."""
    req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"})
    md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace")
    rows, cat = [], ""
    for line in md.splitlines():
        if line.startswith("### "):
            cat = line[4:].strip()
            continue
        if not line.startswith("|") or "http" not in line:
            continue
        cells = [c.strip() for c in line.strip("|").split("|")]
        if len(cells) < 5:
            continue
        m = RE_URL.search(cells[0])
        if not m:
            continue
        url = m.group(1)
        title = RE_LINK.sub(r"\1", cells[0]).strip()
        desc, auth = cells[1], cells[2]
        if "No" not in auth:
            continue
        blob = (title + " " + desc).lower()
        if cat not in CAT_OK and not any(k in blob for k in KEEP_ANY_CAT):
            continue
        if any(b in blob for b in BAD):
            continue
        host = url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_").replace("-", "_")
        rows.append({"name": "dis_" + host[:26], "url": url, "adapter": "generic_json",
                     "kind": "json", "tags": ["abuse", "public_api"], "group": "json", "limit": 15,
                     "note": (title + " — " + desc)[:120], "cat": cat})
    uniq = {}
    for r in rows:
        uniq.setdefault(r["name"], r)
    cands = load(CAND, [])
    have = {c["name"] for c in cands} | {c["url"] for c in cands}
    new = [r for r in uniq.values() if r["name"] not in have and r["url"] not in have][:limit]
    cands.extend(new)
    save(CAND, cands)
    print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands)))
    for r in new[:12]:
        print("  + %-30s %s" % (r["name"], r["note"][:80]))
    return new


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--probe", action="store_true")
    ap.add_argument("--write", action="store_true")
    ap.add_argument("--write-only", action="store_true", help="сгенерировать из data/source_health.json без сети")
    ap.add_argument("--discover", action="store_true")
    ap.add_argument("--add", nargs=2, metavar=("NAME", "URL"))
    ap.add_argument("--adapter", default="generic_json")
    ap.add_argument("--pause", type=float, default=2.0)
    a = ap.parse_args()

    if a.add:
        cands = load(CAND, [])
        cands.append({"name": a.add[0], "url": a.add[1], "adapter": a.adapter,
                      "kind": "json", "tags": ["abuse"], "limit": 20})
        save(CAND, cands)
        print("added", a.add[0])
        return 0
    if a.discover:
        discover()
        return 0

    cands = load(CAND, [])
    if a.write_only:
        health = load(HEALTH, {})
        results = [(c, (health.get(c["name"]) or {}).get("items", 0), [], "") for c in cands]
        write_module(cands, results)
        return 0
    if not cands:
        print("нет кандидатов в", CAND)
        return 1
    results = probe(cands, pause=a.pause)
    live = sum(1 for _, n, _, _ in results if n)
    print("\nLIVE %d / %d, items %d" % (live, len(results), sum(n for _, n, _, _ in results)))
    if a.write:
        write_module(cands, results)
    return 0


if __name__ == "__main__":
    sys.exit(main())