"""Автодобавление источников: находит фиды по домашним страницам и каталогам.

Логика: берём затравки (хосты уже подключённых источников + известные каталоги
сообществ), вытаскиваем ссылки на фиды (link rel=alternate, /feed, /rss,
/latest.json, подфорумы), прогоняем каждый кандидат реальным адаптером и
добавляем только те, что отдали записи. Всё добавленное помечается auto и
попадает в панель как обычный источник.
"""
from __future__ import annotations

import json
import re
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor

from bs4 import BeautifulSoup

from . import feeds, net, source_state, sources

FEED_PATHS = ("/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml",
              "/latest.json", "/?feed=rss2", "/feed/", "/blog/feed")
SEED_CATALOGS = [
    # каталоги сообществ и подфорумов, откуда удобно цеплять новые фиды
    "https://www.reddit.com/r/LocalLLaMA/about.json",
    "https://news.ycombinator.com/",
    "https://lobste.rs/",
]
UA = getattr(feeds, "UA", {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})


def _hosts():
    seen = {}
    for s in sources.all_sources():
        try:
            h = urllib.parse.urlsplit(s.url).netloc.lower()
        except Exception:
            continue
        if not h or h in seen:
            continue
        seen[h] = s.category
    return seen


def _feed_links(html, base):
    out = []
    soup = BeautifulSoup(html, "lxml")
    for ln in soup.find_all("link", href=True):
        t = (ln.get("type") or "").lower()
        rel = " ".join(ln.get("rel") or []).lower()
        if "alternate" in rel and ("rss" in t or "atom" in t or "json" in t):
            out.append(urllib.parse.urljoin(base, ln["href"]))
    for a in soup.find_all("a", href=True):
        href = str(a["href"])
        if re.search(r"(/feed|/rss|\.rss|atom\.xml|/latest\.json)$", href):
            out.append(urllib.parse.urljoin(base, href))
    return out


def candidates(limit_hosts=60, extra_seeds=()):
    hosts = _hosts()
    seeds = []
    for h, cat in list(hosts.items())[:limit_hosts]:
        seeds.append(("https://%s/" % h, cat))
    for u in list(extra_seeds) + SEED_CATALOGS:
        seeds.append((u, "auto"))
    return seeds


def probe(url, category="auto", limit=6):
    """Проверка одного URL реальным адаптером. Возвращает (ok, n, adapter)."""
    low = url.lower()
    if "t.me/s/" in low:
        ad, kind = "tg_web", "html"
    elif low.endswith(".json") or "/latest.json" in low:
        ad, kind = ("discourse", "json") if "/latest.json" in low else ("generic_json", "json")
    elif any(x in low for x in (".rss", "/rss", "/feed", "atom.xml", "feed.xml", "index.xml")):
        ad, kind = "rss", "rss"
    else:
        ad, kind = "html_listing", "html"
    src = sources.Source(name="probe", category=category, kind=kind, url=url,
                         adapter=ad, tags=("auto",), limit=limit, detail=False, extra={})
    fn = feeds.ADAPTERS.get(ad)
    if not fn:
        return False, 0, ad
    try:
        items = fn(src, limit) or []
    except Exception:
        return False, 0, ad
    return len(items) > 0, len(items), ad


def discover(limit_hosts=40, workers=8, max_add=40, dry=False):
    seeds = candidates(limit_hosts=limit_hosts)
    found = []
    def scan(seed):
        base, cat = seed
        try:
            r = net.get(base, headers=UA)
            if r.status != 200:
                return []
        except Exception:
            return []
        links = _feed_links(r.text[:400000], base)
        dom = urllib.parse.urlsplit(base)
        for p in FEED_PATHS:
            links.append("%s://%s%s" % (dom.scheme, dom.netloc, p))
        # подфорумы Discourse/Reddit
        if "reddit.com" in base:
            for sub in ("LocalLLaMA", "MachineLearning", "mcp", "selfhosted"):
                links.append("https://www.reddit.com/r/%s/.rss" % sub)
        return [(u, cat) for u in dict.fromkeys(links)]

    with ThreadPoolExecutor(max_workers=workers) as ex:
        for res in ex.map(scan, seeds):
            found.extend(res)
    print("кандидатов найдено:", len(found), flush=True)

    known = {s.url for s in sources.all_sources()}
    found = [(u, c) for u, c in found if u not in known]
    print("после дедупа:", len(found), flush=True)

    def check(pair):
        url, cat = pair
        ok, n, ad = probe(url, cat)
        return url, cat, ok, n, ad

    # только машинные фиды: html-листинги на автодобавлении дают мусор
    best = {}
    with ThreadPoolExecutor(max_workers=workers) as ex:
        for url, cat, ok, n, ad in ex.map(check, found):
            if not ok or ad == "html_listing" or n < 3:
                continue
            dom = urllib.parse.urlsplit(url).netloc.lower()
            key = (dom, ad)
            if key not in best or n > best[key][2]:
                best[key] = (url, cat, n, ad)

    added = []
    for (dom, ad), (url, cat, n, _ad) in sorted(best.items(), key=lambda kv: -kv[1][2]):
        name = "auto_" + re.sub(r"[^a-z0-9]+", "_", dom).strip("_")[:34]
        if any(a["name"] == name for a in added):
            continue
        rec = {"name": name, "category": cat if cat in ("abuse", "forum", "security") else "community",
               "kind": "rss" if ad == "rss" else ("json" if ad in ("generic_json", "discourse") else "html"),
               "url": url, "adapter": ad, "tags": ["auto"], "lim": 15, "detail": False}
        added.append(rec)
        if len(added) >= max_add:
            break
    print("живых фидов:", len(added), flush=True)
    if not dry:
        for rec in added:
            source_state.add_custom(rec["name"], rec["category"], rec["url"], kind=rec["kind"],
                                    adapter=rec["adapter"], tags=rec["tags"], limit=rec["lim"],
                                    detail=rec["detail"])
    return {"found": len(found), "alive": len(added), "added": added if dry else [a["name"] for a in added]}


if __name__ == "__main__":
    import sys
    lim = int(sys.argv[1]) if len(sys.argv) > 1 else 30
    res = discover(limit_hosts=lim)
    print(json.dumps({"found": res["found"], "alive": res["alive"], "added": res["added"][:20]},
                     ensure_ascii=False, indent=1))