"""Забор источников из реестра sources.py.

Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
"""
import html
import json
import re
import urllib.parse
import xml.etree.ElementTree as ET

from bs4 import BeautifulSoup

from . import config, net
from .model import Item

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
      "Accept-Language": "en,ru;q=0.8"}


def _txt(x):
    if x is None:
        return ""
    return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()


def _item(src, kind, url, title, text="", author="", published="", extra=None):
    ex = dict(extra or {})
    ex["tags"] = list(src.tags)
    return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
                author=_txt(author)[:80], published=str(published or ""), source=src.category,
                extra=ex)


# ---------------- rss ----------------
def _arxiv_api(src, limit):
    """arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
    m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
    cat = m.group(1) if m else "cs.AI"
    url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
           "&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
    r = net.get(url, headers=UA)
    if r.status != 200 or "<entry" not in (r.text or ""):
        return []
    out = []
    try:
        root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
    except Exception:
        return []
    ns = {"a": "http://www.w3.org/2005/Atom"}
    for e in root.findall(".//a:entry", ns)[:limit]:
        title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
        summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
        link = e.findtext("a:id", namespaces=ns) or ""
        pub = e.findtext("a:published", namespaces=ns) or ""
        auth = e.findtext("a:author/a:name", namespaces=ns) or ""
        if title:
            out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
                             {"cat": cat}))
    return out


def _rss(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status in (429, 503):
        import time
        time.sleep(2.5)
        r = net.get(src.url, headers=UA)
    if r.status != 200 or not r.text.strip():
        return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
    out = []
    try:
        root = ET.fromstring(r.text.encode("utf-8", "ignore"))
    except Exception:
        return _rss_regex(src, r.text, limit)
    ns = {"a": "http://www.w3.org/2005/Atom"}
    entries = root.findall(".//item") or root.findall(".//a:entry", ns)
    for e in entries[:limit]:
        g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
        link = g("link")
        if not link:
            le = e.find("a:link", ns) or e.find("link")
            link = le.get("href") if le is not None and le.get("href") else ""
        body = g("description") or g("summary") or g("content")
        out.append(_item(src, "entry", link, g("title"), body,
                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
    if not out and "arxiv.org" in src.url:
        return _arxiv_api(src, limit)
    return out


def _rss_regex(src, text, limit):
    out = []
    for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
        blob = m.group(1)
        g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
        out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
        if len(out) >= limit:
            break
    return out


# ---------------- json adapters ----------------
def _hn_top(src, limit):
    ids = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for sid in ids[:limit]:
        d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
        if not d.get("title"):
            continue
        out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
                         d.get("title"), d.get("text", ""), d.get("by", ""),
                         str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
                                                  "hn": f"https://news.ycombinator.com/item?id={sid}"}))
    return out


def _reddit_listing(src, limit):
    """Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
    sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
    rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
    import time
    children = []
    if rss_first:
        rr = net.get(rss_first, headers=UA)
        if rr.status == 200 and "<entry" in (rr.text or ""):
            return _reddit_atom(src, rr.text, limit)
        if rr.status in (429, 503):
            time.sleep(4)
            rr = net.get(rss_first, headers=UA)
            if rr.status == 200 and "<entry" in (rr.text or ""):
                return _reddit_atom(src, rr.text, limit)
    r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
    out = []
    if r.status == 200 and isinstance(r.json(), dict):
        children = (((r.json() or {}).get("data") or {}).get("children") or [])
    for ch in children[:limit]:
        d = ch.get("data") or {}
        if not d.get("title"):
            continue
        out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
                         d.get("title"), d.get("selftext", ""), d.get("author", ""),
                         str(d.get("created_utc", "")),
                         {"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
    if out:
        return out

    rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
    r2 = net.get(rss_url, headers=UA) if not rss_first else None
    if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
        return _reddit_atom(src, r2.text, limit)

    sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
    if sub:
        alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
                      f"&size={limit}&sort=desc", headers=UA, json_mode=True)
        for d in ((alt.json() or {}).get("data") or [])[:limit]:
            if d.get("title"):
                out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
                                 d.get("title"), d.get("selftext", ""), d.get("author", ""),
                                 str(d.get("created_utc", "")), {"ups": d.get("score")}))
    return out


def _reddit_atom(src, text, limit):
    out = []
    try:
        root = ET.fromstring(text.encode("utf-8", "ignore"))
    except Exception:
        return out
    ns = {"a": "http://www.w3.org/2005/Atom"}
    for e in root.findall(".//a:entry", ns)[:limit]:
        title = e.findtext("a:title", namespaces=ns) or ""
        link_el = e.find("a:link", ns)
        link = link_el.get("href") if link_el is not None else ""
        content = e.findtext("a:content", namespaces=ns) or ""
        author = e.findtext("a:author/a:name", namespaces=ns) or ""
        upd = e.findtext("a:updated", namespaces=ns) or ""
        if title:
            out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
    return out


def _hf_generic(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
                                          or d.get("spaces") or d.get("datasets") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        if "repoData" in x:  # trending: {repoType, repoData:{...}}
            x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
        rid = x.get("id") or x.get("repoId") or x.get("name") or ""
        if not rid:
            continue
        kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
        url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
        out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
                         "", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
    return out


def _hf_models(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
                         "", "", x.get("createdAt", ""),
                         {"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
    return out


def _hf_daily_papers(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        p = x.get("paper") or x
        pid = p.get("id") or ""
        out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
                         p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
                         p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
    return out


def _hf_rows(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for row in (d.get("rows") or [])[:limit]:
        r = row.get("row") or {}
        name = r.get("fullname") or r.get("model") or r.get("Model") or ""
        if not name:
            continue
        out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
                         "", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
    return out


def _openalex(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for w in (d.get("results") or [])[:limit]:
        out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
                         w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
                         {"cited": w.get("cited_by_count")}))
    return out


def _s2(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for p in (d.get("data") or [])[:limit]:
        out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
                         ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
                         str(p.get("year", "")), {"citations": p.get("citationCount")}))
    return out


def _samsung(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
                         x.get("title", ""), x.get("summary") or x.get("content", ""),
                         "", str(x.get("regDate") or x.get("date") or ""), {}))
    return out


def _lg_ai(src, limit):
    d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
                         x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
    return out


def _github_org(src, limit):
    hdr = dict(UA)
    tok = config.KEYS.get("GITHUB_TOKEN")
    if tok:
        hdr["Authorization"] = f"token {tok}"
    url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
    d = net.get(url, headers=hdr, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
                         x.get("description", ""), x.get("owner", {}).get("login", ""),
                         x.get("pushed_at", ""),
                         {"stars": x.get("stargazers_count"), "lang": x.get("language")}))
    return out


def _discourse(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    base = src.url.split("/latest.json")[0]
    out = []
    for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
        out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
                         t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
                         "", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
    return out


def _xenforo_latest(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    out = []
    for a in soup.select("a[href*='/threads/']"):
        title = a.get_text(" ", strip=True)
        href = str(a.get("href", "") or "")
        if len(title) < 12 or href.rstrip("/").endswith("/threads"):
            continue
        if not href.startswith("http"):
            href = "https://www.blackhatworld.com" + href
        out.append(_item(src, "thread", href, title))
        if len(out) >= limit:
            break
    return out


def _tg_web(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    out = []
    for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]:
        text_el = msg.select_one("div.tgme_widget_message_text")
        link_el = msg.select_one("a.tgme_widget_message_date")
        text = text_el.get_text("\n", strip=True) if text_el else ""
        link = str(link_el.get("href") or src.url) if link_el else src.url
        if not text:
            continue
        out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {}))
    return out


def _html_listing(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    inc = src.extra.get("include_prefixes") or []
    inc_c = src.extra.get("include_contains") or []
    exc = set(src.extra.get("exclude_exact") or [])
    base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
    seen, links = set(), []
    for a in soup.find_all("a", href=True):
        href = urllib.parse.urljoin(base, str(a["href"]))
        path = urllib.parse.urlsplit(href).path
        if path in exc or len(path) < 6:
            continue
        if inc and not any(path.startswith(p) for p in inc):
            continue
        if inc_c and not any(c in path for c in inc_c):
            continue
        if href in seen:
            continue
        seen.add(href)
        links.append((href, a.get_text(" ", strip=True)))
        if len(links) >= limit:
            break
    out = []
    for href, title in links:
        text = ""
        if src.detail:
            d = net.get(href, headers=UA)
            if d.status == 200:
                ds = BeautifulSoup(d.text, "lxml")
                for bad in ds(["script", "style", "nav", "footer", "header"]):
                    bad.decompose()
                paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
                text = " ".join(paras)[:4000]
                if not title:
                    t = ds.find("h1") or ds.title
                    title = t.get_text(" ", strip=True) if t else href
        out.append(_item(src, "entry", href, title or href, text))
    return out


ADAPTERS = {
    "rss": _rss,
    "hn_top": _hn_top,
    "reddit_listing": _reddit_listing,
    "hf_trending": _hf_generic,
    "hf_generic": _hf_generic,
    "hf_models": _hf_models,
    "hf_daily_papers": _hf_daily_papers,
    "hf_rows": _hf_rows,
    "openalex": _openalex,
    "s2": _s2,
    "samsung": _samsung,
    "lg_ai": _lg_ai,
    "github_org": _github_org,
    "discourse": _discourse,
    "xenforo_latest": _xenforo_latest,
    "tg_web": _tg_web,
    "html_listing": _html_listing,
}


def fetch(src, limit=None):
    fn = ADAPTERS.get(src.adapter)
    if not fn:
        return []
    n = limit or src.limit
    try:
        return fn(src, n) or []
    except Exception:
        return []


SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")


def fetch_many(sources, limit=None, workers=8):
    """Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
    import time
    from concurrent.futures import ThreadPoolExecutor
    slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
    fast = [s for s in sources if s not in slow]
    items, report = [], []
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [(ex.submit(fetch, s, limit), s) for s in fast]
        for f, s in futs:
            got = f.result()
            report.append((s.name, len(got)))
            items.extend(got)
    for s in slow:
        got = fetch(s, limit)
        if not got and "reddit.com" in s.url:
            time.sleep(7)
            got = fetch(s, limit)
        report.append((s.name, len(got)))
        items.extend(got)
        time.sleep(3.5 if "reddit.com" in s.url else 1.0)
    return items, report