"""Забор источников из реестра sources.py.

Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
"""
import html
import json
import re
import urllib.parse
import xml.etree.ElementTree as ET

from bs4 import BeautifulSoup

from . import config, net
from .model import Item

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
      "Accept-Language": "en,ru;q=0.8"}


def _txt(x):
    if x is None:
        return ""
    return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()


def _item(src, kind, url, title, text="", author="", published="", extra=None):
    ex = dict(extra or {})
    ex["tags"] = list(src.tags)
    t = _txt(title)
    body = _txt(text)
    if sum(ch.isalpha() for ch in t) < 12 and body:
        cand = [ln for ln in body.splitlines() if sum(ch.isalpha() for ch in ln) >= 12]
        if cand:
            t = max(cand, key=len)[:140]
    return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
                author=_txt(author)[:80], published=str(published or ""), source=src.name,
                extra=ex)


# ---------------- rss ----------------
def _arxiv_api(src, limit):
    """arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
    m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
    cat = m.group(1) if m else "cs.AI"
    url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
           "&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
    r = net.get(url, headers=UA)
    if r.status != 200 or "<entry" not in (r.text or ""):
        return []
    out = []
    try:
        root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
    except Exception:
        return []
    ns = {"a": "http://www.w3.org/2005/Atom"}
    for e in root.findall(".//a:entry", ns)[:limit]:
        title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
        summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
        link = e.findtext("a:id", namespaces=ns) or ""
        pub = e.findtext("a:published", namespaces=ns) or ""
        auth = e.findtext("a:author/a:name", namespaces=ns) or ""
        if title:
            out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
                             {"cat": cat}))
    return out


_FEED_CACHE = None


def _cache_url(src):
    global _FEED_CACHE
    if _FEED_CACHE is None:
        try:
            import pathlib as _pl
            f = _pl.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
            _FEED_CACHE = __import__("json").loads(f.read_text(encoding="utf-8")) if f.exists() else {}
        except Exception:
            _FEED_CACHE = {}
    hit = _FEED_CACHE.get(src.name)
    return hit.get("url") if isinstance(hit, dict) and hit.get("url") else src.url


def _rss(src, limit):
    r = net.get(_cache_url(src), headers=UA)
    if r.status in (429, 503):
        import time
        time.sleep(2.5)
        r = net.get(src.url, headers=UA)
    if r.status != 200 or not r.text.strip():
        return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
    out = []
    try:
        root = ET.fromstring(r.text.encode("utf-8", "ignore"))
    except Exception:
        return _rss_regex(src, r.text, limit)
    ns = {"a": "http://www.w3.org/2005/Atom"}
    entries = root.findall(".//item") or root.findall(".//a:entry", ns)
    for e in entries[:limit]:
        g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
        link = g("link")
        if not link:
            le = e.find("a:link", ns) or e.find("link")
            link = le.get("href") if le is not None and le.get("href") else ""
        body = g("description") or g("summary") or g("content")
        out.append(_item(src, "entry", link, g("title"), body,
                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
    if not out and "arxiv.org" in src.url:
        return _arxiv_api(src, limit)
    return out


def _rss_regex(src, text, limit):
    out = []
    for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
        blob = m.group(1)
        g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
        out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
        if len(out) >= limit:
            break
    return out


# ---------------- json adapters ----------------
def _hn_top(src, limit):
    ids = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for sid in ids[:limit]:
        d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
        if not d.get("title"):
            continue
        out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
                         d.get("title"), d.get("text", ""), d.get("by", ""),
                         str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
                                                  "hn": f"https://news.ycombinator.com/item?id={sid}"}))
    return out


def _reddit_listing(src, limit):
    """Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
    sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
    rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
    import time
    children = []
    if rss_first:
        rr = net.get(rss_first, headers=UA)
        if rr.status == 200 and "<entry" in (rr.text or ""):
            return _reddit_atom(src, rr.text, limit)
        if rr.status in (429, 503):
            time.sleep(4)
            rr = net.get(rss_first, headers=UA)
            if rr.status == 200 and "<entry" in (rr.text or ""):
                return _reddit_atom(src, rr.text, limit)
    r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
    out = []
    if r.status == 200 and isinstance(r.json(), dict):
        children = (((r.json() or {}).get("data") or {}).get("children") or [])
    for ch in children[:limit]:
        d = ch.get("data") or {}
        if not d.get("title"):
            continue
        out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
                         d.get("title"), d.get("selftext", ""), d.get("author", ""),
                         str(d.get("created_utc", "")),
                         {"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
    if out:
        return out

    rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
    r2 = net.get(rss_url, headers=UA) if not rss_first else None
    if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
        return _reddit_atom(src, r2.text, limit)

    sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
    if sub:
        alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
                      f"&size={limit}&sort=desc", headers=UA, json_mode=True)
        for d in ((alt.json() or {}).get("data") or [])[:limit]:
            if d.get("title"):
                out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
                                 d.get("title"), d.get("selftext", ""), d.get("author", ""),
                                 str(d.get("created_utc", "")), {"ups": d.get("score")}))
    return out


def _reddit_atom(src, text, limit):
    out = []
    try:
        root = ET.fromstring(text.encode("utf-8", "ignore"))
    except Exception:
        return out
    ns = {"a": "http://www.w3.org/2005/Atom"}
    for e in root.findall(".//a:entry", ns)[:limit]:
        title = e.findtext("a:title", namespaces=ns) or ""
        link_el = e.find("a:link", ns)
        link = link_el.get("href") if link_el is not None else ""
        content = e.findtext("a:content", namespaces=ns) or ""
        author = e.findtext("a:author/a:name", namespaces=ns) or ""
        upd = e.findtext("a:updated", namespaces=ns) or ""
        if title:
            out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
    return out


def _hf_generic(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
                                          or d.get("spaces") or d.get("datasets") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        if "repoData" in x:  # trending: {repoType, repoData:{...}}
            x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
        rid = x.get("id") or x.get("repoId") or x.get("name") or ""
        if not rid:
            continue
        kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
        url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
        out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
                         "", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
    return out


def _hf_models(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
                         "", "", x.get("createdAt", ""),
                         {"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
    return out


def _hf_daily_papers(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        p = x.get("paper") or x
        pid = p.get("id") or ""
        out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
                         p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
                         p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
    return out


def _hf_rows(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for row in (d.get("rows") or [])[:limit]:
        r = row.get("row") or {}
        name = r.get("fullname") or r.get("model") or r.get("Model") or ""
        if not name:
            continue
        out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
                         "", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
    return out


def _openalex(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for w in (d.get("results") or [])[:limit]:
        out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
                         w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
                         {"cited": w.get("cited_by_count")}))
    return out


def _s2(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    out = []
    for p in (d.get("data") or [])[:limit]:
        out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
                         ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
                         str(p.get("year", "")), {"citations": p.get("citationCount")}))
    return out


def _samsung(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
                         x.get("title", ""), x.get("summary") or x.get("content", ""),
                         "", str(x.get("regDate") or x.get("date") or ""), {}))
    return out


def _lg_ai(src, limit):
    d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
    rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
    out = []
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
        out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
                         x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
    return out


def _github_org(src, limit):
    hdr = dict(UA)
    tok = config.KEYS.get("GITHUB_TOKEN")
    if tok:
        hdr["Authorization"] = f"token {tok}"
    url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
    d = net.get(url, headers=hdr, json_mode=True).json() or []
    out = []
    for x in (d if isinstance(d, list) else [])[:limit]:
        out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
                         x.get("description", ""), x.get("owner", {}).get("login", ""),
                         x.get("pushed_at", ""),
                         {"stars": x.get("stargazers_count"), "lang": x.get("language")}))
    return out


def _discourse(src, limit):
    d = net.get(src.url, headers=UA, json_mode=True).json() or {}
    base = src.url.split("/latest.json")[0]
    out = []
    for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
        out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
                         t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
                         "", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
    return out


def _xenforo_latest(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    out = []
    for a in soup.select("a[href*='/threads/']"):
        title = a.get_text(" ", strip=True)
        href = str(a.get("href", "") or "")
        if len(title) < 12 or href.rstrip("/").endswith("/threads"):
            continue
        if not href.startswith("http"):
                    parts = urllib.parse.urlsplit(src.url)
                    href = "%s://%s%s" % (parts.scheme, parts.netloc, href)
        out.append(_item(src, "thread", href, title))
        if len(out) >= limit:
            break
    return out


SERVICE_RE = re.compile(
    r"^(channel (created|photo|name|description|username|deleted)|pinned|"
    r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)


def _tg_web(src, limit):
    """Публичное веб-превью t.me/s/<канал>.

    Отсекаем служебное (Channel created / photo updated / pinned) и пустыши:
    раньше в базу уезжали заголовки "@upenbullet" и "Channel photo removed"."""
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    out = []
    for msg in soup.select("div.tgme_widget_message_wrap")[-max(limit * 2, 20):]:
        if msg.select_one("div.tgme_widget_message_service"):
            continue
        text_el = msg.select_one("div.tgme_widget_message_text")
        if not text_el:
            continue
        text = text_el.get_text("\n", strip=True)
        text = re.sub(r"\n{2,}", "\n", text).strip()
        # лив-каналы пишут коротко («<домен> 2M users»), им порог ниже
        min_len = 12 if "leak" in src.tags else 25
        min_alpha = 8 if "leak" in src.tags else 15
        if len(text) < min_len:
            continue
        if SERVICE_RE.match(text):
            continue
        if sum(ch.isalpha() for ch in text) < min_alpha:   # только ссылки/эмодзи/цифры
            continue
        link_el = msg.select_one("a.tgme_widget_message_date")
        link = str(link_el.get("href") or src.url) if link_el else src.url
        dt = ""
        t_el = msg.select_one("time[datetime]")
        if t_el:
            dt = str(t_el.get("datetime") or "")
        views = ""
        v_el = msg.select_one("span.tgme_widget_message_views")
        if v_el:
            views = v_el.get_text(" ", strip=True)
        via = ""
        vv = msg.select_one("span.tgme_widget_message_via")
        if vv:
            via = vv.get_text(" ", strip=True)
        lines = [ln.strip() for ln in text.split("\n")
                 if ln.strip() and not ln.strip().startswith(("http", "t.co/"))]
        lines = [ln for ln in lines if sum(ch.isalpha() for ch in ln) >= 12]
        if not lines:
            continue
        first = [ln for ln in lines[:6] if len(ln) >= 20]
        head = (first[0] if first else max(lines[:6], key=len)).strip(' ,.;:-——')
        out.append(_item(src, "post", link, head[:140], text, via, dt[:10],
                         {"views": views, "handle": src.url.rsplit("/", 1)[-1], "lang": ""}))
        if len(out) >= limit:
            break
    return out


def _html_listing(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status != 200:
        return []
    soup = BeautifulSoup(r.text, "lxml")
    inc = src.extra.get("include_prefixes") or []
    inc_c = src.extra.get("include_contains") or []
    exc = set(src.extra.get("exclude_exact") or [])
    base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
    seen, links = set(), []
    for a in soup.find_all("a", href=True):
        href = urllib.parse.urljoin(base, str(a["href"]))
        path = urllib.parse.urlsplit(href).path
        if path in exc or len(path) < 6:
            continue
        if inc and not any(path.startswith(p) for p in inc):
            continue
        if inc_c and not any(c in path for c in inc_c):
            continue
        if href in seen:
            continue
        seen.add(href)
        links.append((href, a.get_text(" ", strip=True)))
        if len(links) >= limit:
            break
    out = []
    for href, title in links:
        text = ""
        if src.detail:
            d = net.get(href, headers=UA)
            if d.status == 200:
                ds = BeautifulSoup(d.text, "lxml")
                for bad in ds(["script", "style", "nav", "footer", "header"]):
                    bad.decompose()
                paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
                text = " ".join(paras)[:4000]
                if not title:
                    t = ds.find("h1") or ds.title
                    title = t.get_text(" ", strip=True) if t else href
        out.append(_item(src, "entry", href, title or href, text))
    return out


def _json_of(url, timeout=35):
    """JSON без зависимостей от конкретного клиента: net.get + json.loads."""
    try:
        r = net.get(url, json_mode=True, timeout=timeout)
    except Exception:
        return None
    if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
        return None
    try:
        return json.loads(r.text)
    except Exception:
        return None


def _generic_json(src, limit):
    """Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам."""
    d = _json_of(src.url)
    rows = []
    if isinstance(d, list):
        rows = d
    elif isinstance(d, dict):
        for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"):
            v = d.get(key)
            if isinstance(v, list):
                rows = v
                break
        if not rows:
            for v in d.values():
                if isinstance(v, list) and v and isinstance(v[0], dict):
                    rows = v
                    break
    out = []
    for r in rows[:limit]:
        if not isinstance(r, dict):
            continue
        title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id")
        url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id")
        if isinstance(url, str) and url and not url.startswith("http"):
            if "github.com" in src.url:
                url = "https://github.com/" + url.lstrip("/")
            elif "huggingface.co" in src.url:
                url = "https://huggingface.co/" + url.lstrip("/")
        text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or ""
        if isinstance(text, dict):
            text = json.dumps(text, ensure_ascii=False)[:400]
        published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or ""
        if not title:
            continue
        out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800],
                         str(r.get("author") or r.get("owner") or ""), str(published)[:24]))
    return out


def _or_models(src, limit):
    d = (_json_of(src.url) or {}).get("data", [])
    out = []
    for m in d[:limit]:
        mid = m.get("id") or ""
        pricing = m.get("pricing") or {}
        price = None
        try:
            price = round(float(pricing.get("prompt", 0)) * 1_000_000, 3)
        except Exception:
            price = None
        name = m.get("name") or mid
        ctx = m.get("context_length") or 0
        out.append(_item(src, "model", f"https://openrouter.ai/models/{mid}", f"{name}",
                         f"context {ctx} · prompt ${price}/1M", "", str(m.get("created") or ""),
                         extra={"price": price}))
    return out


def _cisa_kev(src, limit):
    d = _json_of(src.url, timeout=45) or {}
    vulns = d.get("vulnerabilities", [])
    vulns = sorted(vulns, key=lambda v: v.get("dateAdded") or "", reverse=True)
    out = []
    for v in vulns[:limit]:
        cve = v.get("cveID") or ""
        out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cve}",
                         f"{cve} · {v.get('vendorProject','')} {v.get('product','')}".strip(),
                         (v.get("shortDescription") or "")[:600],
                         v.get("vendorProject") or "", v.get("dateAdded") or ""))
    return out


def _nvd(src, limit):
    d = _json_of(src.url, timeout=45) or {}
    out = []
    for v in d.get("vulnerabilities", [])[:limit]:
        c = v.get("cve", {})
        cid = c.get("id") or ""
        desc = ""
        for dd in c.get("descriptions", []):
            if dd.get("lang") == "en":
                desc = dd.get("value") or ""
                break
        sev = ""
        metrics = c.get("metrics", {})
        for key in ("cvssMetricV31", "cvssMetricV40", "cvssMetricV2"):
            if metrics.get(key):
                try:
                    sev = str(metrics[key][0]["cvssData"].get("baseScore"))
                except Exception:
                    sev = ""
                break
        out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cid}",
                         f"{cid}{(' · CVSS ' + sev) if sev else ''}", desc[:600],
                         "", c.get("published") or ""))
    return out



def _biorxiv(src, limit):
    import json as _j
    d = _json_of(src.url)
    rows = d.get("collection") if isinstance(d, dict) else d
    out = []
    for r in (rows or [])[:limit]:
        title = r.get("title") or ""
        doi = r.get("doi") or ""
        if not title:
            continue
        ver = r.get("version") or "1"
        host = "medrxiv" if "medrxiv" in src.url else "biorxiv"
        url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url
        out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800],
                         str(r.get("authors") or ""), str(r.get("date") or "")[:24]))
    return out


def _dblp(src, limit):
    d = _json_of(src.url)
    hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or []
    out = []
    for h in hits[:limit]:
        info = h.get("info") or {}
        title = info.get("title") or ""
        if not title:
            continue
        auth = info.get("authors") or {}
        if isinstance(auth, dict):
            a = auth.get("author")
            if isinstance(a, list):
                auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a)
            elif isinstance(a, dict):
                auth = a.get("text", "")
        out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""),
                         str(auth or ""), str(info.get("year") or "")))
    return out


def _hal(src, limit):
    d = _json_of(src.url)
    docs = ((d or {}).get("response") or {}).get("docs") or []
    out = []
    for r in docs[:limit]:
        t = r.get("title_s")
        title = (t[0] if isinstance(t, list) and t else t) or ""
        if not title:
            continue
        ab = r.get("abstract_s")
        if isinstance(ab, list):
            ab = ab[0] if ab else ""
        out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "",
                         str(r.get("producedDateY_i") or "")))
    return out


def _zenodo(src, limit):
    d = _json_of(src.url)
    rows = ((d or {}).get("hits") or {}).get("hits") or []
    out = []
    for r in rows[:limit]:
        md = r.get("metadata") or {}
        title = md.get("title") or ""
        if not title:
            continue
        url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or ""
        out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800],
                         str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""),
                         str(md.get("publication_date") or "")))
    return out


def _cve_circl(src, limit):
    d = _json_of(src.url)
    rows = d if isinstance(d, list) else []
    out = []
    for r in rows[:limit]:
        cid = r.get("id") or ""
        if not cid:
            continue
        out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140],
                         str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24]))
    return out


def _ghsa(src, limit):
    d = _json_of(src.url)
    rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else []
    out = []
    for r in rows[:limit]:
        gid = r.get("ghsa_id") or ""
        title = r.get("summary") or gid
        if not title:
            continue
        sev = r.get("severity") or ""
        out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title),
                         (sev + " | " + (r.get("description") or ""))[:800], "",
                         str(r.get("published_at") or "")[:24]))
    return out


def _models_dev(src, limit):
    d = _json_of(src.url)
    out = []
    if not isinstance(d, dict):
        return out
    for pid, prov in d.items():
        if not isinstance(prov, dict):
            continue
        pname = prov.get("name") or pid
        models = prov.get("models") or {}
        if isinstance(models, dict):
            iterable = models.items()
        elif isinstance(models, list):
            iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)]
        else:
            continue
        for mid, m in iterable:
            if not isinstance(m, dict):
                continue
            name = m.get("name") or mid
            out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name),
                             "ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"),
                                                            (m.get("cost") or {}).get("input", "?"),
                                                            (m.get("cost") or {}).get("output", "?")),
                             str(pid), ""))
            if len(out) >= limit:
                return out
    return out


def _x_user(src, limit=12):
    """Лента X-аккаунта через api.fxtwitter.com (без ключей)."""
    from .providers import x as xp
    handle = (src.extra or {}).get("handle") or src.url.rsplit("/", 1)[-1]
    return xp.profile(handle, limit=max(4, min(limit, 25)))


ADAPTERS = {
    "x_user": _x_user,
    "rss": _rss,
    "generic_json": _generic_json,
    "or_models": _or_models,
    "cisa_kev": _cisa_kev,
    "nvd": _nvd,
    "hn_top": _hn_top,
    "reddit_listing": _reddit_listing,
    "hf_trending": _hf_generic,
    "hf_generic": _hf_generic,
    "hf_models": _hf_models,
    "hf_daily_papers": _hf_daily_papers,
    "hf_rows": _hf_rows,
    "openalex": _openalex,
    "s2": _s2,
    "samsung": _samsung,
    "lg_ai": _lg_ai,
    "github_org": _github_org,
    "discourse": _discourse,
    "xenforo_latest": _xenforo_latest,
    "tg_web": _tg_web,
    "html_listing": _html_listing,
    "biorxiv": _biorxiv,
    "dblp": _dblp,
    "hal": _hal,
    "zenodo": _zenodo,
    "cve_circl": _cve_circl,
    "ghsa": _ghsa,
    "models_dev": _models_dev,

}


def fetch(src, limit=None):
    fn = ADAPTERS.get(src.adapter)
    if not fn:
        return []
    n = limit or src.limit
    try:
        return fn(src, n) or []
    except Exception:
        return []


SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")


def fetch_many(sources, limit=None, workers=8):
    """Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
    import time
    from concurrent.futures import ThreadPoolExecutor
    slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
    fast = [s for s in sources if s not in slow]
    items, report = [], []
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [(ex.submit(fetch, s, limit), s) for s in fast]
        for f, s in futs:
            got = f.result()
            report.append((s.name, len(got)))
            items.extend(got)
    for s in slow:
        got = fetch(s, limit)
        if not got and "reddit.com" in s.url:
            time.sleep(7)
            got = fetch(s, limit)
        report.append((s.name, len(got)))
        items.extend(got)
        time.sleep(3.5 if "reddit.com" in s.url else 1.0)
    return items, report