"""Реестр источников: один список на весь парсер.

Типы:
  rss    — RSS/Atom лента
  json   — JSON API/листинг
  html   — страница-список, ссылки вытаскиваются по префиксу, тело тянется с детальной
  github — org/user repos через api.github.com (без токена, лимит 60/час)

Категории: papers, models, community, company, company_cn, company_kr, security, benchmark, forum.
"""
from dataclasses import dataclass, field

from .sources_ext import EXTRA
from .sources_ext2 import EXTRA2
from .sources_ext3 import EXTRA3
from .sources_abuse import ABUSE_LIVE
from .abuse_sources import ABUSE as ABUSE_DEEP
from .sources_tg_abuse import TG_ABUSE
from .sources_ext4 import EXTRA5
from .sources_ext5 import EXTRA6
from .sources_fresh import fresh as _fresh_sources
from .sources_x import X_SOURCES


@dataclass
class Source:
    name: str
    category: str
    kind: str
    url: str
    adapter: str = "rss"
    tags: tuple = ()
    limit: int = 20
    detail: bool = False
    extra: dict = field(default_factory=dict)


S = Source

SOURCES_BASE = [
    # ---------- papers ----------
    S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")),
    S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")),
    S("arxiv_cs_cl", "papers", "rss", "https://rss.arxiv.org/rss/cs.CL", "rss", ("paper", "arxiv")),
    S("arxiv_cs_cv", "papers", "rss", "https://rss.arxiv.org/rss/cs.CV", "rss", ("paper", "arxiv")),
    S("arxiv_cs_ro", "papers", "rss", "https://rss.arxiv.org/rss/cs.RO", "rss", ("paper", "arxiv")),
    S("arxiv_cs_ir", "papers", "rss", "https://rss.arxiv.org/rss/cs.IR", "rss", ("paper", "arxiv")),
    S("arxiv_cs_cr", "papers", "rss", "https://rss.arxiv.org/rss/cs.CR", "rss", ("paper", "arxiv", "security")),
    S("arxiv_cs_se", "papers", "rss", "https://rss.arxiv.org/rss/cs.SE", "rss", ("paper", "arxiv")),
    S("arxiv_cs_ne", "papers", "rss", "https://rss.arxiv.org/rss/cs.NE", "rss", ("paper", "arxiv")),
    S("arxiv_cs_ma", "papers", "rss", "https://rss.arxiv.org/rss/cs.MA", "rss", ("paper", "arxiv")),
    S("arxiv_cs_dc", "papers", "rss", "https://rss.arxiv.org/rss/cs.DC", "rss", ("paper", "arxiv")),
    S("arxiv_cs_db", "papers", "rss", "https://rss.arxiv.org/rss/cs.DB", "rss", ("paper", "arxiv")),
    S("arxiv_cs_hc", "papers", "rss", "https://rss.arxiv.org/rss/cs.HC", "rss", ("paper", "arxiv")),
    S("arxiv_stat_ml", "papers", "rss", "https://rss.arxiv.org/rss/stat.ML", "rss", ("paper", "arxiv")),
    S("arxiv_eess_as", "papers", "rss", "https://rss.arxiv.org/rss/eess.AS", "rss", ("paper", "arxiv", "audio")),
    S("hf_daily_papers", "papers", "json", "https://huggingface.co/api/daily_papers", "hf_daily_papers", ("paper", "hf")),
    S("openalex_recent", "papers", "json",
      "https://api.openalex.org/works?filter=from_publication_date:2026-01-01&sort=publication_date:desc&per_page=20",
      "openalex", ("paper", "openalex")),
    S("semanticscholar_recent", "papers", "json",
      "https://api.semanticscholar.org/graph/v1/paper/search?query=large+language+model&limit=20&fields=title,abstract,url,venue,year,authors,citationCount",
      "s2", ("paper", "semanticscholar")),

    # ---------- models ----------
    S("hf_trending_models", "models", "json", "https://huggingface.co/api/trending?type=model", "hf_trending", ("model", "hf")),
    S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")),
    S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")),
    S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")),
    S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"),
      detail=False, extra={"include_contains": ["/library/"]}),

    # ---------- benchmark ----------
    S("open_llm_leaderboard", "benchmark", "json",
      "https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=0&length=25",
      "hf_rows", ("benchmark", "leaderboard")),
    S("lmarena_overview", "benchmark", "html", "https://lmarena.ai/leaderboard", "html_listing", ("benchmark", "arena")),

    # ---------- community ----------
    S("hn_top", "community", "json", "https://hacker-news.firebaseio.com/v0/topstories.json", "hn_top", ("hn",), limit=20),
    S("hn_new", "community", "json", "https://hacker-news.firebaseio.com/v0/newstories.json", "hn_top", ("hn",), limit=15),
    S("lobsters", "community", "rss", "https://lobste.rs/rss", "rss", ("lobsters",)),
    S("lobsters_ai", "community", "rss", "https://lobste.rs/t/ai.rss", "rss", ("lobsters", "ai")),
    S("lobsters_security", "community", "rss", "https://lobste.rs/t/security.rss", "rss", ("lobsters", "security")),
    S("geeknews", "community", "rss", "https://news.hada.io/rss/news", "rss", ("kr",)),
    S("reddit_machinelearning", "community", "json", "https://www.reddit.com/r/MachineLearning/.json?limit=25", "reddit_listing", ("reddit", "ml")),
    S("reddit_localllama", "community", "json", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", "reddit_listing", ("reddit", "local")),
    S("reddit_openai", "community", "json", "https://www.reddit.com/r/OpenAI/.json?limit=25", "reddit_listing", ("reddit",)),
    S("reddit_singularity", "community", "json", "https://www.reddit.com/r/singularity/.json?limit=25", "reddit_listing", ("reddit",)),
    S("reddit_stablediffusion", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25", "reddit_listing", ("reddit", "image")),
    S("reddit_netsec", "community", "json", "https://www.reddit.com/r/netsec/.json?limit=25", "reddit_listing", ("reddit", "security")),
    S("reddit_cybersecurity", "community", "json", "https://www.reddit.com/r/cybersecurity/.json?limit=25", "reddit_listing", ("reddit", "security")),
    S("reddit_webscraping", "community", "json", "https://www.reddit.com/r/webscraping/.json?limit=25", "reddit_listing", ("reddit", "scraping")),
    S("reddit_selfhosted", "community", "json", "https://www.reddit.com/r/selfhosted/.json?limit=25", "reddit_listing", ("reddit", "selfhosted")),
    S("reddit_python", "community", "json", "https://www.reddit.com/r/Python/.json?limit=25", "reddit_listing", ("reddit", "python")),
    S("reddit_artificial", "community", "json", "https://www.reddit.com/r/artificial/.json?limit=25", "reddit_listing", ("reddit", "ai")),

    # ---------- company ----------
    S("openai_news", "company", "rss", "https://openai.com/news/rss.xml", "rss", ("openai",)),
    S("anthropic_news", "company", "html", "https://www.anthropic.com/news", "html_listing", ("anthropic",), detail=True,
      extra={"include_prefixes": ["/news/"]}),
    S("deepmind_blog", "company", "html", "https://deepmind.google/blog/", "html_listing", ("deepmind",), detail=True,
      extra={"include_contains": ["/blog/"], "exclude_exact": ["/blog", "/blog/"]}),
    S("google_ai_blog", "company", "rss", "https://blog.google/technology/ai/rss/", "rss", ("google",)),
    S("google_research_blog", "company", "rss", "https://research.google/blog/rss/", "rss", ("google",)),
    S("microsoft_research", "company", "rss", "https://www.microsoft.com/en-us/research/feed/", "rss", ("microsoft",)),
    S("nvidia_press", "company", "rss", "https://nvidianews.nvidia.com/cats/press_release.xml", "rss", ("nvidia",)),
    S("nvidia_research_ai", "company", "html",
      "https://research.nvidia.com/research-area/machine-learning-artificial-intelligence", "html_listing",
      ("nvidia",), detail=True, extra={"include_prefixes": ["/publication/"]}),
    S("apple_ml", "company", "rss", "https://machinelearning.apple.com/rss.xml", "rss", ("apple",)),
    S("amazon_science", "company", "rss", "https://www.amazon.science/index.rss", "rss", ("amazon",)),
    S("hf_blog", "company", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("hf",)),
    S("meta_ai_blog", "company", "rss", "https://ai.meta.com/blog/rss/", "rss", ("meta",)),
    S("mistral_news", "company", "html", "https://mistral.ai/news/", "html_listing", ("mistral",), detail=True,
      extra={"include_prefixes": ["/news/"], "exclude_exact": ["/news", "/news/"]}),
    S("cohere_blog", "company", "rss", "https://cohere.com/blog/rss.xml", "rss", ("cohere",)),
    S("together_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)),
    S("perplexity_blog", "company", "rss", "https://www.perplexity.ai/hub/blog/rss.xml", "rss", ("perplexity",)),
    S("groq_newsroom", "company", "html", "https://groq.com/newsroom", "html_listing", ("groq",), detail=True,
      extra={"include_prefixes": ["/newsroom/"], "exclude_exact": ["/newsroom"]}),
    S("cerebras_blog", "company", "rss", "https://www.cerebras.ai/blog/rss.xml", "rss", ("cerebras",)),
    S("stability_news", "company", "html", "https://stability.ai/news-updates", "html_listing", ("stability",), detail=True,
      extra={"include_prefixes": ["/news-updates/"], "exclude_exact": ["/news-updates"]}),
    S("databricks_blog", "company", "rss", "https://www.databricks.com/feed", "rss", ("databricks",)),
    S("anyscale_blog", "company", "rss", "https://www.anyscale.com/blog/rss.xml", "rss", ("anyscale",)),
    S("modal_blog", "company", "rss", "https://modal.com/blog/feed.xml", "rss", ("modal",)),
    S("wandb_blog", "company", "rss", "https://wandb.ai/fully-connected/rss.xml", "rss", ("wandb",)),
    S("langchain_blog", "company", "rss", "https://blog.langchain.dev/rss/", "rss", ("langchain",)),
    S("llamaindex_blog", "company", "rss", "https://www.llamaindex.ai/blog/rss.xml", "rss", ("llamaindex",)),
    S("vllm_blog", "company", "rss", "https://blog.vllm.ai/rss.xml", "rss", ("vllm",)),
    S("replicate_blog", "company", "rss", "https://replicate.com/blog/rss.xml", "rss", ("replicate",)),
    S("fireworks_blog", "company", "rss", "https://fireworks.ai/blog/rss.xml", "rss", ("fireworks",)),
    S("sambanova_blog", "company", "rss", "https://sambanova.ai/blog/rss.xml", "rss", ("sambanova",)),
    S("scale_blog", "company", "rss", "https://scale.com/blog/rss.xml", "rss", ("scale",)),
    S("xai_news", "company", "html", "https://x.ai/news", "html_listing", ("xai",), detail=True,
      extra={"include_prefixes": ["/news/"]}),
    S("salesforce_ai", "company", "rss", "https://www.salesforce.com/blog/category/ai-research/feed/", "rss", ("salesforce",)),
    S("aws_ml_blog", "company", "rss", "https://aws.amazon.com/blogs/machine-learning/feed/", "rss", ("aws",)),
    S("azure_ai_blog", "company", "rss", "https://devblogs.microsoft.com/azure-ai/feed/", "rss", ("azure",)),

    # ---------- company_cn ----------
    S("qwen_blog", "company_cn", "rss", "https://qwenlm.github.io/blog/index.xml", "rss", ("qwen",)),
    S("deepseek_updates", "company_cn", "html", "https://api-docs.deepseek.com/updates/", "html_listing",
      ("deepseek",), detail=True, extra={"include_prefixes": ["/news/"]}),
    S("moonshot_blog", "company_cn", "rss", "https://moonshotai.github.io/rss.xml", "rss", ("moonshot", "kimi")),
    S("zhipu_blog", "company_cn", "rss", "https://zhipuai.cn/rss.xml", "rss", ("zhipu", "glm")),
    S("baichuan_news", "company_cn", "rss", "https://www.baichuan-ai.com/rss.xml", "rss", ("baichuan",)),
    S("minimax_blog", "company_cn", "rss", "https://www.minimaxi.com/rss.xml", "rss", ("minimax",)),
    S("github_tencent_hunyuan", "company_cn", "github", "Tencent-Hunyuan", "github_org", ("tencent", "cn")),
    S("github_paddlepaddle", "company_cn", "github", "PaddlePaddle", "github_org", ("paddlepaddle", "cn")),
    S("github_bytedance", "company_cn", "github", "bytedance", "github_org", ("bytedance", "cn")),
    S("github_mindspore", "company_cn", "github", "mindspore-ai", "github_org", ("mindspore", "cn")),
    S("github_qwenlm", "company_cn", "github", "QwenLM", "github_org", ("qwen", "cn")),
    S("github_deepseek", "company_cn", "github", "deepseek-ai", "github_org", ("deepseek", "cn")),
    S("github_moonshot", "company_cn", "github", "MoonshotAI", "github_org", ("moonshot", "cn")),
    S("github_zhipu", "company_cn", "github", "THUDM", "github_org", ("zhipu", "cn")),

    # ---------- company_kr ----------
    S("kakao_tech", "company_kr", "rss", "https://tech.kakao.com/feed/", "rss", ("kakao", "kr")),
    S("naver_cloud_blog", "company_kr", "rss", "https://rss.blog.naver.com/n_cloudplatform.xml", "rss", ("naver", "kr")),
    S("samsung_research", "company_kr", "json", "https://research.samsung.com/blogMain/list.json", "samsung", ("samsung", "kr")),
    S("lg_ai_research", "company_kr", "json", "https://www.lgresearch.ai/api/blog/list?page=1&size=20", "lg_ai", ("lg", "kr")),

    # ---------- security ----------
    S("hackernews_security", "security", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("news",)),
    S("bleepingcomputer", "security", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("news",)),
    S("krebs", "security", "rss", "https://krebsonsecurity.com/feed/", "rss", ("news",)),
    S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed_full.xml", "rss", ("news",)),
    S("packetstorm", "security", "rss", "https://packetstormsecurity.com/rss.xml", "rss", ("tools",)),
    S("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
    S("exploitdb", "security", "rss", "https://www.exploit-db.com/rss.xml", "rss", ("exploit",)),
    S("projectzero", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default", "rss", ("research",)),
    S("portswigger_research", "security", "rss", "https://portswigger.net/research/rss", "rss", ("research",)),
    S("github_security_blog", "security", "rss", "https://github.blog/tag/security/feed/", "rss", ("github",)),

    # ---------- forum ----------
    S("hn_security_discourse", "forum", "json", "https://community.openai.com/latest.json", "discourse", ("openai",)),
    S("discourse_huggingface", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)),
    S("discourse_python", "forum", "json", "https://discuss.python.org/latest.json", "discourse", ("python",)),
    S("discourse_cloudflare", "forum", "json", "https://community.cloudflare.com/latest.json", "discourse", ("cloudflare",)),
    S("discourse_elastic", "forum", "json", "https://discuss.elastic.co/latest.json", "discourse", ("elastic",)),
    S("discourse_gitlab", "forum", "json", "https://forum.gitlab.com/latest.json", "discourse", ("gitlab",)),
    S("xenforo_bhw", "forum", "html", "https://www.blackhatworld.com/forums/", "xenforo_latest", ("im", "seo")),
    S("tg_openai", "forum", "html", "https://t.me/s/openai", "tg_web", ("telegram",)),
    S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
]
# ---------- свои каналы (проверены 26.10.26) ----------
EXTRA4 = [
    {"name": "tg_alstack", "category": "community", "kind": "html",
     "url": "https://t.me/s/AlStack", "adapter": "tg_web", "tags": ["telegram", "own"], "limit": 20},
    {"name": "tg_abuz_ai_soft_posts", "category": "abuse", "kind": "html",
     "url": "https://t.me/s/Abuz_ai_soft", "adapter": "tg_web", "tags": ["telegram", "abuse", "own"], "limit": 20},
]

def _dedup(rows):
    seen, out = set(), []
    for s in rows:
        if s.url in seen:
            continue
        seen.add(s.url)
        out.append(s)
    return out


SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES])

BY_NAME = {s.name: s for s in SOURCES}
CATEGORIES = sorted({s.category for s in SOURCES})

_RADAR = {"at": 0.0, "rows": []}


def radar_sources():
    """Источники из abuse-radar: data/radar_sources.json (см. omni radar --apply).

    Домены идут html_listing (root страница с маркерами), каналы — tg_web.
    Файл перечитывается при изменении mtime, реестр не разрастается дублями.
    """
    import json
    from pathlib import Path
    p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
    try:
        mtime = p.stat().st_mtime
    except OSError:
        return []
    if _RADAR["rows"] and _RADAR["at"] == mtime:
        return _RADAR["rows"]
    rows = []
    try:
        d = json.loads(p.read_text(encoding="utf-8"))
    except Exception:
        return []
    for item in d.get("domains") or []:
        dom, score, title = (list(item) + ["", 0, ""])[:3] if isinstance(item, list) else (item.get("domain"), item.get("score"), item.get("title"))
        name = "rad_" + str(dom).replace(".", "_").replace("-", "_")
        rows.append(Source(name=name, category="abuse", kind="site", url="https://" + str(dom),
                           adapter="html_listing", tags=("radar", "abuse"),
                           limit=12, detail=False, extra={"score": score, "title": (title or "")[:120], "opt_in": True}))
    for ch in d.get("channels") or []:
        name = "tg_" + str(ch).lstrip("@")
        rows.append(Source(name=name, category="abuse", kind="tg", url="https://t.me/" + str(ch).lstrip("@"),
                           adapter="tg_web", tags=("tg", "abuse", "radar"), limit=25,
                           detail=False, extra={"opt_in": True}))
    _RADAR.update(at=mtime, rows=rows)
    return rows


_PANELS = {"rows": [], "at": 0.0}


def panel_sources():
    """Панели new-api из data/panels.json (tools/panel_hunt.py --apply).

    Адаптер newapi_pricing тянет /api/pricing и отдаёт находку: сколько моделей
    на панели бесплатны. Это не статьи про абузы, а сами раздачи API.
    """
    import json
    from pathlib import Path
    p = Path(__file__).resolve().parents[1] / "data" / "panels.json"
    try:
        mtime = p.stat().st_mtime
    except OSError:
        return []
    if _PANELS["rows"] and _PANELS["at"] == mtime:
        return _PANELS["rows"]
    try:
        d = json.loads(p.read_text(encoding="utf-8"))
    except Exception:
        return []
    rows = []
    for r in d.get("panels") or []:
        host = str(r.get("host") or "").strip()
        if not host:
            continue
        name = "pn_" + host.replace(".", "_").replace("-", "_")
        rows.append(Source(name=name, category="abuse", kind="site", url="https://" + host,
                           adapter="newapi_pricing", tags=("panel", "abuse", "freeapi"),
                           limit=1, detail=False,
                           extra={"free_count": r.get("free_count") or 0, "models": r.get("models") or 0}))
    _PANELS.update(at=mtime, rows=rows)
    return rows


def _gen_sources():
    """Сгенерированные наборы: китайские (sources_cn) + дарк-форумы (sources_dark)."""
    out = []
    for mod, attr in (("sources_cn", "CN"), ("sources_dark", "DARK")):
        try:
            rows = getattr(__import__("omni." + mod, fromlist=[attr]), attr)
        except Exception:
            continue
        for d in rows:
            try:
                out.append(Source(**d))
            except Exception:
                continue
    return out


def all_sources():
    """Реестр + пользовательские источники из панели, без дублей по url и по имени.

    Дедуп по двум ключам обязателен:
      * url — одна площадка не должна обходиться дважды (двойные записи в базе);
      * name — имя источника является первичным ключом в source_flags/runs, поэтому
        зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
        www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна →
    пользовательские → abuse-radar → panels.json.
    """
    from . import source_state
    extra = []
    for d in source_state.custom():
        if d["name"] in BY_NAME:
            continue
        d = {k: v for k, v in d.items() if k in
             ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
        extra.append(Source(**d))
    seen_url, seen_name, out = set(), set(), []
    for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources():
        key = (s.url or "").rstrip("/").lower()
        name = (s.name or "").strip().lower()
        if (key and key in seen_url) or (name and name in seen_name):
            continue
        if key:
            seen_url.add(key)
        if name:
            seen_name.add(name)
        out.append(s)
    return out


def enabled_sources(include_opt_in=False):
    """Только включённые: выключенные в панели источники не дёргаем.

    Источники из abuse-radar помечены extra["opt_in"] — их 2000+, в общий обход
    они не лезут; берутся только когда явно просят (--tags radar / --only / category abuse).
    """
    from . import source_state
    fl = source_state.flags()
    out = []
    for s in all_sources():
        if not fl.get(s.name, {}).get("enabled", True):
            continue
        if not include_opt_in and (s.extra or {}).get("opt_in"):
            continue
        out.append(s)
    return out


def pick(category=None, only=None, tags=None, respect_enabled=True):
    explicit = bool(only or tags or category)
    out = enabled_sources(include_opt_in=explicit) if respect_enabled else all_sources()
    if category:
        cats = {c.strip() for c in category.split(",") if c.strip()}
        out = [s for s in out if s.category in cats]
    if only:
        names = {n.strip() for n in only.split(",") if n.strip()}
        out = [s for s in out if s.name in names]
    if tags:
        wanted = {t.strip() for t in tags.split(",") if t.strip()}
        out = [s for s in out if wanted & set(s.tags)]
    return out


def stats(use_all=True):
    out = {}
    for s in (all_sources() if use_all else SOURCES):
        out.setdefault(s.category, []).append(s.name)
    return {k: len(v) for k, v in sorted(out.items())}