391 lines
24 KiB
Python
391 lines
24 KiB
Python
"""Реестр источников: один список на весь парсер.
|
||
|
||
Типы:
|
||
rss — RSS/Atom лента
|
||
json — JSON API/листинг
|
||
html — страница-список, ссылки вытаскиваются по префиксу, тело тянется с детальной
|
||
github — org/user repos через api.github.com (без токена, лимит 60/час)
|
||
|
||
Категории: papers, models, community, company, company_cn, company_kr, security, benchmark, forum.
|
||
"""
|
||
from dataclasses import dataclass, field
|
||
|
||
from .sources_ext import EXTRA
|
||
from .sources_ext2 import EXTRA2
|
||
from .sources_ext3 import EXTRA3
|
||
from .sources_abuse import ABUSE_LIVE
|
||
from .abuse_sources import ABUSE as ABUSE_DEEP
|
||
from .sources_tg_abuse import TG_ABUSE
|
||
from .sources_ext4 import EXTRA5
|
||
from .sources_ext5 import EXTRA6
|
||
from .sources_fresh import fresh as _fresh_sources
|
||
from .sources_quality import quality_sources as _quality_sources
|
||
from .sources_x import X_SOURCES
|
||
from .sources_auto import AUTO as AUTO_SOURCES
|
||
|
||
|
||
@dataclass
|
||
class Source:
|
||
name: str
|
||
category: str
|
||
kind: str
|
||
url: str
|
||
adapter: str = "rss"
|
||
tags: tuple = ()
|
||
limit: int = 20
|
||
detail: bool = False
|
||
extra: dict = field(default_factory=dict)
|
||
|
||
|
||
S = Source
|
||
|
||
SOURCES_BASE = [
|
||
# ---------- papers ----------
|
||
S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_cl", "papers", "rss", "https://rss.arxiv.org/rss/cs.CL", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_cv", "papers", "rss", "https://rss.arxiv.org/rss/cs.CV", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_ro", "papers", "rss", "https://rss.arxiv.org/rss/cs.RO", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_ir", "papers", "rss", "https://rss.arxiv.org/rss/cs.IR", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_cr", "papers", "rss", "https://rss.arxiv.org/rss/cs.CR", "rss", ("paper", "arxiv", "security")),
|
||
S("arxiv_cs_se", "papers", "rss", "https://rss.arxiv.org/rss/cs.SE", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_ne", "papers", "rss", "https://rss.arxiv.org/rss/cs.NE", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_ma", "papers", "rss", "https://rss.arxiv.org/rss/cs.MA", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_dc", "papers", "rss", "https://rss.arxiv.org/rss/cs.DC", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_db", "papers", "rss", "https://rss.arxiv.org/rss/cs.DB", "rss", ("paper", "arxiv")),
|
||
S("arxiv_cs_hc", "papers", "rss", "https://rss.arxiv.org/rss/cs.HC", "rss", ("paper", "arxiv")),
|
||
S("arxiv_stat_ml", "papers", "rss", "https://rss.arxiv.org/rss/stat.ML", "rss", ("paper", "arxiv")),
|
||
S("arxiv_eess_as", "papers", "rss", "https://rss.arxiv.org/rss/eess.AS", "rss", ("paper", "arxiv", "audio")),
|
||
S("hf_daily_papers", "papers", "json", "https://huggingface.co/api/daily_papers", "hf_daily_papers", ("paper", "hf")),
|
||
S("openalex_recent", "papers", "json",
|
||
"https://api.openalex.org/works?filter=from_publication_date:2026-01-01&sort=publication_date:desc&per_page=20",
|
||
"openalex", ("paper", "openalex")),
|
||
S("semanticscholar_recent", "papers", "json",
|
||
"https://api.semanticscholar.org/graph/v1/paper/search?query=large+language+model&limit=20&fields=title,abstract,url,venue,year,authors,citationCount",
|
||
"s2", ("paper", "semanticscholar")),
|
||
|
||
# ---------- models ----------
|
||
S("hf_trending_models", "models", "json", "https://huggingface.co/api/trending?type=model", "hf_trending", ("model", "hf")),
|
||
S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")),
|
||
S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")),
|
||
S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")),
|
||
S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"),
|
||
detail=False, extra={"include_contains": ["/library/"]}),
|
||
|
||
# ---------- benchmark ----------
|
||
S("open_llm_leaderboard", "benchmark", "json",
|
||
"https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=0&length=25",
|
||
"hf_rows", ("benchmark", "leaderboard")),
|
||
S("lmarena_overview", "benchmark", "html", "https://lmarena.ai/leaderboard", "html_listing", ("benchmark", "arena")),
|
||
|
||
# ---------- community ----------
|
||
S("hn_top", "community", "json", "https://hacker-news.firebaseio.com/v0/topstories.json", "hn_top", ("hn",), limit=20),
|
||
S("hn_new", "community", "json", "https://hacker-news.firebaseio.com/v0/newstories.json", "hn_top", ("hn",), limit=15),
|
||
S("lobsters", "community", "rss", "https://lobste.rs/rss", "rss", ("lobsters",)),
|
||
S("lobsters_ai", "community", "rss", "https://lobste.rs/t/ai.rss", "rss", ("lobsters", "ai")),
|
||
S("lobsters_security", "community", "rss", "https://lobste.rs/t/security.rss", "rss", ("lobsters", "security")),
|
||
S("geeknews", "community", "rss", "https://news.hada.io/rss/news", "rss", ("kr",)),
|
||
S("reddit_machinelearning", "community", "json", "https://www.reddit.com/r/MachineLearning/.json?limit=25", "reddit_listing", ("reddit", "ml")),
|
||
S("reddit_localllama", "community", "json", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", "reddit_listing", ("reddit", "local")),
|
||
S("reddit_openai", "community", "json", "https://www.reddit.com/r/OpenAI/.json?limit=25", "reddit_listing", ("reddit",)),
|
||
S("reddit_singularity", "community", "json", "https://www.reddit.com/r/singularity/.json?limit=25", "reddit_listing", ("reddit",)),
|
||
S("reddit_stablediffusion", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25", "reddit_listing", ("reddit", "image")),
|
||
S("reddit_netsec", "community", "json", "https://www.reddit.com/r/netsec/.json?limit=25", "reddit_listing", ("reddit", "security")),
|
||
S("reddit_cybersecurity", "community", "json", "https://www.reddit.com/r/cybersecurity/.json?limit=25", "reddit_listing", ("reddit", "security")),
|
||
S("reddit_webscraping", "community", "json", "https://www.reddit.com/r/webscraping/.json?limit=25", "reddit_listing", ("reddit", "scraping")),
|
||
S("reddit_selfhosted", "community", "json", "https://www.reddit.com/r/selfhosted/.json?limit=25", "reddit_listing", ("reddit", "selfhosted")),
|
||
S("reddit_python", "community", "json", "https://www.reddit.com/r/Python/.json?limit=25", "reddit_listing", ("reddit", "python")),
|
||
S("reddit_artificial", "community", "json", "https://www.reddit.com/r/artificial/.json?limit=25", "reddit_listing", ("reddit", "ai")),
|
||
|
||
# ---------- company ----------
|
||
S("openai_news", "company", "rss", "https://openai.com/news/rss.xml", "rss", ("openai",)),
|
||
S("anthropic_news", "company", "html", "https://www.anthropic.com/news", "html_listing", ("anthropic",), detail=True,
|
||
extra={"include_prefixes": ["/news/"]}),
|
||
S("deepmind_blog", "company", "html", "https://deepmind.google/blog/", "html_listing", ("deepmind",), detail=True,
|
||
extra={"include_contains": ["/blog/"], "exclude_exact": ["/blog", "/blog/"]}),
|
||
S("google_ai_blog", "company", "rss", "https://blog.google/technology/ai/rss/", "rss", ("google",)),
|
||
S("google_research_blog", "company", "rss", "https://research.google/blog/rss/", "rss", ("google",)),
|
||
S("microsoft_research", "company", "rss", "https://www.microsoft.com/en-us/research/feed/", "rss", ("microsoft",)),
|
||
S("nvidia_press", "company", "rss", "https://nvidianews.nvidia.com/cats/press_release.xml", "rss", ("nvidia",)),
|
||
S("nvidia_research_ai", "company", "html",
|
||
"https://research.nvidia.com/research-area/machine-learning-artificial-intelligence", "html_listing",
|
||
("nvidia",), detail=True, extra={"include_prefixes": ["/publication/"]}),
|
||
S("apple_ml", "company", "rss", "https://machinelearning.apple.com/rss.xml", "rss", ("apple",)),
|
||
S("amazon_science", "company", "rss", "https://www.amazon.science/index.rss", "rss", ("amazon",)),
|
||
S("hf_blog", "company", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("hf",)),
|
||
S("meta_ai_blog", "company", "rss", "https://ai.meta.com/blog/rss/", "rss", ("meta",)),
|
||
S("mistral_news", "company", "html", "https://mistral.ai/news/", "html_listing", ("mistral",), detail=True,
|
||
extra={"include_prefixes": ["/news/"], "exclude_exact": ["/news", "/news/"]}),
|
||
S("cohere_blog", "company", "rss", "https://cohere.com/blog/rss.xml", "rss", ("cohere",)),
|
||
S("together_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)),
|
||
S("perplexity_blog", "company", "rss", "https://www.perplexity.ai/hub/blog/rss.xml", "rss", ("perplexity",)),
|
||
S("groq_newsroom", "company", "html", "https://groq.com/newsroom", "html_listing", ("groq",), detail=True,
|
||
extra={"include_prefixes": ["/newsroom/"], "exclude_exact": ["/newsroom"]}),
|
||
S("cerebras_blog", "company", "rss", "https://www.cerebras.ai/blog/rss.xml", "rss", ("cerebras",)),
|
||
S("stability_news", "company", "html", "https://stability.ai/news-updates", "html_listing", ("stability",), detail=True,
|
||
extra={"include_prefixes": ["/news-updates/"], "exclude_exact": ["/news-updates"]}),
|
||
S("databricks_blog", "company", "rss", "https://www.databricks.com/feed", "rss", ("databricks",)),
|
||
S("anyscale_blog", "company", "rss", "https://www.anyscale.com/blog/rss.xml", "rss", ("anyscale",)),
|
||
S("modal_blog", "company", "rss", "https://modal.com/blog/feed.xml", "rss", ("modal",)),
|
||
S("wandb_blog", "company", "rss", "https://wandb.ai/fully-connected/rss.xml", "rss", ("wandb",)),
|
||
S("langchain_blog", "company", "rss", "https://blog.langchain.dev/rss/", "rss", ("langchain",)),
|
||
S("llamaindex_blog", "company", "rss", "https://www.llamaindex.ai/blog/rss.xml", "rss", ("llamaindex",)),
|
||
S("vllm_blog", "company", "rss", "https://blog.vllm.ai/rss.xml", "rss", ("vllm",)),
|
||
S("replicate_blog", "company", "rss", "https://replicate.com/blog/rss.xml", "rss", ("replicate",)),
|
||
S("fireworks_blog", "company", "rss", "https://fireworks.ai/blog/rss.xml", "rss", ("fireworks",)),
|
||
S("sambanova_blog", "company", "rss", "https://sambanova.ai/blog/rss.xml", "rss", ("sambanova",)),
|
||
S("scale_blog", "company", "rss", "https://scale.com/blog/rss.xml", "rss", ("scale",)),
|
||
S("xai_news", "company", "html", "https://x.ai/news", "html_listing", ("xai",), detail=True,
|
||
extra={"include_prefixes": ["/news/"]}),
|
||
S("salesforce_ai", "company", "rss", "https://www.salesforce.com/blog/category/ai-research/feed/", "rss", ("salesforce",)),
|
||
S("aws_ml_blog", "company", "rss", "https://aws.amazon.com/blogs/machine-learning/feed/", "rss", ("aws",)),
|
||
S("azure_ai_blog", "company", "rss", "https://devblogs.microsoft.com/azure-ai/feed/", "rss", ("azure",)),
|
||
|
||
# ---------- company_cn ----------
|
||
S("qwen_blog", "company_cn", "rss", "https://qwenlm.github.io/blog/index.xml", "rss", ("qwen",)),
|
||
S("deepseek_updates", "company_cn", "html", "https://api-docs.deepseek.com/updates/", "html_listing",
|
||
("deepseek",), detail=True, extra={"include_prefixes": ["/news/"]}),
|
||
S("moonshot_blog", "company_cn", "rss", "https://moonshotai.github.io/rss.xml", "rss", ("moonshot", "kimi")),
|
||
S("zhipu_blog", "company_cn", "rss", "https://zhipuai.cn/rss.xml", "rss", ("zhipu", "glm")),
|
||
S("baichuan_news", "company_cn", "rss", "https://www.baichuan-ai.com/rss.xml", "rss", ("baichuan",)),
|
||
S("minimax_blog", "company_cn", "rss", "https://www.minimaxi.com/rss.xml", "rss", ("minimax",)),
|
||
S("github_tencent_hunyuan", "company_cn", "github", "Tencent-Hunyuan", "github_org", ("tencent", "cn")),
|
||
S("github_paddlepaddle", "company_cn", "github", "PaddlePaddle", "github_org", ("paddlepaddle", "cn")),
|
||
S("github_bytedance", "company_cn", "github", "bytedance", "github_org", ("bytedance", "cn")),
|
||
S("github_mindspore", "company_cn", "github", "mindspore-ai", "github_org", ("mindspore", "cn")),
|
||
S("github_qwenlm", "company_cn", "github", "QwenLM", "github_org", ("qwen", "cn")),
|
||
S("github_deepseek", "company_cn", "github", "deepseek-ai", "github_org", ("deepseek", "cn")),
|
||
S("github_moonshot", "company_cn", "github", "MoonshotAI", "github_org", ("moonshot", "cn")),
|
||
S("github_zhipu", "company_cn", "github", "THUDM", "github_org", ("zhipu", "cn")),
|
||
|
||
# ---------- company_kr ----------
|
||
S("kakao_tech", "company_kr", "rss", "https://tech.kakao.com/feed/", "rss", ("kakao", "kr")),
|
||
S("naver_cloud_blog", "company_kr", "rss", "https://rss.blog.naver.com/n_cloudplatform.xml", "rss", ("naver", "kr")),
|
||
S("samsung_research", "company_kr", "json", "https://research.samsung.com/blogMain/list.json", "samsung", ("samsung", "kr")),
|
||
S("lg_ai_research", "company_kr", "json", "https://www.lgresearch.ai/api/blog/list?page=1&size=20", "lg_ai", ("lg", "kr")),
|
||
|
||
# ---------- security ----------
|
||
S("hackernews_security", "security", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("news",)),
|
||
S("bleepingcomputer", "security", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("news",)),
|
||
S("krebs", "security", "rss", "https://krebsonsecurity.com/feed/", "rss", ("news",)),
|
||
S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed_full.xml", "rss", ("news",)),
|
||
S("packetstorm", "security", "rss", "https://packetstormsecurity.com/rss.xml", "rss", ("tools",)),
|
||
S("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
|
||
S("exploitdb", "security", "rss", "https://www.exploit-db.com/rss.xml", "rss", ("exploit",)),
|
||
S("projectzero", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default", "rss", ("research",)),
|
||
S("portswigger_research", "security", "rss", "https://portswigger.net/research/rss", "rss", ("research",)),
|
||
S("github_security_blog", "security", "rss", "https://github.blog/tag/security/feed/", "rss", ("github",)),
|
||
|
||
# ---------- forum ----------
|
||
S("hn_security_discourse", "forum", "json", "https://community.openai.com/latest.json", "discourse", ("openai",)),
|
||
S("discourse_huggingface", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)),
|
||
S("discourse_python", "forum", "json", "https://discuss.python.org/latest.json", "discourse", ("python",)),
|
||
S("discourse_cloudflare", "forum", "json", "https://community.cloudflare.com/latest.json", "discourse", ("cloudflare",)),
|
||
S("discourse_elastic", "forum", "json", "https://discuss.elastic.co/latest.json", "discourse", ("elastic",)),
|
||
S("discourse_gitlab", "forum", "json", "https://forum.gitlab.com/latest.json", "discourse", ("gitlab",)),
|
||
S("xenforo_bhw", "forum", "html", "https://www.blackhatworld.com/forums/", "xenforo_latest", ("im", "seo")),
|
||
S("tg_openai", "forum", "html", "https://t.me/s/openai", "tg_web", ("telegram",)),
|
||
S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
|
||
]
|
||
# ---------- свои каналы (проверены 26.10.26) ----------
|
||
EXTRA4 = [
|
||
{"name": "tg_alstack", "category": "community", "kind": "html",
|
||
"url": "https://t.me/s/AlStack", "adapter": "tg_web", "tags": ["telegram", "own"], "limit": 20},
|
||
{"name": "tg_abuz_ai_soft_posts", "category": "abuse", "kind": "html",
|
||
"url": "https://t.me/s/Abuz_ai_soft", "adapter": "tg_web", "tags": ["telegram", "abuse", "own"], "limit": 20},
|
||
]
|
||
|
||
def _dedup(rows):
|
||
seen, out = set(), []
|
||
for s in rows:
|
||
if s.url in seen:
|
||
continue
|
||
seen.add(s.url)
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES] + [Source(**d) for d in AUTO_SOURCES])
|
||
|
||
BY_NAME = {s.name: s for s in SOURCES}
|
||
CATEGORIES = sorted({s.category for s in SOURCES})
|
||
|
||
_RADAR = {"at": 0.0, "rows": []}
|
||
|
||
|
||
def radar_sources():
|
||
"""Источники из abuse-radar: data/radar_sources.json (см. omni radar --apply).
|
||
|
||
Домены идут html_listing (root страница с маркерами), каналы — tg_web.
|
||
Файл перечитывается при изменении mtime, реестр не разрастается дублями.
|
||
"""
|
||
import json
|
||
from pathlib import Path
|
||
p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
|
||
try:
|
||
mtime = p.stat().st_mtime
|
||
except OSError:
|
||
return []
|
||
if _RADAR["rows"] and _RADAR["at"] == mtime:
|
||
return _RADAR["rows"]
|
||
rows = []
|
||
try:
|
||
d = json.loads(p.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return []
|
||
for item in d.get("domains") or []:
|
||
dom, score, title = (list(item) + ["", 0, ""])[:3] if isinstance(item, list) else (item.get("domain"), item.get("score"), item.get("title"))
|
||
name = "rad_" + str(dom).replace(".", "_").replace("-", "_")
|
||
rows.append(Source(name=name, category="abuse", kind="site", url="https://" + str(dom),
|
||
adapter="html_listing", tags=("radar", "abuse"),
|
||
limit=12, detail=False, extra={"score": score, "title": (title or "")[:120], "opt_in": True}))
|
||
for ch in d.get("channels") or []:
|
||
name = "tg_" + str(ch).lstrip("@")
|
||
rows.append(Source(name=name, category="abuse", kind="tg", url="https://t.me/" + str(ch).lstrip("@"),
|
||
adapter="tg_web", tags=("tg", "abuse", "radar"), limit=25,
|
||
detail=False, extra={"opt_in": True}))
|
||
_RADAR.update(at=mtime, rows=rows)
|
||
return rows
|
||
|
||
|
||
_PANELS = {"rows": [], "at": 0.0}
|
||
|
||
|
||
def panel_sources():
|
||
"""Панели new-api из data/panels.json (tools/panel_hunt.py --apply).
|
||
|
||
Адаптер newapi_pricing тянет /api/pricing и отдаёт находку: сколько моделей
|
||
на панели бесплатны. Это не статьи про абузы, а сами раздачи API.
|
||
"""
|
||
import json
|
||
from pathlib import Path
|
||
p = Path(__file__).resolve().parents[1] / "data" / "panels.json"
|
||
try:
|
||
mtime = p.stat().st_mtime
|
||
except OSError:
|
||
return []
|
||
if _PANELS["rows"] and _PANELS["at"] == mtime:
|
||
return _PANELS["rows"]
|
||
try:
|
||
d = json.loads(p.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return []
|
||
rows = []
|
||
for r in d.get("panels") or []:
|
||
host = str(r.get("host") or "").strip()
|
||
if not host:
|
||
continue
|
||
name = "pn_" + host.replace(".", "_").replace("-", "_")
|
||
rows.append(Source(name=name, category="abuse", kind="site", url="https://" + host,
|
||
adapter="newapi_pricing", tags=("panel", "abuse", "freeapi"),
|
||
limit=1, detail=False,
|
||
extra={"free_count": r.get("free_count") or 0, "models": r.get("models") or 0}))
|
||
_PANELS.update(at=mtime, rows=rows)
|
||
return rows
|
||
|
||
|
||
def _gen_sources():
|
||
"""Сгенерированные наборы: китайские (sources_cn) + дарк-форумы (sources_dark)."""
|
||
out = []
|
||
for mod, attr in (("sources_cn", "CN"), ("sources_dark", "DARK")):
|
||
try:
|
||
rows = getattr(__import__("omni." + mod, fromlist=[attr]), attr)
|
||
except Exception:
|
||
continue
|
||
for d in rows:
|
||
try:
|
||
out.append(Source(**d))
|
||
except Exception:
|
||
continue
|
||
return out
|
||
|
||
|
||
def all_sources(include_fresh=True):
|
||
"""Реестр + пользовательские источники из панели, без дублей по url и по имени.
|
||
|
||
Дедуп по двум ключам обязателен:
|
||
* url — одна площадка не должна обходиться дважды (двойные записи в базе);
|
||
* name — имя источника является первичным ключом в source_flags/runs, поэтому
|
||
зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
|
||
www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
|
||
Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая и quality-волны →
|
||
пользовательские → abuse-radar → panels.json.
|
||
|
||
include_fresh=False исключает текущий generated fresh-модуль при регенерации.
|
||
"""
|
||
from . import source_state
|
||
extra = []
|
||
for d in source_state.custom():
|
||
if d["name"] in BY_NAME:
|
||
continue
|
||
d = {k: v for k, v in d.items() if k in
|
||
("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
|
||
extra.append(Source(**d))
|
||
seen_url, seen_name, out = set(), set(), []
|
||
fresh = _fresh_sources(S) if include_fresh else []
|
||
for s in SOURCES + _gen_sources() + fresh + _quality_sources(S) + extra + radar_sources() + panel_sources():
|
||
# Панели — отдельный адаптер (/api/pricing), а не обход сайта: их url не должен
|
||
# схлопываться с rad_*-источником того же домена, иначе теряется 170 из 175 панелей.
|
||
key = (s.url or "").rstrip("/").lower()
|
||
if (s.name or "").startswith("pn_") and key:
|
||
key += "#panel"
|
||
name = (s.name or "").strip().lower()
|
||
if (key and key in seen_url) or (name and name in seen_name):
|
||
continue
|
||
if key:
|
||
seen_url.add(key)
|
||
if name:
|
||
seen_name.add(name)
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def enabled_sources(include_opt_in=False):
|
||
"""Только включённые: выключенные в панели источники не дёргаем.
|
||
|
||
Источники из abuse-radar помечены extra["opt_in"] — их 2000+, в общий обход
|
||
они не лезут; берутся только когда явно просят (--tags radar / --only / category abuse).
|
||
"""
|
||
from . import source_state
|
||
fl = source_state.flags()
|
||
out = []
|
||
for s in all_sources():
|
||
if not fl.get(s.name, {}).get("enabled", True):
|
||
continue
|
||
if not include_opt_in and (s.extra or {}).get("opt_in"):
|
||
continue
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def pick(category=None, only=None, tags=None, respect_enabled=True):
|
||
explicit = bool(only or tags or category)
|
||
out = enabled_sources(include_opt_in=explicit) if respect_enabled else all_sources()
|
||
if category:
|
||
cats = {c.strip() for c in category.split(",") if c.strip()}
|
||
out = [s for s in out if s.category in cats]
|
||
if only:
|
||
names = {n.strip() for n in only.split(",") if n.strip()}
|
||
|
||
def _hit(src):
|
||
if src.name in names:
|
||
return True
|
||
for n in names:
|
||
if n.endswith("*") and src.name.startswith(n[:-1]):
|
||
return True
|
||
if n.endswith("_") and src.name.startswith(n):
|
||
return True
|
||
return False
|
||
|
||
out = [x for x in out if _hit(x)]
|
||
if tags:
|
||
wanted = {t.strip() for t in tags.split(",") if t.strip()}
|
||
out = [s for s in out if wanted & set(s.tags)]
|
||
return out
|
||
|
||
|
||
def stats(use_all=True):
|
||
out = {}
|
||
for s in (all_sources() if use_all else SOURCES):
|
||
out.setdefault(s.category, []).append(s.name)
|
||
return {k: len(v) for k, v in sorted(out.items())} |