reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
2b9f4e1d88c2edd0ed0db72ce600797b0cd84dcf
omni-parse/omni/sources.py
ENI 2b9f4e1d88
Some checks failed
ci / test (3.10) (push) Has been cancelled
Details
ci / test (3.11) (push) Has been cancelled
Details
ci / test (3.12) (push) Has been cancelled
Details
дарк-форумы + cookie-проход CF + opt_in радара в обходе 
- omni/sources_dark.py: cracked.st, nulledbb, forum.tuts4you — реально отдают темы без логина
- tools/cf_cookies.py: Camoufox снимает cookies с JS-челленджей в data/cookies.json (вне git)
- config: cookies.json подхватывается автоматически
- feeds: include_contains матчится по полному href (showthread.php?tid= не резался по path)
- sources_abuse/abuse_sources: A(extra=...) + tuts4you на html_listing (/topic/)
- sources: _gen_sources() = китайские + дарк наборы
- README: ключи/радар/китайцы/дарк
2026-10-26 23:39:47 +03:00

320 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Реестр источников: один список на весь парсер.
Типы:
rss — RSS/Atom лента
json — JSON API/листинг
html — страница-список, ссылки вытаскиваются по префиксу, тело тянется с детальной
github — org/user repos через api.github.com (без токена, лимит 60/час)
Категории: papers, models, community, company, company_cn, company_kr, security, benchmark, forum.
"""
from dataclasses import dataclass, field
from .sources_ext import EXTRA
from .sources_ext2 import EXTRA2
from .sources_ext3 import EXTRA3
from .sources_abuse import ABUSE_LIVE
from .abuse_sources import ABUSE as ABUSE_DEEP
from .sources_tg_abuse import TG_ABUSE
from .sources_ext4 import EXTRA5
from .sources_ext5 import EXTRA6
from .sources_x import X_SOURCES
@dataclass
class Source:
name: str
category: str
kind: str
url: str
adapter: str = "rss"
tags: tuple = ()
limit: int = 20
detail: bool = False
extra: dict = field(default_factory=dict)
S = Source
SOURCES_BASE = [
# ---------- papers ----------
S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")),
S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")),
S("arxiv_cs_cl", "papers", "rss", "https://rss.arxiv.org/rss/cs.CL", "rss", ("paper", "arxiv")),
S("arxiv_cs_cv", "papers", "rss", "https://rss.arxiv.org/rss/cs.CV", "rss", ("paper", "arxiv")),
S("arxiv_cs_ro", "papers", "rss", "https://rss.arxiv.org/rss/cs.RO", "rss", ("paper", "arxiv")),
S("arxiv_cs_ir", "papers", "rss", "https://rss.arxiv.org/rss/cs.IR", "rss", ("paper", "arxiv")),
S("arxiv_cs_cr", "papers", "rss", "https://rss.arxiv.org/rss/cs.CR", "rss", ("paper", "arxiv", "security")),
S("arxiv_cs_se", "papers", "rss", "https://rss.arxiv.org/rss/cs.SE", "rss", ("paper", "arxiv")),
S("arxiv_cs_ne", "papers", "rss", "https://rss.arxiv.org/rss/cs.NE", "rss", ("paper", "arxiv")),
S("arxiv_cs_ma", "papers", "rss", "https://rss.arxiv.org/rss/cs.MA", "rss", ("paper", "arxiv")),
S("arxiv_cs_dc", "papers", "rss", "https://rss.arxiv.org/rss/cs.DC", "rss", ("paper", "arxiv")),
S("arxiv_cs_db", "papers", "rss", "https://rss.arxiv.org/rss/cs.DB", "rss", ("paper", "arxiv")),
S("arxiv_cs_hc", "papers", "rss", "https://rss.arxiv.org/rss/cs.HC", "rss", ("paper", "arxiv")),
S("arxiv_stat_ml", "papers", "rss", "https://rss.arxiv.org/rss/stat.ML", "rss", ("paper", "arxiv")),
S("arxiv_eess_as", "papers", "rss", "https://rss.arxiv.org/rss/eess.AS", "rss", ("paper", "arxiv", "audio")),
S("hf_daily_papers", "papers", "json", "https://huggingface.co/api/daily_papers", "hf_daily_papers", ("paper", "hf")),
S("openalex_recent", "papers", "json",
"https://api.openalex.org/works?filter=from_publication_date:2026-01-01&sort=publication_date:desc&per_page=20",
"openalex", ("paper", "openalex")),
S("semanticscholar_recent", "papers", "json",
"https://api.semanticscholar.org/graph/v1/paper/search?query=large+language+model&limit=20&fields=title,abstract,url,venue,year,authors,citationCount",
"s2", ("paper", "semanticscholar")),
# ---------- models ----------
S("hf_trending_models", "models", "json", "https://huggingface.co/api/trending?type=model", "hf_trending", ("model", "hf")),
S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")),
S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")),
S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")),
S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"),
detail=False, extra={"include_contains": ["/library/"]}),
# ---------- benchmark ----------
S("open_llm_leaderboard", "benchmark", "json",
"https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=0&length=25",
"hf_rows", ("benchmark", "leaderboard")),
S("lmarena_overview", "benchmark", "html", "https://lmarena.ai/leaderboard", "html_listing", ("benchmark", "arena")),
# ---------- community ----------
S("hn_top", "community", "json", "https://hacker-news.firebaseio.com/v0/topstories.json", "hn_top", ("hn",), limit=20),
S("hn_new", "community", "json", "https://hacker-news.firebaseio.com/v0/newstories.json", "hn_top", ("hn",), limit=15),
S("lobsters", "community", "rss", "https://lobste.rs/rss", "rss", ("lobsters",)),
S("lobsters_ai", "community", "rss", "https://lobste.rs/t/ai.rss", "rss", ("lobsters", "ai")),
S("lobsters_security", "community", "rss", "https://lobste.rs/t/security.rss", "rss", ("lobsters", "security")),
S("geeknews", "community", "rss", "https://news.hada.io/rss/news", "rss", ("kr",)),
S("reddit_machinelearning", "community", "json", "https://www.reddit.com/r/MachineLearning/.json?limit=25", "reddit_listing", ("reddit", "ml")),
S("reddit_localllama", "community", "json", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", "reddit_listing", ("reddit", "local")),
S("reddit_openai", "community", "json", "https://www.reddit.com/r/OpenAI/.json?limit=25", "reddit_listing", ("reddit",)),
S("reddit_singularity", "community", "json", "https://www.reddit.com/r/singularity/.json?limit=25", "reddit_listing", ("reddit",)),
S("reddit_stablediffusion", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25", "reddit_listing", ("reddit", "image")),
S("reddit_netsec", "community", "json", "https://www.reddit.com/r/netsec/.json?limit=25", "reddit_listing", ("reddit", "security")),
S("reddit_cybersecurity", "community", "json", "https://www.reddit.com/r/cybersecurity/.json?limit=25", "reddit_listing", ("reddit", "security")),
S("reddit_webscraping", "community", "json", "https://www.reddit.com/r/webscraping/.json?limit=25", "reddit_listing", ("reddit", "scraping")),
S("reddit_selfhosted", "community", "json", "https://www.reddit.com/r/selfhosted/.json?limit=25", "reddit_listing", ("reddit", "selfhosted")),
S("reddit_python", "community", "json", "https://www.reddit.com/r/Python/.json?limit=25", "reddit_listing", ("reddit", "python")),
S("reddit_artificial", "community", "json", "https://www.reddit.com/r/artificial/.json?limit=25", "reddit_listing", ("reddit", "ai")),
# ---------- company ----------
S("openai_news", "company", "rss", "https://openai.com/news/rss.xml", "rss", ("openai",)),
S("anthropic_news", "company", "html", "https://www.anthropic.com/news", "html_listing", ("anthropic",), detail=True,
extra={"include_prefixes": ["/news/"]}),
S("deepmind_blog", "company", "html", "https://deepmind.google/blog/", "html_listing", ("deepmind",), detail=True,
extra={"include_contains": ["/blog/"], "exclude_exact": ["/blog", "/blog/"]}),
S("google_ai_blog", "company", "rss", "https://blog.google/technology/ai/rss/", "rss", ("google",)),
S("google_research_blog", "company", "rss", "https://research.google/blog/rss/", "rss", ("google",)),
S("microsoft_research", "company", "rss", "https://www.microsoft.com/en-us/research/feed/", "rss", ("microsoft",)),
S("nvidia_press", "company", "rss", "https://nvidianews.nvidia.com/cats/press_release.xml", "rss", ("nvidia",)),
S("nvidia_research_ai", "company", "html",
"https://research.nvidia.com/research-area/machine-learning-artificial-intelligence", "html_listing",
("nvidia",), detail=True, extra={"include_prefixes": ["/publication/"]}),
S("apple_ml", "company", "rss", "https://machinelearning.apple.com/rss.xml", "rss", ("apple",)),
S("amazon_science", "company", "rss", "https://www.amazon.science/index.rss", "rss", ("amazon",)),
S("hf_blog", "company", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("hf",)),
S("meta_ai_blog", "company", "rss", "https://ai.meta.com/blog/rss/", "rss", ("meta",)),
S("mistral_news", "company", "html", "https://mistral.ai/news/", "html_listing", ("mistral",), detail=True,
extra={"include_prefixes": ["/news/"], "exclude_exact": ["/news", "/news/"]}),
S("cohere_blog", "company", "rss", "https://cohere.com/blog/rss.xml", "rss", ("cohere",)),
S("together_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)),
S("perplexity_blog", "company", "rss", "https://www.perplexity.ai/hub/blog/rss.xml", "rss", ("perplexity",)),
S("groq_newsroom", "company", "html", "https://groq.com/newsroom", "html_listing", ("groq",), detail=True,
extra={"include_prefixes": ["/newsroom/"], "exclude_exact": ["/newsroom"]}),
S("cerebras_blog", "company", "rss", "https://www.cerebras.ai/blog/rss.xml", "rss", ("cerebras",)),
S("stability_news", "company", "html", "https://stability.ai/news-updates", "html_listing", ("stability",), detail=True,
extra={"include_prefixes": ["/news-updates/"], "exclude_exact": ["/news-updates"]}),
S("databricks_blog", "company", "rss", "https://www.databricks.com/feed", "rss", ("databricks",)),
S("anyscale_blog", "company", "rss", "https://www.anyscale.com/blog/rss.xml", "rss", ("anyscale",)),
S("modal_blog", "company", "rss", "https://modal.com/blog/feed.xml", "rss", ("modal",)),
S("wandb_blog", "company", "rss", "https://wandb.ai/fully-connected/rss.xml", "rss", ("wandb",)),
S("langchain_blog", "company", "rss", "https://blog.langchain.dev/rss/", "rss", ("langchain",)),
S("llamaindex_blog", "company", "rss", "https://www.llamaindex.ai/blog/rss.xml", "rss", ("llamaindex",)),
S("vllm_blog", "company", "rss", "https://blog.vllm.ai/rss.xml", "rss", ("vllm",)),
S("replicate_blog", "company", "rss", "https://replicate.com/blog/rss.xml", "rss", ("replicate",)),
S("fireworks_blog", "company", "rss", "https://fireworks.ai/blog/rss.xml", "rss", ("fireworks",)),
S("sambanova_blog", "company", "rss", "https://sambanova.ai/blog/rss.xml", "rss", ("sambanova",)),
S("scale_blog", "company", "rss", "https://scale.com/blog/rss.xml", "rss", ("scale",)),
S("xai_news", "company", "html", "https://x.ai/news", "html_listing", ("xai",), detail=True,
extra={"include_prefixes": ["/news/"]}),
S("salesforce_ai", "company", "rss", "https://www.salesforce.com/blog/category/ai-research/feed/", "rss", ("salesforce",)),
S("aws_ml_blog", "company", "rss", "https://aws.amazon.com/blogs/machine-learning/feed/", "rss", ("aws",)),
S("azure_ai_blog", "company", "rss", "https://devblogs.microsoft.com/azure-ai/feed/", "rss", ("azure",)),
# ---------- company_cn ----------
S("qwen_blog", "company_cn", "rss", "https://qwenlm.github.io/blog/index.xml", "rss", ("qwen",)),
S("deepseek_updates", "company_cn", "html", "https://api-docs.deepseek.com/updates/", "html_listing",
("deepseek",), detail=True, extra={"include_prefixes": ["/news/"]}),
S("moonshot_blog", "company_cn", "rss", "https://moonshotai.github.io/rss.xml", "rss", ("moonshot", "kimi")),
S("zhipu_blog", "company_cn", "rss", "https://zhipuai.cn/rss.xml", "rss", ("zhipu", "glm")),
S("baichuan_news", "company_cn", "rss", "https://www.baichuan-ai.com/rss.xml", "rss", ("baichuan",)),
S("minimax_blog", "company_cn", "rss", "https://www.minimaxi.com/rss.xml", "rss", ("minimax",)),
S("github_tencent_hunyuan", "company_cn", "github", "Tencent-Hunyuan", "github_org", ("tencent", "cn")),
S("github_paddlepaddle", "company_cn", "github", "PaddlePaddle", "github_org", ("paddlepaddle", "cn")),
S("github_bytedance", "company_cn", "github", "bytedance", "github_org", ("bytedance", "cn")),
S("github_mindspore", "company_cn", "github", "mindspore-ai", "github_org", ("mindspore", "cn")),
S("github_qwenlm", "company_cn", "github", "QwenLM", "github_org", ("qwen", "cn")),
S("github_deepseek", "company_cn", "github", "deepseek-ai", "github_org", ("deepseek", "cn")),
S("github_moonshot", "company_cn", "github", "MoonshotAI", "github_org", ("moonshot", "cn")),
S("github_zhipu", "company_cn", "github", "THUDM", "github_org", ("zhipu", "cn")),
# ---------- company_kr ----------
S("kakao_tech", "company_kr", "rss", "https://tech.kakao.com/feed/", "rss", ("kakao", "kr")),
S("naver_cloud_blog", "company_kr", "rss", "https://rss.blog.naver.com/n_cloudplatform.xml", "rss", ("naver", "kr")),
S("samsung_research", "company_kr", "json", "https://research.samsung.com/blogMain/list.json", "samsung", ("samsung", "kr")),
S("lg_ai_research", "company_kr", "json", "https://www.lgresearch.ai/api/blog/list?page=1&size=20", "lg_ai", ("lg", "kr")),
# ---------- security ----------
S("hackernews_security", "security", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("news",)),
S("bleepingcomputer", "security", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("news",)),
S("krebs", "security", "rss", "https://krebsonsecurity.com/feed/", "rss", ("news",)),
S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed_full.xml", "rss", ("news",)),
S("packetstorm", "security", "rss", "https://packetstormsecurity.com/rss.xml", "rss", ("tools",)),
S("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
S("exploitdb", "security", "rss", "https://www.exploit-db.com/rss.xml", "rss", ("exploit",)),
S("projectzero", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default", "rss", ("research",)),
S("portswigger_research", "security", "rss", "https://portswigger.net/research/rss", "rss", ("research",)),
S("github_security_blog", "security", "rss", "https://github.blog/tag/security/feed/", "rss", ("github",)),
# ---------- forum ----------
S("hn_security_discourse", "forum", "json", "https://community.openai.com/latest.json", "discourse", ("openai",)),
S("discourse_huggingface", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)),
S("discourse_python", "forum", "json", "https://discuss.python.org/latest.json", "discourse", ("python",)),
S("discourse_cloudflare", "forum", "json", "https://community.cloudflare.com/latest.json", "discourse", ("cloudflare",)),
S("discourse_elastic", "forum", "json", "https://discuss.elastic.co/latest.json", "discourse", ("elastic",)),
S("discourse_gitlab", "forum", "json", "https://forum.gitlab.com/latest.json", "discourse", ("gitlab",)),
S("xenforo_bhw", "forum", "html", "https://www.blackhatworld.com/forums/", "xenforo_latest", ("im", "seo")),
S("tg_openai", "forum", "html", "https://t.me/s/openai", "tg_web", ("telegram",)),
S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
]
# ---------- свои каналы (проверены 26.10.26) ----------
EXTRA4 = [
{"name": "tg_alstack", "category": "community", "kind": "html",
"url": "https://t.me/s/AlStack", "adapter": "tg_web", "tags": ["telegram", "own"], "limit": 20},
{"name": "tg_abuz_ai_soft_posts", "category": "abuse", "kind": "html",
"url": "https://t.me/s/Abuz_ai_soft", "adapter": "tg_web", "tags": ["telegram", "abuse", "own"], "limit": 20},
]
def _dedup(rows):
seen, out = set(), []
for s in rows:
if s.url in seen:
continue
seen.add(s.url)
out.append(s)
return out
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES])
BY_NAME = {s.name: s for s in SOURCES}
CATEGORIES = sorted({s.category for s in SOURCES})
_RADAR = {"at": 0.0, "rows": []}
def radar_sources():
"""Источники из abuse-radar: data/radar_sources.json (см. omni radar --apply).
Домены идут html_listing (root страница с маркерами), каналы — tg_web.
Файл перечитывается при изменении mtime, реестр не разрастается дублями.
"""
import json
from pathlib import Path
p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
try:
mtime = p.stat().st_mtime
except OSError:
return []
if _RADAR["rows"] and _RADAR["at"] == mtime:
return _RADAR["rows"]
rows = []
try:
d = json.loads(p.read_text(encoding="utf-8"))
except Exception:
return []
for item in d.get("domains") or []:
dom, score, title = (list(item) + ["", 0, ""])[:3] if isinstance(item, list) else (item.get("domain"), item.get("score"), item.get("title"))
name = "rad_" + str(dom).replace(".", "_").replace("-", "_")
rows.append(Source(name=name, category="abuse", kind="site", url="https://" + str(dom),
adapter="html_listing", tags=("radar", "abuse"),
limit=12, detail=False, extra={"score": score, "title": (title or "")[:120], "opt_in": True}))
for ch in d.get("channels") or []:
name = "tg_" + str(ch).lstrip("@")
rows.append(Source(name=name, category="abuse", kind="tg", url="https://t.me/" + str(ch).lstrip("@"),
adapter="tg_web", tags=("tg", "abuse", "radar"), limit=25,
detail=False, extra={"opt_in": True}))
_RADAR.update(at=mtime, rows=rows)
return rows
def _gen_sources():
"""Сгенерированные наборы: китайские (sources_cn) + дарк-форумы (sources_dark)."""
out = []
for mod, attr in (("sources_cn", "CN"), ("sources_dark", "DARK")):
try:
rows = getattr(__import__("omni." + mod, fromlist=[attr]), attr)
except Exception:
continue
for d in rows:
try:
out.append(Source(**d))
except Exception:
continue
return out
def all_sources():
"""Реестр + пользовательские источники из панели, без дублей по url."""
from . import source_state
extra = []
for d in source_state.custom():
if d["name"] in BY_NAME:
continue
d = {k: v for k, v in d.items() if k in
("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
extra.append(Source(**d))
seen, out = set(), []
for s in SOURCES + _gen_sources() + extra + radar_sources():
key = (s.url or "").rstrip("/").lower()
if key and key in seen:
continue
seen.add(key)
out.append(s)
return out
def enabled_sources(include_opt_in=False):
"""Только включённые: выключенные в панели источники не дёргаем.
Источники из abuse-radar помечены extra["opt_in"] — их 2000+, в общий обход
они не лезут; берутся только когда явно просят (--tags radar / --only / category abuse).
"""
from . import source_state
fl = source_state.flags()
out = []
for s in all_sources():
if not fl.get(s.name, {}).get("enabled", True):
continue
if not include_opt_in and (s.extra or {}).get("opt_in"):
continue
out.append(s)
return out
def pick(category=None, only=None, tags=None, respect_enabled=True):
explicit = bool(only or tags or category)
out = enabled_sources(include_opt_in=explicit) if respect_enabled else all_sources()
if category:
cats = {c.strip() for c in category.split(",") if c.strip()}
out = [s for s in out if s.category in cats]
if only:
names = {n.strip() for n in only.split(",") if n.strip()}
out = [s for s in out if s.name in names]
if tags:
wanted = {t.strip() for t in tags.split(",") if t.strip()}
out = [s for s in out if wanted & set(s.tags)]
return out
def stats(use_all=True):
out = {}
for s in (all_sources() if use_all else SOURCES):
out.setdefault(s.category, []).append(s.name)
return {k: len(v) for k, v in sorted(out.items())}
Reference in New Issue View Git Blame Copy Permalink