From c4d188520f73ed7f43c139b7a6df5c4457e956d9 Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 25 Oct 2026 18:54:33 +0300 Subject: [PATCH] =?UTF-8?q?omni:=20=D1=80=D0=B5=D0=B5=D1=81=D1=82=D1=80=20?= =?UTF-8?q?113=20=D0=B8=D1=81=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=20(papers/company/cn/kr/community/security/forum/models/?= =?UTF-8?q?benchmark)=20+=20=D0=BA=D0=BE=D0=BC=D0=B0=D0=BD=D0=B4=D1=8B=20s?= =?UTF-8?q?ources=20=D0=B8=20feed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 26 +++ omni/cli.py | 45 ++++++ omni/feeds.py | 413 ++++++++++++++++++++++++++++++++++++++++++++++++ omni/sources.py | 199 +++++++++++++++++++++++ 4 files changed, 683 insertions(+) create mode 100644 omni/feeds.py create mode 100644 omni/sources.py diff --git a/README.md b/README.md index 6fd6da0..58a70f0 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ | `firecrawl` | Firecrawl search | `FIRECRAWL_API_KEY` | | `cards` | карточки товаров: JSON-LD Product/Offer, OpenGraph, microdata, цена/валюта/наличие | нет | | `treg` | treg-поиск | `TREG_TOKEN` | +| `feed` | обход реестра из 113 лент/API без запроса (см. ниже) | нет | `python -m omni providers` печатает, что реально поднялось в этом окружении. @@ -39,6 +40,15 @@ python -m omni providers # поиск по одной теме в четырёх бесплатных источниках python -m omni search "scraping twitter without api" --providers reddit,forums,ddg,bing --limit 12 --show 10 +# реестр источников: сколько и каких +python -m omni sources +python -m omni sources --category papers,security --list + +# обход реестра БЕЗ запроса: 113 источников -> база +python -m omni feed --category community --limit 8 +python -m omni feed --limit 15 # всё подряд +python -m omni feed --only arxiv_cs_ai,hn_top,reddit_localllama + # прогон по списку тем из themes.txt (по теме на строку) python -m omni fanout --providers reddit,forums,ddg,bing --limit 10 @@ -54,6 +64,22 @@ python -m omni cards https://example.com/product python -m omni export --format jsonl --out data/dump.jsonl ``` +## Реестр источников + +`omni/sources.py` — 113 источников в 9 категориях, каждый с адаптером: + +- **papers 18** — arxiv RSS по 15 разделам (cs.AI/LG/CL/CV/RO/IR/CR/SE/NE/MA/DC/DB/HC, stat.ML, eess.AS), HF daily papers, OpenAlex, Semantic Scholar +- **company 34** — ленты OpenAI, Google AI/Research, Microsoft Research, NVIDIA press+research, Apple ML, Amazon Science, HF blog, Meta AI, Mistral, Cohere, Together, Perplexity, Groq, Cerebras, Stability, Databricks, Anyscale, Modal, W&B, LangChain, LlamaIndex, vLLM, Replicate, Fireworks, SambaNova, Scale, xAI, Salesforce, AWS ML, Azure AI +- **company_cn 14** — Qwen, DeepSeek changelog, Moonshot, Zhipu, Baichuan, MiniMax + GitHub-орги Tencent-Hunyuan, PaddlePaddle, bytedance, mindspore-ai, QwenLM, deepseek-ai, MoonshotAI, THUDM +- **company_kr 4** — Kakao tech, Naver Cloud, Samsung Research, LG AI Research +- **community 17** — HN top/new, Lobsters (3 тега), GeekNews, 11 сабреддитов +- **security 10** — The Hacker News, BleepingComputer, Krebs, SANS ISC, Packet Storm, CISA, Exploit-DB, Project Zero, PortSwigger, GitHub Security +- **forum 9** — 6 Discourse-инстансов, XenForo BHW, веб-версии Telegram-каналов +- **models 5** — HF trending/new (модели, спейсы, датасеты), Ollama library +- **benchmark 2** — Open LLM Leaderboard (HF datasets-server), LMArena + +Адаптеры: `rss`, `json` (HF, HN, OpenAlex, S2, Discourse), `html` (список + детальная страница), `github` (org repos). Медленные хосты (reddit, github, HF) обходятся последовательно с паузой, остальные — в 8 потоков. + ## Как устроено - `omni/net.py` — лестница транспорта: httpx → curl_cffi (подмена TLS-фингерпринта) → прокси из env. Троттлинг по хосту, лимит байт, пометка блокировок. diff --git a/omni/cli.py b/omni/cli.py index b236963..f82be0b 100644 --- a/omni/cli.py +++ b/omni/cli.py @@ -5,6 +5,7 @@ import json import sys from . import config, rank, store +from . import feeds, sources from . import filter as flt from .providers import FUNCS, MODULES, get, status @@ -146,6 +147,34 @@ def cmd_stats(args): return 0 +def cmd_sources(args): + st = sources.stats() + print("источников:", len(sources.SOURCES), "| по категориям:", + ", ".join(f"{k}={v}" for k, v in st.items())) + if args.list: + for s in sources.pick(args.category, args.only, args.tags): + print(f" {s.category:<12} {s.adapter:<18} {s.name:<26} {s.url[:70]}") + return 0 + + +def cmd_feed(args): + picked = sources.pick(args.category, args.only, args.tags) + if not picked: + print("нет источников под фильтр") + return 1 + print(f"обход {len(picked)} источников, лимит {args.limit} на источник") + items, report = feeds.fetch_many(picked, limit=args.limit, workers=args.workers) + items = rank.rank(items) + new, dup = store.add(items) + bad = [(n, c) for n, c in report if c == 0] + for name, cnt in sorted(report, key=lambda x: -x[1])[:20]: + print(f" {name:<28} {cnt}") + print(f"пусто: {len(bad)}/{len(report)} | собрано {len(items)}, новых {new}, дублей {dup}") + if bad and args.verbose: + print("без отдачи:", ", ".join(n for n, _ in bad)) + return 0 + + def cmd_export(args): n, path = store.export(args.out, args.fmt, args.limit) print(f"выгружено {n} записей -> {path}") @@ -201,6 +230,22 @@ def main(argv=None): sp = sub.add_parser("stats", help="сводка базы") sp.set_defaults(fn=cmd_stats) + sp = sub.add_parser("sources", help="реестр источников") + sp.add_argument("--category", default=None) + sp.add_argument("--only", default=None) + sp.add_argument("--tags", default=None) + sp.add_argument("--list", action="store_true") + sp.set_defaults(fn=cmd_sources) + + sp = sub.add_parser("feed", help="обход реестра источников без запроса") + sp.add_argument("--category", default=None) + sp.add_argument("--only", default=None) + sp.add_argument("--tags", default=None) + sp.add_argument("--limit", type=int, default=15) + sp.add_argument("--workers", type=int, default=8) + sp.add_argument("--verbose", action="store_true") + sp.set_defaults(fn=cmd_feed) + sp = sub.add_parser("export", help="выгрузка jsonl/csv") sp.add_argument("--out", default="data/top.jsonl") sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"]) diff --git a/omni/feeds.py b/omni/feeds.py new file mode 100644 index 0000000..a4e5b18 --- /dev/null +++ b/omni/feeds.py @@ -0,0 +1,413 @@ +"""Забор источников из реестра sources.py. + +Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter. +Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com). +""" +import html +import json +import re +import urllib.parse +import xml.etree.ElementTree as ET + +from bs4 import BeautifulSoup + +from . import config, net +from .model import Item + +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36", + "Accept-Language": "en,ru;q=0.8"} + + +def _txt(x): + if x is None: + return "" + return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip() + + +def _item(src, kind, url, title, text="", author="", published="", extra=None): + ex = dict(extra or {}) + ex["tags"] = list(src.tags) + return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000], + author=_txt(author)[:80], published=str(published or ""), source=src.category, + extra=ex) + + +# ---------------- rss ---------------- +def _rss(src, limit): + r = net.get(src.url, headers=UA) + if r.status in (429, 503): + import time + time.sleep(2.5) + r = net.get(src.url, headers=UA) + if r.status != 200 or not r.text.strip(): + return [] + out = [] + try: + root = ET.fromstring(r.text.encode("utf-8", "ignore")) + except Exception: + return _rss_regex(src, r.text, limit) + ns = {"a": "http://www.w3.org/2005/Atom"} + entries = root.findall(".//item") or root.findall(".//a:entry", ns) + for e in entries[:limit]: + g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "") + link = g("link") + if not link: + le = e.find("a:link", ns) or e.find("link") + link = le.get("href") if le is not None and le.get("href") else "" + body = g("description") or g("summary") or g("content") + out.append(_item(src, "entry", link, g("title"), body, + g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated"))) + return out + + +def _rss_regex(src, text, limit): + out = [] + for m in re.finditer(r"(.*?)", text, re.S | re.I): + blob = m.group(1) + g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)", blob, re.S | re.I) or [None, ""])[1] + out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate"))) + if len(out) >= limit: + break + return out + + +# ---------------- json adapters ---------------- +def _hn_top(src, limit): + ids = net.get(src.url, headers=UA, json_mode=True).json() or [] + out = [] + for sid in ids[:limit]: + d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {} + if not d.get("title"): + continue + out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}", + d.get("title"), d.get("text", ""), d.get("by", ""), + str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"), + "hn": f"https://news.ycombinator.com/item?id={sid}"})) + return out + + +def _reddit_listing(src, limit): + """Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush.""" + sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url) + rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None + import time + children = [] + if rss_first: + rr = net.get(rss_first, headers=UA) + if rr.status == 200 and "= limit: + break + return out + + +def _tg_web(src, limit): + r = net.get(src.url, headers=UA) + if r.status != 200: + return [] + soup = BeautifulSoup(r.text, "lxml") + out = [] + for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]: + text_el = msg.select_one("div.tgme_widget_message_text") + link_el = msg.select_one("a.tgme_widget_message_date") + text = text_el.get_text("\n", strip=True) if text_el else "" + link = str(link_el.get("href") or src.url) if link_el else src.url + if not text: + continue + out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {})) + return out + + +def _html_listing(src, limit): + r = net.get(src.url, headers=UA) + if r.status != 200: + return [] + soup = BeautifulSoup(r.text, "lxml") + inc = src.extra.get("include_prefixes") or [] + inc_c = src.extra.get("include_contains") or [] + exc = set(src.extra.get("exclude_exact") or []) + base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url)) + seen, links = set(), [] + for a in soup.find_all("a", href=True): + href = urllib.parse.urljoin(base, str(a["href"])) + path = urllib.parse.urlsplit(href).path + if path in exc or len(path) < 6: + continue + if inc and not any(path.startswith(p) for p in inc): + continue + if inc_c and not any(c in path for c in inc_c): + continue + if href in seen: + continue + seen.add(href) + links.append((href, a.get_text(" ", strip=True))) + if len(links) >= limit: + break + out = [] + for href, title in links: + text = "" + if src.detail: + d = net.get(href, headers=UA) + if d.status == 200: + ds = BeautifulSoup(d.text, "lxml") + for bad in ds(["script", "style", "nav", "footer", "header"]): + bad.decompose() + paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")] + text = " ".join(paras)[:4000] + if not title: + t = ds.find("h1") or ds.title + title = t.get_text(" ", strip=True) if t else href + out.append(_item(src, "entry", href, title or href, text)) + return out + + +ADAPTERS = { + "rss": _rss, + "hn_top": _hn_top, + "reddit_listing": _reddit_listing, + "hf_trending": _hf_generic, + "hf_generic": _hf_generic, + "hf_models": _hf_models, + "hf_daily_papers": _hf_daily_papers, + "hf_rows": _hf_rows, + "openalex": _openalex, + "s2": _s2, + "samsung": _samsung, + "lg_ai": _lg_ai, + "github_org": _github_org, + "discourse": _discourse, + "xenforo_latest": _xenforo_latest, + "tg_web": _tg_web, + "html_listing": _html_listing, +} + + +def fetch(src, limit=None): + fn = ADAPTERS.get(src.adapter) + if not fn: + return [] + n = limit or src.limit + try: + return fn(src, n) or [] + except Exception: + return [] + + +SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co") + + +def fetch_many(sources, limit=None, workers=8): + """Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой.""" + import time + from concurrent.futures import ThreadPoolExecutor + slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)] + fast = [s for s in sources if s not in slow] + items, report = [], [] + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = [(ex.submit(fetch, s, limit), s) for s in fast] + for f, s in futs: + got = f.result() + report.append((s.name, len(got))) + items.extend(got) + for s in slow: + got = fetch(s, limit) + if not got and "reddit.com" in s.url: + time.sleep(7) + got = fetch(s, limit) + report.append((s.name, len(got))) + items.extend(got) + time.sleep(3.5 if "reddit.com" in s.url else 1.0) + return items, report \ No newline at end of file diff --git a/omni/sources.py b/omni/sources.py new file mode 100644 index 0000000..09cd82b --- /dev/null +++ b/omni/sources.py @@ -0,0 +1,199 @@ +"""Реестр источников: один список на весь парсер. + +Типы: + rss — RSS/Atom лента + json — JSON API/листинг + html — страница-список, ссылки вытаскиваются по префиксу, тело тянется с детальной + github — org/user repos через api.github.com (без токена, лимит 60/час) + +Категории: papers, models, community, company, company_cn, company_kr, security, benchmark, forum. +""" +from dataclasses import dataclass, field + + +@dataclass +class Source: + name: str + category: str + kind: str + url: str + adapter: str = "rss" + tags: tuple = () + limit: int = 20 + detail: bool = False + extra: dict = field(default_factory=dict) + + +S = Source + +SOURCES = [ + # ---------- papers ---------- + S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")), + S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")), + S("arxiv_cs_cl", "papers", "rss", "https://rss.arxiv.org/rss/cs.CL", "rss", ("paper", "arxiv")), + S("arxiv_cs_cv", "papers", "rss", "https://rss.arxiv.org/rss/cs.CV", "rss", ("paper", "arxiv")), + S("arxiv_cs_ro", "papers", "rss", "https://rss.arxiv.org/rss/cs.RO", "rss", ("paper", "arxiv")), + S("arxiv_cs_ir", "papers", "rss", "https://rss.arxiv.org/rss/cs.IR", "rss", ("paper", "arxiv")), + S("arxiv_cs_cr", "papers", "rss", "https://rss.arxiv.org/rss/cs.CR", "rss", ("paper", "arxiv", "security")), + S("arxiv_cs_se", "papers", "rss", "https://rss.arxiv.org/rss/cs.SE", "rss", ("paper", "arxiv")), + S("arxiv_cs_ne", "papers", "rss", "https://rss.arxiv.org/rss/cs.NE", "rss", ("paper", "arxiv")), + S("arxiv_cs_ma", "papers", "rss", "https://rss.arxiv.org/rss/cs.MA", "rss", ("paper", "arxiv")), + S("arxiv_cs_dc", "papers", "rss", "https://rss.arxiv.org/rss/cs.DC", "rss", ("paper", "arxiv")), + S("arxiv_cs_db", "papers", "rss", "https://rss.arxiv.org/rss/cs.DB", "rss", ("paper", "arxiv")), + S("arxiv_cs_hc", "papers", "rss", "https://rss.arxiv.org/rss/cs.HC", "rss", ("paper", "arxiv")), + S("arxiv_stat_ml", "papers", "rss", "https://rss.arxiv.org/rss/stat.ML", "rss", ("paper", "arxiv")), + S("arxiv_eess_as", "papers", "rss", "https://rss.arxiv.org/rss/eess.AS", "rss", ("paper", "arxiv", "audio")), + S("hf_daily_papers", "papers", "json", "https://huggingface.co/api/daily_papers", "hf_daily_papers", ("paper", "hf")), + S("openalex_recent", "papers", "json", + "https://api.openalex.org/works?filter=from_publication_date:2026-01-01&sort=publication_date:desc&per_page=20", + "openalex", ("paper", "openalex")), + S("semanticscholar_recent", "papers", "json", + "https://api.semanticscholar.org/graph/v1/paper/search?query=large+language+model&limit=20&fields=title,abstract,url,venue,year,authors,citationCount", + "s2", ("paper", "semanticscholar")), + + # ---------- models ---------- + S("hf_trending_models", "models", "json", "https://huggingface.co/api/trending?type=model", "hf_trending", ("model", "hf")), + S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")), + S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")), + S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")), + S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"), detail=False), + + # ---------- benchmark ---------- + S("open_llm_leaderboard", "benchmark", "json", + "https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=0&length=25", + "hf_rows", ("benchmark", "leaderboard")), + S("lmarena_overview", "benchmark", "html", "https://lmarena.ai/leaderboard", "html_listing", ("benchmark", "arena")), + + # ---------- community ---------- + S("hn_top", "community", "json", "https://hacker-news.firebaseio.com/v0/topstories.json", "hn_top", ("hn",), limit=20), + S("hn_new", "community", "json", "https://hacker-news.firebaseio.com/v0/newstories.json", "hn_top", ("hn",), limit=15), + S("lobsters", "community", "rss", "https://lobste.rs/rss", "rss", ("lobsters",)), + S("lobsters_ai", "community", "rss", "https://lobste.rs/t/ai.rss", "rss", ("lobsters", "ai")), + S("lobsters_security", "community", "rss", "https://lobste.rs/t/security.rss", "rss", ("lobsters", "security")), + S("geeknews", "community", "rss", "https://news.hada.io/rss/news", "rss", ("kr",)), + S("reddit_machinelearning", "community", "json", "https://www.reddit.com/r/MachineLearning/.json?limit=25", "reddit_listing", ("reddit", "ml")), + S("reddit_localllama", "community", "json", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", "reddit_listing", ("reddit", "local")), + S("reddit_openai", "community", "json", "https://www.reddit.com/r/OpenAI/.json?limit=25", "reddit_listing", ("reddit",)), + S("reddit_singularity", "community", "json", "https://www.reddit.com/r/singularity/.json?limit=25", "reddit_listing", ("reddit",)), + S("reddit_stablediffusion", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25", "reddit_listing", ("reddit", "image")), + S("reddit_netsec", "community", "json", "https://www.reddit.com/r/netsec/.json?limit=25", "reddit_listing", ("reddit", "security")), + S("reddit_cybersecurity", "community", "json", "https://www.reddit.com/r/cybersecurity/.json?limit=25", "reddit_listing", ("reddit", "security")), + S("reddit_webscraping", "community", "json", "https://www.reddit.com/r/webscraping/.json?limit=25", "reddit_listing", ("reddit", "scraping")), + S("reddit_selfhosted", "community", "json", "https://www.reddit.com/r/selfhosted/.json?limit=25", "reddit_listing", ("reddit", "selfhosted")), + S("reddit_python", "community", "json", "https://www.reddit.com/r/Python/.json?limit=25", "reddit_listing", ("reddit", "python")), + S("reddit_artificial", "community", "json", "https://www.reddit.com/r/artificial/.json?limit=25", "reddit_listing", ("reddit", "ai")), + + # ---------- company ---------- + S("openai_news", "company", "rss", "https://openai.com/news/rss.xml", "rss", ("openai",)), + S("anthropic_news", "company", "html", "https://www.anthropic.com/news", "html_listing", ("anthropic",), detail=True, + extra={"include_prefixes": ["/news/"]}), + S("deepmind_blog", "company", "html", "https://deepmind.google/blog/", "html_listing", ("deepmind",), detail=True, + extra={"include_contains": ["/blog/"], "exclude_exact": ["/blog", "/blog/"]}), + S("google_ai_blog", "company", "rss", "https://blog.google/technology/ai/rss/", "rss", ("google",)), + S("google_research_blog", "company", "rss", "https://research.google/blog/rss/", "rss", ("google",)), + S("microsoft_research", "company", "rss", "https://www.microsoft.com/en-us/research/feed/", "rss", ("microsoft",)), + S("nvidia_press", "company", "rss", "https://nvidianews.nvidia.com/cats/press_release.xml", "rss", ("nvidia",)), + S("nvidia_research_ai", "company", "html", + "https://research.nvidia.com/research-area/machine-learning-artificial-intelligence", "html_listing", + ("nvidia",), detail=True, extra={"include_prefixes": ["/publication/"]}), + S("apple_ml", "company", "rss", "https://machinelearning.apple.com/rss.xml", "rss", ("apple",)), + S("amazon_science", "company", "rss", "https://www.amazon.science/index.rss", "rss", ("amazon",)), + S("hf_blog", "company", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("hf",)), + S("meta_ai_blog", "company", "rss", "https://ai.meta.com/blog/rss/", "rss", ("meta",)), + S("mistral_news", "company", "html", "https://mistral.ai/news/", "html_listing", ("mistral",), detail=True, + extra={"include_prefixes": ["/news/"], "exclude_exact": ["/news", "/news/"]}), + S("cohere_blog", "company", "rss", "https://cohere.com/blog/rss.xml", "rss", ("cohere",)), + S("together_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)), + S("perplexity_blog", "company", "rss", "https://www.perplexity.ai/hub/blog/rss.xml", "rss", ("perplexity",)), + S("groq_newsroom", "company", "html", "https://groq.com/newsroom", "html_listing", ("groq",), detail=True, + extra={"include_prefixes": ["/newsroom/"], "exclude_exact": ["/newsroom"]}), + S("cerebras_blog", "company", "rss", "https://www.cerebras.ai/blog/rss.xml", "rss", ("cerebras",)), + S("stability_news", "company", "html", "https://stability.ai/news-updates", "html_listing", ("stability",), detail=True, + extra={"include_prefixes": ["/news-updates/"], "exclude_exact": ["/news-updates"]}), + S("databricks_blog", "company", "rss", "https://www.databricks.com/feed", "rss", ("databricks",)), + S("anyscale_blog", "company", "rss", "https://www.anyscale.com/blog/rss.xml", "rss", ("anyscale",)), + S("modal_blog", "company", "rss", "https://modal.com/blog/feed.xml", "rss", ("modal",)), + S("wandb_blog", "company", "rss", "https://wandb.ai/fully-connected/rss.xml", "rss", ("wandb",)), + S("langchain_blog", "company", "rss", "https://blog.langchain.dev/rss/", "rss", ("langchain",)), + S("llamaindex_blog", "company", "rss", "https://www.llamaindex.ai/blog/rss.xml", "rss", ("llamaindex",)), + S("vllm_blog", "company", "rss", "https://blog.vllm.ai/rss.xml", "rss", ("vllm",)), + S("replicate_blog", "company", "rss", "https://replicate.com/blog/rss.xml", "rss", ("replicate",)), + S("fireworks_blog", "company", "rss", "https://fireworks.ai/blog/rss.xml", "rss", ("fireworks",)), + S("sambanova_blog", "company", "rss", "https://sambanova.ai/blog/rss.xml", "rss", ("sambanova",)), + S("scale_blog", "company", "rss", "https://scale.com/blog/rss.xml", "rss", ("scale",)), + S("xai_news", "company", "html", "https://x.ai/news", "html_listing", ("xai",), detail=True, + extra={"include_prefixes": ["/news/"]}), + S("salesforce_ai", "company", "rss", "https://www.salesforce.com/blog/category/ai-research/feed/", "rss", ("salesforce",)), + S("aws_ml_blog", "company", "rss", "https://aws.amazon.com/blogs/machine-learning/feed/", "rss", ("aws",)), + S("azure_ai_blog", "company", "rss", "https://devblogs.microsoft.com/azure-ai/feed/", "rss", ("azure",)), + + # ---------- company_cn ---------- + S("qwen_blog", "company_cn", "rss", "https://qwenlm.github.io/blog/index.xml", "rss", ("qwen",)), + S("deepseek_updates", "company_cn", "html", "https://api-docs.deepseek.com/updates/", "html_listing", + ("deepseek",), detail=True, extra={"include_prefixes": ["/news/"]}), + S("moonshot_blog", "company_cn", "rss", "https://moonshotai.github.io/rss.xml", "rss", ("moonshot", "kimi")), + S("zhipu_blog", "company_cn", "rss", "https://zhipuai.cn/rss.xml", "rss", ("zhipu", "glm")), + S("baichuan_news", "company_cn", "rss", "https://www.baichuan-ai.com/rss.xml", "rss", ("baichuan",)), + S("minimax_blog", "company_cn", "rss", "https://www.minimaxi.com/rss.xml", "rss", ("minimax",)), + S("github_tencent_hunyuan", "company_cn", "github", "Tencent-Hunyuan", "github_org", ("tencent", "cn")), + S("github_paddlepaddle", "company_cn", "github", "PaddlePaddle", "github_org", ("paddlepaddle", "cn")), + S("github_bytedance", "company_cn", "github", "bytedance", "github_org", ("bytedance", "cn")), + S("github_mindspore", "company_cn", "github", "mindspore-ai", "github_org", ("mindspore", "cn")), + S("github_qwenlm", "company_cn", "github", "QwenLM", "github_org", ("qwen", "cn")), + S("github_deepseek", "company_cn", "github", "deepseek-ai", "github_org", ("deepseek", "cn")), + S("github_moonshot", "company_cn", "github", "MoonshotAI", "github_org", ("moonshot", "cn")), + S("github_zhipu", "company_cn", "github", "THUDM", "github_org", ("zhipu", "cn")), + + # ---------- company_kr ---------- + S("kakao_tech", "company_kr", "rss", "https://tech.kakao.com/feed/", "rss", ("kakao", "kr")), + S("naver_cloud_blog", "company_kr", "rss", "https://rss.blog.naver.com/n_cloudplatform.xml", "rss", ("naver", "kr")), + S("samsung_research", "company_kr", "json", "https://research.samsung.com/blogMain/list.json", "samsung", ("samsung", "kr")), + S("lg_ai_research", "company_kr", "json", "https://www.lgresearch.ai/api/blog/list?page=1&size=20", "lg_ai", ("lg", "kr")), + + # ---------- security ---------- + S("hackernews_security", "security", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("news",)), + S("bleepingcomputer", "security", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("news",)), + S("krebs", "security", "rss", "https://krebsonsecurity.com/feed/", "rss", ("news",)), + S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed_full.xml", "rss", ("news",)), + S("packetstorm", "security", "rss", "https://packetstormsecurity.com/rss.xml", "rss", ("tools",)), + S("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)), + S("exploitdb", "security", "rss", "https://www.exploit-db.com/rss.xml", "rss", ("exploit",)), + S("projectzero", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default", "rss", ("research",)), + S("portswigger_research", "security", "rss", "https://portswigger.net/research/rss", "rss", ("research",)), + S("github_security_blog", "security", "rss", "https://github.blog/tag/security/feed/", "rss", ("github",)), + + # ---------- forum ---------- + S("hn_security_discourse", "forum", "json", "https://community.openai.com/latest.json", "discourse", ("openai",)), + S("discourse_huggingface", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)), + S("discourse_python", "forum", "json", "https://discuss.python.org/latest.json", "discourse", ("python",)), + S("discourse_cloudflare", "forum", "json", "https://community.cloudflare.com/latest.json", "discourse", ("cloudflare",)), + S("discourse_elastic", "forum", "json", "https://discuss.elastic.co/latest.json", "discourse", ("elastic",)), + S("discourse_gitlab", "forum", "json", "https://forum.gitlab.com/latest.json", "discourse", ("gitlab",)), + S("xenforo_bhw", "forum", "html", "https://www.blackhatworld.com/forums/", "xenforo_latest", ("im", "seo")), + S("tg_openai", "forum", "html", "https://t.me/s/openai", "tg_web", ("telegram",)), + S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)), +] + +BY_NAME = {s.name: s for s in SOURCES} +CATEGORIES = sorted({s.category for s in SOURCES}) + + +def pick(category=None, only=None, tags=None): + out = SOURCES + if category: + cats = {c.strip() for c in category.split(",") if c.strip()} + out = [s for s in out if s.category in cats] + if only: + names = {n.strip() for n in only.split(",") if n.strip()} + out = [s for s in out if s.name in names] + if tags: + wanted = {t.strip() for t in tags.split(",") if t.strip()} + out = [s for s in out if wanted & set(s.tags)] + return out + + +def stats(): + out = {} + for s in SOURCES: + out.setdefault(s.category, []).append(s.name) + return {k: len(v) for k, v in sorted(out.items())} \ No newline at end of file