omni: реестр 113 источников (papers/company/cn/kr/community/security/forum/models/benchmark) + команды sources и feed
This commit is contained in:
dev
45
omni/cli.py
45
omni/cli.py
@@ -5,6 +5,7 @@ import json
|
||||
import sys
|
||||
|
||||
from . import config, rank, store
|
||||
from . import feeds, sources
|
||||
from . import filter as flt
|
||||
from .providers import FUNCS, MODULES, get, status
|
||||
|
||||
@@ -146,6 +147,34 @@ def cmd_stats(args):
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sources(args):
|
||||
st = sources.stats()
|
||||
print("источников:", len(sources.SOURCES), "| по категориям:",
|
||||
", ".join(f"{k}={v}" for k, v in st.items()))
|
||||
if args.list:
|
||||
for s in sources.pick(args.category, args.only, args.tags):
|
||||
print(f" {s.category:<12} {s.adapter:<18} {s.name:<26} {s.url[:70]}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_feed(args):
|
||||
picked = sources.pick(args.category, args.only, args.tags)
|
||||
if not picked:
|
||||
print("нет источников под фильтр")
|
||||
return 1
|
||||
print(f"обход {len(picked)} источников, лимит {args.limit} на источник")
|
||||
items, report = feeds.fetch_many(picked, limit=args.limit, workers=args.workers)
|
||||
items = rank.rank(items)
|
||||
new, dup = store.add(items)
|
||||
bad = [(n, c) for n, c in report if c == 0]
|
||||
for name, cnt in sorted(report, key=lambda x: -x[1])[:20]:
|
||||
print(f" {name:<28} {cnt}")
|
||||
print(f"пусто: {len(bad)}/{len(report)} | собрано {len(items)}, новых {new}, дублей {dup}")
|
||||
if bad and args.verbose:
|
||||
print("без отдачи:", ", ".join(n for n, _ in bad))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_export(args):
|
||||
n, path = store.export(args.out, args.fmt, args.limit)
|
||||
print(f"выгружено {n} записей -> {path}")
|
||||
@@ -201,6 +230,22 @@ def main(argv=None):
|
||||
sp = sub.add_parser("stats", help="сводка базы")
|
||||
sp.set_defaults(fn=cmd_stats)
|
||||
|
||||
sp = sub.add_parser("sources", help="реестр источников")
|
||||
sp.add_argument("--category", default=None)
|
||||
sp.add_argument("--only", default=None)
|
||||
sp.add_argument("--tags", default=None)
|
||||
sp.add_argument("--list", action="store_true")
|
||||
sp.set_defaults(fn=cmd_sources)
|
||||
|
||||
sp = sub.add_parser("feed", help="обход реестра источников без запроса")
|
||||
sp.add_argument("--category", default=None)
|
||||
sp.add_argument("--only", default=None)
|
||||
sp.add_argument("--tags", default=None)
|
||||
sp.add_argument("--limit", type=int, default=15)
|
||||
sp.add_argument("--workers", type=int, default=8)
|
||||
sp.add_argument("--verbose", action="store_true")
|
||||
sp.set_defaults(fn=cmd_feed)
|
||||
|
||||
sp = sub.add_parser("export", help="выгрузка jsonl/csv")
|
||||
sp.add_argument("--out", default="data/top.jsonl")
|
||||
sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"])
|
||||
|
||||
413
omni/feeds.py
Normal file
413
omni/feeds.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""Забор источников из реестра sources.py.
|
||||
|
||||
Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
|
||||
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
|
||||
"""
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from . import config, net
|
||||
from .model import Item
|
||||
|
||||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "en,ru;q=0.8"}
|
||||
|
||||
|
||||
def _txt(x):
|
||||
if x is None:
|
||||
return ""
|
||||
return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()
|
||||
|
||||
|
||||
def _item(src, kind, url, title, text="", author="", published="", extra=None):
|
||||
ex = dict(extra or {})
|
||||
ex["tags"] = list(src.tags)
|
||||
return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
|
||||
author=_txt(author)[:80], published=str(published or ""), source=src.category,
|
||||
extra=ex)
|
||||
|
||||
|
||||
# ---------------- rss ----------------
|
||||
def _rss(src, limit):
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status in (429, 503):
|
||||
import time
|
||||
time.sleep(2.5)
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status != 200 or not r.text.strip():
|
||||
return []
|
||||
out = []
|
||||
try:
|
||||
root = ET.fromstring(r.text.encode("utf-8", "ignore"))
|
||||
except Exception:
|
||||
return _rss_regex(src, r.text, limit)
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
entries = root.findall(".//item") or root.findall(".//a:entry", ns)
|
||||
for e in entries[:limit]:
|
||||
g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
|
||||
link = g("link")
|
||||
if not link:
|
||||
le = e.find("a:link", ns) or e.find("link")
|
||||
link = le.get("href") if le is not None and le.get("href") else ""
|
||||
body = g("description") or g("summary") or g("content")
|
||||
out.append(_item(src, "entry", link, g("title"), body,
|
||||
g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
|
||||
return out
|
||||
|
||||
|
||||
def _rss_regex(src, text, limit):
|
||||
out = []
|
||||
for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
|
||||
blob = m.group(1)
|
||||
g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
|
||||
out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
# ---------------- json adapters ----------------
|
||||
def _hn_top(src, limit):
|
||||
ids = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||||
out = []
|
||||
for sid in ids[:limit]:
|
||||
d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
|
||||
if not d.get("title"):
|
||||
continue
|
||||
out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
|
||||
d.get("title"), d.get("text", ""), d.get("by", ""),
|
||||
str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
|
||||
"hn": f"https://news.ycombinator.com/item?id={sid}"}))
|
||||
return out
|
||||
|
||||
|
||||
def _reddit_listing(src, limit):
|
||||
"""Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
|
||||
sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||||
rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
|
||||
import time
|
||||
children = []
|
||||
if rss_first:
|
||||
rr = net.get(rss_first, headers=UA)
|
||||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||||
return _reddit_atom(src, rr.text, limit)
|
||||
if rr.status in (429, 503):
|
||||
time.sleep(4)
|
||||
rr = net.get(rss_first, headers=UA)
|
||||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||||
return _reddit_atom(src, rr.text, limit)
|
||||
r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
|
||||
out = []
|
||||
for ch in children[:limit]:
|
||||
d = ch.get("data") or {}
|
||||
if not d.get("title"):
|
||||
continue
|
||||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||||
str(d.get("created_utc", "")),
|
||||
{"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
|
||||
if out:
|
||||
return out
|
||||
|
||||
rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
|
||||
r2 = net.get(rss_url, headers=UA) if not rss_first else None
|
||||
if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
|
||||
return _reddit_atom(src, r2.text, limit)
|
||||
|
||||
sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||||
if sub:
|
||||
alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
|
||||
f"&size={limit}&sort=desc", headers=UA, json_mode=True)
|
||||
for d in ((alt.json() or {}).get("data") or [])[:limit]:
|
||||
if d.get("title"):
|
||||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||||
str(d.get("created_utc", "")), {"ups": d.get("score")}))
|
||||
return out
|
||||
|
||||
|
||||
def _reddit_atom(src, text, limit):
|
||||
out = []
|
||||
try:
|
||||
root = ET.fromstring(text.encode("utf-8", "ignore"))
|
||||
except Exception:
|
||||
return out
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
for e in root.findall(".//a:entry", ns)[:limit]:
|
||||
title = e.findtext("a:title", namespaces=ns) or ""
|
||||
link_el = e.find("a:link", ns)
|
||||
link = link_el.get("href") if link_el is not None else ""
|
||||
content = e.findtext("a:content", namespaces=ns) or ""
|
||||
author = e.findtext("a:author/a:name", namespaces=ns) or ""
|
||||
upd = e.findtext("a:updated", namespaces=ns) or ""
|
||||
if title:
|
||||
out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
|
||||
return out
|
||||
|
||||
|
||||
def _hf_generic(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
|
||||
or d.get("spaces") or d.get("datasets") or [])
|
||||
out = []
|
||||
for x in rows[:limit]:
|
||||
if not isinstance(x, dict):
|
||||
continue
|
||||
rid = x.get("id") or x.get("repoId") or x.get("name") or ""
|
||||
if not rid:
|
||||
continue
|
||||
kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
|
||||
url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
|
||||
out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
|
||||
"", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
|
||||
return out
|
||||
|
||||
|
||||
def _hf_models(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||||
out = []
|
||||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||||
out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
|
||||
"", "", x.get("createdAt", ""),
|
||||
{"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
|
||||
return out
|
||||
|
||||
|
||||
def _hf_daily_papers(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||||
out = []
|
||||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||||
p = x.get("paper") or x
|
||||
pid = p.get("id") or ""
|
||||
out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
|
||||
p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
|
||||
p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
|
||||
return out
|
||||
|
||||
|
||||
def _hf_rows(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
out = []
|
||||
for row in (d.get("rows") or [])[:limit]:
|
||||
r = row.get("row") or {}
|
||||
name = r.get("fullname") or r.get("model") or r.get("Model") or ""
|
||||
if not name:
|
||||
continue
|
||||
out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
|
||||
"", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
|
||||
return out
|
||||
|
||||
|
||||
def _openalex(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
out = []
|
||||
for w in (d.get("results") or [])[:limit]:
|
||||
out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
|
||||
w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
|
||||
{"cited": w.get("cited_by_count")}))
|
||||
return out
|
||||
|
||||
|
||||
def _s2(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
out = []
|
||||
for p in (d.get("data") or [])[:limit]:
|
||||
out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
|
||||
", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
|
||||
str(p.get("year", "")), {"citations": p.get("citationCount")}))
|
||||
return out
|
||||
|
||||
|
||||
def _samsung(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
|
||||
out = []
|
||||
for x in rows[:limit]:
|
||||
if not isinstance(x, dict):
|
||||
continue
|
||||
out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
|
||||
x.get("title", ""), x.get("summary") or x.get("content", ""),
|
||||
"", str(x.get("regDate") or x.get("date") or ""), {}))
|
||||
return out
|
||||
|
||||
|
||||
def _lg_ai(src, limit):
|
||||
d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
|
||||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
|
||||
out = []
|
||||
for x in rows[:limit]:
|
||||
if not isinstance(x, dict):
|
||||
continue
|
||||
out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
|
||||
x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
|
||||
return out
|
||||
|
||||
|
||||
def _github_org(src, limit):
|
||||
hdr = dict(UA)
|
||||
tok = config.KEYS.get("GITHUB_TOKEN")
|
||||
if tok:
|
||||
hdr["Authorization"] = f"token {tok}"
|
||||
url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
|
||||
d = net.get(url, headers=hdr, json_mode=True).json() or []
|
||||
out = []
|
||||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||||
out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
|
||||
x.get("description", ""), x.get("owner", {}).get("login", ""),
|
||||
x.get("pushed_at", ""),
|
||||
{"stars": x.get("stargazers_count"), "lang": x.get("language")}))
|
||||
return out
|
||||
|
||||
|
||||
def _discourse(src, limit):
|
||||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||||
base = src.url.split("/latest.json")[0]
|
||||
out = []
|
||||
for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
|
||||
out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
|
||||
t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
|
||||
"", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
|
||||
return out
|
||||
|
||||
|
||||
def _xenforo_latest(src, limit):
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status != 200:
|
||||
return []
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
out = []
|
||||
for a in soup.select("a[href*='/threads/']"):
|
||||
title = a.get_text(" ", strip=True)
|
||||
href = str(a.get("href", "") or "")
|
||||
if len(title) < 12 or href.rstrip("/").endswith("/threads"):
|
||||
continue
|
||||
if not href.startswith("http"):
|
||||
href = "https://www.blackhatworld.com" + href
|
||||
out.append(_item(src, "thread", href, title))
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _tg_web(src, limit):
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status != 200:
|
||||
return []
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
out = []
|
||||
for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]:
|
||||
text_el = msg.select_one("div.tgme_widget_message_text")
|
||||
link_el = msg.select_one("a.tgme_widget_message_date")
|
||||
text = text_el.get_text("\n", strip=True) if text_el else ""
|
||||
link = str(link_el.get("href") or src.url) if link_el else src.url
|
||||
if not text:
|
||||
continue
|
||||
out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {}))
|
||||
return out
|
||||
|
||||
|
||||
def _html_listing(src, limit):
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status != 200:
|
||||
return []
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
inc = src.extra.get("include_prefixes") or []
|
||||
inc_c = src.extra.get("include_contains") or []
|
||||
exc = set(src.extra.get("exclude_exact") or [])
|
||||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||||
seen, links = set(), []
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = urllib.parse.urljoin(base, str(a["href"]))
|
||||
path = urllib.parse.urlsplit(href).path
|
||||
if path in exc or len(path) < 6:
|
||||
continue
|
||||
if inc and not any(path.startswith(p) for p in inc):
|
||||
continue
|
||||
if inc_c and not any(c in path for c in inc_c):
|
||||
continue
|
||||
if href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
links.append((href, a.get_text(" ", strip=True)))
|
||||
if len(links) >= limit:
|
||||
break
|
||||
out = []
|
||||
for href, title in links:
|
||||
text = ""
|
||||
if src.detail:
|
||||
d = net.get(href, headers=UA)
|
||||
if d.status == 200:
|
||||
ds = BeautifulSoup(d.text, "lxml")
|
||||
for bad in ds(["script", "style", "nav", "footer", "header"]):
|
||||
bad.decompose()
|
||||
paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
|
||||
text = " ".join(paras)[:4000]
|
||||
if not title:
|
||||
t = ds.find("h1") or ds.title
|
||||
title = t.get_text(" ", strip=True) if t else href
|
||||
out.append(_item(src, "entry", href, title or href, text))
|
||||
return out
|
||||
|
||||
|
||||
ADAPTERS = {
|
||||
"rss": _rss,
|
||||
"hn_top": _hn_top,
|
||||
"reddit_listing": _reddit_listing,
|
||||
"hf_trending": _hf_generic,
|
||||
"hf_generic": _hf_generic,
|
||||
"hf_models": _hf_models,
|
||||
"hf_daily_papers": _hf_daily_papers,
|
||||
"hf_rows": _hf_rows,
|
||||
"openalex": _openalex,
|
||||
"s2": _s2,
|
||||
"samsung": _samsung,
|
||||
"lg_ai": _lg_ai,
|
||||
"github_org": _github_org,
|
||||
"discourse": _discourse,
|
||||
"xenforo_latest": _xenforo_latest,
|
||||
"tg_web": _tg_web,
|
||||
"html_listing": _html_listing,
|
||||
}
|
||||
|
||||
|
||||
def fetch(src, limit=None):
|
||||
fn = ADAPTERS.get(src.adapter)
|
||||
if not fn:
|
||||
return []
|
||||
n = limit or src.limit
|
||||
try:
|
||||
return fn(src, n) or []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
|
||||
|
||||
|
||||
def fetch_many(sources, limit=None, workers=8):
|
||||
"""Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
|
||||
fast = [s for s in sources if s not in slow]
|
||||
items, report = [], []
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futs = [(ex.submit(fetch, s, limit), s) for s in fast]
|
||||
for f, s in futs:
|
||||
got = f.result()
|
||||
report.append((s.name, len(got)))
|
||||
items.extend(got)
|
||||
for s in slow:
|
||||
got = fetch(s, limit)
|
||||
if not got and "reddit.com" in s.url:
|
||||
time.sleep(7)
|
||||
got = fetch(s, limit)
|
||||
report.append((s.name, len(got)))
|
||||
items.extend(got)
|
||||
time.sleep(3.5 if "reddit.com" in s.url else 1.0)
|
||||
return items, report
|
||||
199
omni/sources.py
Normal file
199
omni/sources.py
Normal file
@@ -0,0 +1,199 @@
|
||||
"""Реестр источников: один список на весь парсер.
|
||||
|
||||
Типы:
|
||||
rss — RSS/Atom лента
|
||||
json — JSON API/листинг
|
||||
html — страница-список, ссылки вытаскиваются по префиксу, тело тянется с детальной
|
||||
github — org/user repos через api.github.com (без токена, лимит 60/час)
|
||||
|
||||
Категории: papers, models, community, company, company_cn, company_kr, security, benchmark, forum.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
name: str
|
||||
category: str
|
||||
kind: str
|
||||
url: str
|
||||
adapter: str = "rss"
|
||||
tags: tuple = ()
|
||||
limit: int = 20
|
||||
detail: bool = False
|
||||
extra: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
S = Source
|
||||
|
||||
SOURCES = [
|
||||
# ---------- papers ----------
|
||||
S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_cl", "papers", "rss", "https://rss.arxiv.org/rss/cs.CL", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_cv", "papers", "rss", "https://rss.arxiv.org/rss/cs.CV", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ro", "papers", "rss", "https://rss.arxiv.org/rss/cs.RO", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ir", "papers", "rss", "https://rss.arxiv.org/rss/cs.IR", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_cr", "papers", "rss", "https://rss.arxiv.org/rss/cs.CR", "rss", ("paper", "arxiv", "security")),
|
||||
S("arxiv_cs_se", "papers", "rss", "https://rss.arxiv.org/rss/cs.SE", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ne", "papers", "rss", "https://rss.arxiv.org/rss/cs.NE", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ma", "papers", "rss", "https://rss.arxiv.org/rss/cs.MA", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_dc", "papers", "rss", "https://rss.arxiv.org/rss/cs.DC", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_db", "papers", "rss", "https://rss.arxiv.org/rss/cs.DB", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_hc", "papers", "rss", "https://rss.arxiv.org/rss/cs.HC", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_stat_ml", "papers", "rss", "https://rss.arxiv.org/rss/stat.ML", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_eess_as", "papers", "rss", "https://rss.arxiv.org/rss/eess.AS", "rss", ("paper", "arxiv", "audio")),
|
||||
S("hf_daily_papers", "papers", "json", "https://huggingface.co/api/daily_papers", "hf_daily_papers", ("paper", "hf")),
|
||||
S("openalex_recent", "papers", "json",
|
||||
"https://api.openalex.org/works?filter=from_publication_date:2026-01-01&sort=publication_date:desc&per_page=20",
|
||||
"openalex", ("paper", "openalex")),
|
||||
S("semanticscholar_recent", "papers", "json",
|
||||
"https://api.semanticscholar.org/graph/v1/paper/search?query=large+language+model&limit=20&fields=title,abstract,url,venue,year,authors,citationCount",
|
||||
"s2", ("paper", "semanticscholar")),
|
||||
|
||||
# ---------- models ----------
|
||||
S("hf_trending_models", "models", "json", "https://huggingface.co/api/trending?type=model", "hf_trending", ("model", "hf")),
|
||||
S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")),
|
||||
S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")),
|
||||
S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")),
|
||||
S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"), detail=False),
|
||||
|
||||
# ---------- benchmark ----------
|
||||
S("open_llm_leaderboard", "benchmark", "json",
|
||||
"https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=0&length=25",
|
||||
"hf_rows", ("benchmark", "leaderboard")),
|
||||
S("lmarena_overview", "benchmark", "html", "https://lmarena.ai/leaderboard", "html_listing", ("benchmark", "arena")),
|
||||
|
||||
# ---------- community ----------
|
||||
S("hn_top", "community", "json", "https://hacker-news.firebaseio.com/v0/topstories.json", "hn_top", ("hn",), limit=20),
|
||||
S("hn_new", "community", "json", "https://hacker-news.firebaseio.com/v0/newstories.json", "hn_top", ("hn",), limit=15),
|
||||
S("lobsters", "community", "rss", "https://lobste.rs/rss", "rss", ("lobsters",)),
|
||||
S("lobsters_ai", "community", "rss", "https://lobste.rs/t/ai.rss", "rss", ("lobsters", "ai")),
|
||||
S("lobsters_security", "community", "rss", "https://lobste.rs/t/security.rss", "rss", ("lobsters", "security")),
|
||||
S("geeknews", "community", "rss", "https://news.hada.io/rss/news", "rss", ("kr",)),
|
||||
S("reddit_machinelearning", "community", "json", "https://www.reddit.com/r/MachineLearning/.json?limit=25", "reddit_listing", ("reddit", "ml")),
|
||||
S("reddit_localllama", "community", "json", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", "reddit_listing", ("reddit", "local")),
|
||||
S("reddit_openai", "community", "json", "https://www.reddit.com/r/OpenAI/.json?limit=25", "reddit_listing", ("reddit",)),
|
||||
S("reddit_singularity", "community", "json", "https://www.reddit.com/r/singularity/.json?limit=25", "reddit_listing", ("reddit",)),
|
||||
S("reddit_stablediffusion", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25", "reddit_listing", ("reddit", "image")),
|
||||
S("reddit_netsec", "community", "json", "https://www.reddit.com/r/netsec/.json?limit=25", "reddit_listing", ("reddit", "security")),
|
||||
S("reddit_cybersecurity", "community", "json", "https://www.reddit.com/r/cybersecurity/.json?limit=25", "reddit_listing", ("reddit", "security")),
|
||||
S("reddit_webscraping", "community", "json", "https://www.reddit.com/r/webscraping/.json?limit=25", "reddit_listing", ("reddit", "scraping")),
|
||||
S("reddit_selfhosted", "community", "json", "https://www.reddit.com/r/selfhosted/.json?limit=25", "reddit_listing", ("reddit", "selfhosted")),
|
||||
S("reddit_python", "community", "json", "https://www.reddit.com/r/Python/.json?limit=25", "reddit_listing", ("reddit", "python")),
|
||||
S("reddit_artificial", "community", "json", "https://www.reddit.com/r/artificial/.json?limit=25", "reddit_listing", ("reddit", "ai")),
|
||||
|
||||
# ---------- company ----------
|
||||
S("openai_news", "company", "rss", "https://openai.com/news/rss.xml", "rss", ("openai",)),
|
||||
S("anthropic_news", "company", "html", "https://www.anthropic.com/news", "html_listing", ("anthropic",), detail=True,
|
||||
extra={"include_prefixes": ["/news/"]}),
|
||||
S("deepmind_blog", "company", "html", "https://deepmind.google/blog/", "html_listing", ("deepmind",), detail=True,
|
||||
extra={"include_contains": ["/blog/"], "exclude_exact": ["/blog", "/blog/"]}),
|
||||
S("google_ai_blog", "company", "rss", "https://blog.google/technology/ai/rss/", "rss", ("google",)),
|
||||
S("google_research_blog", "company", "rss", "https://research.google/blog/rss/", "rss", ("google",)),
|
||||
S("microsoft_research", "company", "rss", "https://www.microsoft.com/en-us/research/feed/", "rss", ("microsoft",)),
|
||||
S("nvidia_press", "company", "rss", "https://nvidianews.nvidia.com/cats/press_release.xml", "rss", ("nvidia",)),
|
||||
S("nvidia_research_ai", "company", "html",
|
||||
"https://research.nvidia.com/research-area/machine-learning-artificial-intelligence", "html_listing",
|
||||
("nvidia",), detail=True, extra={"include_prefixes": ["/publication/"]}),
|
||||
S("apple_ml", "company", "rss", "https://machinelearning.apple.com/rss.xml", "rss", ("apple",)),
|
||||
S("amazon_science", "company", "rss", "https://www.amazon.science/index.rss", "rss", ("amazon",)),
|
||||
S("hf_blog", "company", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("hf",)),
|
||||
S("meta_ai_blog", "company", "rss", "https://ai.meta.com/blog/rss/", "rss", ("meta",)),
|
||||
S("mistral_news", "company", "html", "https://mistral.ai/news/", "html_listing", ("mistral",), detail=True,
|
||||
extra={"include_prefixes": ["/news/"], "exclude_exact": ["/news", "/news/"]}),
|
||||
S("cohere_blog", "company", "rss", "https://cohere.com/blog/rss.xml", "rss", ("cohere",)),
|
||||
S("together_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)),
|
||||
S("perplexity_blog", "company", "rss", "https://www.perplexity.ai/hub/blog/rss.xml", "rss", ("perplexity",)),
|
||||
S("groq_newsroom", "company", "html", "https://groq.com/newsroom", "html_listing", ("groq",), detail=True,
|
||||
extra={"include_prefixes": ["/newsroom/"], "exclude_exact": ["/newsroom"]}),
|
||||
S("cerebras_blog", "company", "rss", "https://www.cerebras.ai/blog/rss.xml", "rss", ("cerebras",)),
|
||||
S("stability_news", "company", "html", "https://stability.ai/news-updates", "html_listing", ("stability",), detail=True,
|
||||
extra={"include_prefixes": ["/news-updates/"], "exclude_exact": ["/news-updates"]}),
|
||||
S("databricks_blog", "company", "rss", "https://www.databricks.com/feed", "rss", ("databricks",)),
|
||||
S("anyscale_blog", "company", "rss", "https://www.anyscale.com/blog/rss.xml", "rss", ("anyscale",)),
|
||||
S("modal_blog", "company", "rss", "https://modal.com/blog/feed.xml", "rss", ("modal",)),
|
||||
S("wandb_blog", "company", "rss", "https://wandb.ai/fully-connected/rss.xml", "rss", ("wandb",)),
|
||||
S("langchain_blog", "company", "rss", "https://blog.langchain.dev/rss/", "rss", ("langchain",)),
|
||||
S("llamaindex_blog", "company", "rss", "https://www.llamaindex.ai/blog/rss.xml", "rss", ("llamaindex",)),
|
||||
S("vllm_blog", "company", "rss", "https://blog.vllm.ai/rss.xml", "rss", ("vllm",)),
|
||||
S("replicate_blog", "company", "rss", "https://replicate.com/blog/rss.xml", "rss", ("replicate",)),
|
||||
S("fireworks_blog", "company", "rss", "https://fireworks.ai/blog/rss.xml", "rss", ("fireworks",)),
|
||||
S("sambanova_blog", "company", "rss", "https://sambanova.ai/blog/rss.xml", "rss", ("sambanova",)),
|
||||
S("scale_blog", "company", "rss", "https://scale.com/blog/rss.xml", "rss", ("scale",)),
|
||||
S("xai_news", "company", "html", "https://x.ai/news", "html_listing", ("xai",), detail=True,
|
||||
extra={"include_prefixes": ["/news/"]}),
|
||||
S("salesforce_ai", "company", "rss", "https://www.salesforce.com/blog/category/ai-research/feed/", "rss", ("salesforce",)),
|
||||
S("aws_ml_blog", "company", "rss", "https://aws.amazon.com/blogs/machine-learning/feed/", "rss", ("aws",)),
|
||||
S("azure_ai_blog", "company", "rss", "https://devblogs.microsoft.com/azure-ai/feed/", "rss", ("azure",)),
|
||||
|
||||
# ---------- company_cn ----------
|
||||
S("qwen_blog", "company_cn", "rss", "https://qwenlm.github.io/blog/index.xml", "rss", ("qwen",)),
|
||||
S("deepseek_updates", "company_cn", "html", "https://api-docs.deepseek.com/updates/", "html_listing",
|
||||
("deepseek",), detail=True, extra={"include_prefixes": ["/news/"]}),
|
||||
S("moonshot_blog", "company_cn", "rss", "https://moonshotai.github.io/rss.xml", "rss", ("moonshot", "kimi")),
|
||||
S("zhipu_blog", "company_cn", "rss", "https://zhipuai.cn/rss.xml", "rss", ("zhipu", "glm")),
|
||||
S("baichuan_news", "company_cn", "rss", "https://www.baichuan-ai.com/rss.xml", "rss", ("baichuan",)),
|
||||
S("minimax_blog", "company_cn", "rss", "https://www.minimaxi.com/rss.xml", "rss", ("minimax",)),
|
||||
S("github_tencent_hunyuan", "company_cn", "github", "Tencent-Hunyuan", "github_org", ("tencent", "cn")),
|
||||
S("github_paddlepaddle", "company_cn", "github", "PaddlePaddle", "github_org", ("paddlepaddle", "cn")),
|
||||
S("github_bytedance", "company_cn", "github", "bytedance", "github_org", ("bytedance", "cn")),
|
||||
S("github_mindspore", "company_cn", "github", "mindspore-ai", "github_org", ("mindspore", "cn")),
|
||||
S("github_qwenlm", "company_cn", "github", "QwenLM", "github_org", ("qwen", "cn")),
|
||||
S("github_deepseek", "company_cn", "github", "deepseek-ai", "github_org", ("deepseek", "cn")),
|
||||
S("github_moonshot", "company_cn", "github", "MoonshotAI", "github_org", ("moonshot", "cn")),
|
||||
S("github_zhipu", "company_cn", "github", "THUDM", "github_org", ("zhipu", "cn")),
|
||||
|
||||
# ---------- company_kr ----------
|
||||
S("kakao_tech", "company_kr", "rss", "https://tech.kakao.com/feed/", "rss", ("kakao", "kr")),
|
||||
S("naver_cloud_blog", "company_kr", "rss", "https://rss.blog.naver.com/n_cloudplatform.xml", "rss", ("naver", "kr")),
|
||||
S("samsung_research", "company_kr", "json", "https://research.samsung.com/blogMain/list.json", "samsung", ("samsung", "kr")),
|
||||
S("lg_ai_research", "company_kr", "json", "https://www.lgresearch.ai/api/blog/list?page=1&size=20", "lg_ai", ("lg", "kr")),
|
||||
|
||||
# ---------- security ----------
|
||||
S("hackernews_security", "security", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("news",)),
|
||||
S("bleepingcomputer", "security", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("news",)),
|
||||
S("krebs", "security", "rss", "https://krebsonsecurity.com/feed/", "rss", ("news",)),
|
||||
S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed_full.xml", "rss", ("news",)),
|
||||
S("packetstorm", "security", "rss", "https://packetstormsecurity.com/rss.xml", "rss", ("tools",)),
|
||||
S("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
|
||||
S("exploitdb", "security", "rss", "https://www.exploit-db.com/rss.xml", "rss", ("exploit",)),
|
||||
S("projectzero", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default", "rss", ("research",)),
|
||||
S("portswigger_research", "security", "rss", "https://portswigger.net/research/rss", "rss", ("research",)),
|
||||
S("github_security_blog", "security", "rss", "https://github.blog/tag/security/feed/", "rss", ("github",)),
|
||||
|
||||
# ---------- forum ----------
|
||||
S("hn_security_discourse", "forum", "json", "https://community.openai.com/latest.json", "discourse", ("openai",)),
|
||||
S("discourse_huggingface", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)),
|
||||
S("discourse_python", "forum", "json", "https://discuss.python.org/latest.json", "discourse", ("python",)),
|
||||
S("discourse_cloudflare", "forum", "json", "https://community.cloudflare.com/latest.json", "discourse", ("cloudflare",)),
|
||||
S("discourse_elastic", "forum", "json", "https://discuss.elastic.co/latest.json", "discourse", ("elastic",)),
|
||||
S("discourse_gitlab", "forum", "json", "https://forum.gitlab.com/latest.json", "discourse", ("gitlab",)),
|
||||
S("xenforo_bhw", "forum", "html", "https://www.blackhatworld.com/forums/", "xenforo_latest", ("im", "seo")),
|
||||
S("tg_openai", "forum", "html", "https://t.me/s/openai", "tg_web", ("telegram",)),
|
||||
S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
|
||||
]
|
||||
|
||||
BY_NAME = {s.name: s for s in SOURCES}
|
||||
CATEGORIES = sorted({s.category for s in SOURCES})
|
||||
|
||||
|
||||
def pick(category=None, only=None, tags=None):
|
||||
out = SOURCES
|
||||
if category:
|
||||
cats = {c.strip() for c in category.split(",") if c.strip()}
|
||||
out = [s for s in out if s.category in cats]
|
||||
if only:
|
||||
names = {n.strip() for n in only.split(",") if n.strip()}
|
||||
out = [s for s in out if s.name in names]
|
||||
if tags:
|
||||
wanted = {t.strip() for t in tags.split(",") if t.strip()}
|
||||
out = [s for s in out if wanted & set(s.tags)]
|
||||
return out
|
||||
|
||||
|
||||
def stats():
|
||||
out = {}
|
||||
for s in SOURCES:
|
||||
out.setdefault(s.category, []).append(s.name)
|
||||
return {k: len(v) for k, v in sorted(out.items())}
|
||||
Reference in New Issue
Block a user