781 lines
31 KiB
Python
781 lines
31 KiB
Python
"""Забор источников из реестра sources.py.
|
||
|
||
Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
|
||
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
|
||
"""
|
||
import html
|
||
import json
|
||
import re
|
||
import urllib.parse
|
||
import xml.etree.ElementTree as ET
|
||
|
||
from bs4 import BeautifulSoup
|
||
|
||
from . import config, net
|
||
from .model import Item
|
||
|
||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
|
||
"Accept-Language": "en,ru;q=0.8"}
|
||
|
||
|
||
def _txt(x):
|
||
if x is None:
|
||
return ""
|
||
return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()
|
||
|
||
|
||
def _item(src, kind, url, title, text="", author="", published="", extra=None):
|
||
ex = dict(extra or {})
|
||
ex["tags"] = list(src.tags)
|
||
t = _txt(title)
|
||
body = _txt(text)
|
||
if sum(ch.isalpha() for ch in t) < 12 and body:
|
||
cand = [ln for ln in body.splitlines() if sum(ch.isalpha() for ch in ln) >= 12]
|
||
if cand:
|
||
t = max(cand, key=len)[:140]
|
||
# CVE-каналы постят тело advisory без идентификатора в заголовке — выносим его вперёд
|
||
if "CVE-" not in t:
|
||
m = re.search(r"CVE-\d{4}-\d{4,7}", body or "")
|
||
if m:
|
||
t = "%s · %s" % (m.group(0), t[:120])
|
||
return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
|
||
author=_txt(author)[:80], published=str(published or ""), source=src.name,
|
||
extra=ex)
|
||
|
||
|
||
# ---------------- rss ----------------
|
||
def _arxiv_api(src, limit):
|
||
"""arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
|
||
m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
|
||
cat = m.group(1) if m else "cs.AI"
|
||
url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
|
||
"&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
|
||
r = net.get(url, headers=UA)
|
||
if r.status != 200 or "<entry" not in (r.text or ""):
|
||
return []
|
||
out = []
|
||
try:
|
||
root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return []
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
for e in root.findall(".//a:entry", ns)[:limit]:
|
||
title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
|
||
summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
|
||
link = e.findtext("a:id", namespaces=ns) or ""
|
||
pub = e.findtext("a:published", namespaces=ns) or ""
|
||
auth = e.findtext("a:author/a:name", namespaces=ns) or ""
|
||
if title:
|
||
out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
|
||
{"cat": cat}))
|
||
return out
|
||
|
||
|
||
_FEED_CACHE = None
|
||
|
||
|
||
def _cache_url(src):
|
||
global _FEED_CACHE
|
||
if _FEED_CACHE is None:
|
||
try:
|
||
import pathlib as _pl
|
||
f = _pl.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
|
||
_FEED_CACHE = __import__("json").loads(f.read_text(encoding="utf-8")) if f.exists() else {}
|
||
except Exception:
|
||
_FEED_CACHE = {}
|
||
hit = _FEED_CACHE.get(src.name)
|
||
return hit.get("url") if isinstance(hit, dict) and hit.get("url") else src.url
|
||
|
||
|
||
def _rss(src, limit):
|
||
r = net.get(_cache_url(src), headers=UA)
|
||
if r.status in (429, 503):
|
||
import time
|
||
time.sleep(2.5)
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200 or not r.text.strip():
|
||
return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
|
||
out = []
|
||
try:
|
||
root = ET.fromstring(r.text.encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return _rss_regex(src, r.text, limit)
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
entries = root.findall(".//item") or root.findall(".//a:entry", ns)
|
||
for e in entries[:limit]:
|
||
g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
|
||
link = g("link")
|
||
if not link:
|
||
le = e.find("a:link", ns) or e.find("link")
|
||
link = le.get("href") if le is not None and le.get("href") else ""
|
||
body = g("description") or g("summary") or g("content")
|
||
out.append(_item(src, "entry", link, g("title"), body,
|
||
g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
|
||
if not out and "arxiv.org" in src.url:
|
||
return _arxiv_api(src, limit)
|
||
return out
|
||
|
||
|
||
def _rss_regex(src, text, limit):
|
||
out = []
|
||
for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
|
||
blob = m.group(1)
|
||
g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
|
||
out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
# ---------------- json adapters ----------------
|
||
def _hn_top(src, limit):
|
||
ids = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for sid in ids[:limit]:
|
||
d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
|
||
if not d.get("title"):
|
||
continue
|
||
out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
|
||
d.get("title"), d.get("text", ""), d.get("by", ""),
|
||
str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
|
||
"hn": f"https://news.ycombinator.com/item?id={sid}"}))
|
||
return out
|
||
|
||
|
||
def _reddit_listing(src, limit):
|
||
"""Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
|
||
sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||
rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
|
||
import time
|
||
children = []
|
||
if rss_first:
|
||
rr = net.get(rss_first, headers=UA)
|
||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||
return _reddit_atom(src, rr.text, limit)
|
||
if rr.status in (429, 503):
|
||
time.sleep(4)
|
||
rr = net.get(rss_first, headers=UA)
|
||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||
return _reddit_atom(src, rr.text, limit)
|
||
r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
|
||
out = []
|
||
if r.status == 200 and isinstance(r.json(), dict):
|
||
children = (((r.json() or {}).get("data") or {}).get("children") or [])
|
||
for ch in children[:limit]:
|
||
d = ch.get("data") or {}
|
||
if not d.get("title"):
|
||
continue
|
||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||
str(d.get("created_utc", "")),
|
||
{"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
|
||
if out:
|
||
return out
|
||
|
||
rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
|
||
r2 = net.get(rss_url, headers=UA) if not rss_first else None
|
||
if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
|
||
return _reddit_atom(src, r2.text, limit)
|
||
|
||
sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||
if sub:
|
||
alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
|
||
f"&size={limit}&sort=desc", headers=UA, json_mode=True)
|
||
for d in ((alt.json() or {}).get("data") or [])[:limit]:
|
||
if d.get("title"):
|
||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||
str(d.get("created_utc", "")), {"ups": d.get("score")}))
|
||
return out
|
||
|
||
|
||
def _reddit_atom(src, text, limit):
|
||
out = []
|
||
try:
|
||
root = ET.fromstring(text.encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return out
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
for e in root.findall(".//a:entry", ns)[:limit]:
|
||
title = e.findtext("a:title", namespaces=ns) or ""
|
||
link_el = e.find("a:link", ns)
|
||
link = link_el.get("href") if link_el is not None else ""
|
||
content = e.findtext("a:content", namespaces=ns) or ""
|
||
author = e.findtext("a:author/a:name", namespaces=ns) or ""
|
||
upd = e.findtext("a:updated", namespaces=ns) or ""
|
||
if title:
|
||
out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
|
||
return out
|
||
|
||
|
||
def _hf_generic(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
|
||
or d.get("spaces") or d.get("datasets") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
if "repoData" in x: # trending: {repoType, repoData:{...}}
|
||
x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
|
||
rid = x.get("id") or x.get("repoId") or x.get("name") or ""
|
||
if not rid:
|
||
continue
|
||
kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
|
||
url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
|
||
out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
|
||
"", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
|
||
return out
|
||
|
||
|
||
def _hf_models(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
|
||
"", "", x.get("createdAt", ""),
|
||
{"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
|
||
return out
|
||
|
||
|
||
def _hf_daily_papers(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
p = x.get("paper") or x
|
||
pid = p.get("id") or ""
|
||
out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
|
||
p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
|
||
p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
|
||
return out
|
||
|
||
|
||
def _hf_rows(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for row in (d.get("rows") or [])[:limit]:
|
||
r = row.get("row") or {}
|
||
name = r.get("fullname") or r.get("model") or r.get("Model") or ""
|
||
if not name:
|
||
continue
|
||
out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
|
||
"", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
|
||
return out
|
||
|
||
|
||
def _openalex(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for w in (d.get("results") or [])[:limit]:
|
||
out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
|
||
w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
|
||
{"cited": w.get("cited_by_count")}))
|
||
return out
|
||
|
||
|
||
def _s2(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for p in (d.get("data") or [])[:limit]:
|
||
out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
|
||
", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
|
||
str(p.get("year", "")), {"citations": p.get("citationCount")}))
|
||
return out
|
||
|
||
|
||
def _samsung(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
|
||
x.get("title", ""), x.get("summary") or x.get("content", ""),
|
||
"", str(x.get("regDate") or x.get("date") or ""), {}))
|
||
return out
|
||
|
||
|
||
def _lg_ai(src, limit):
|
||
d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
|
||
x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
|
||
return out
|
||
|
||
|
||
def _github_org(src, limit):
|
||
hdr = dict(UA)
|
||
tok = config.KEYS.get("GITHUB_TOKEN")
|
||
if tok:
|
||
hdr["Authorization"] = f"token {tok}"
|
||
url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
|
||
d = net.get(url, headers=hdr, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
|
||
x.get("description", ""), x.get("owner", {}).get("login", ""),
|
||
x.get("pushed_at", ""),
|
||
{"stars": x.get("stargazers_count"), "lang": x.get("language")}))
|
||
return out
|
||
|
||
|
||
def _discourse(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
base = src.url.split("/latest.json")[0]
|
||
out = []
|
||
for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
|
||
out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
|
||
t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
|
||
"", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
|
||
return out
|
||
|
||
|
||
def _xenforo_latest(src, limit):
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
out = []
|
||
for a in soup.select("a[href*='/threads/']"):
|
||
title = a.get_text(" ", strip=True)
|
||
href = str(a.get("href", "") or "")
|
||
if len(title) < 12 or href.rstrip("/").endswith("/threads"):
|
||
continue
|
||
if not href.startswith("http"):
|
||
parts = urllib.parse.urlsplit(src.url)
|
||
href = "%s://%s%s" % (parts.scheme, parts.netloc, href)
|
||
out.append(_item(src, "thread", href, title))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
SERVICE_RE = re.compile(
|
||
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
|
||
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
|
||
|
||
|
||
def _tg_web(src, limit):
|
||
"""Публичное веб-превью t.me/s/<канал>.
|
||
|
||
Отсекаем служебное (Channel created / photo updated / pinned) и пустыши:
|
||
раньше в базу уезжали заголовки "@upenbullet" и "Channel photo removed"."""
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
out = []
|
||
for msg in soup.select("div.tgme_widget_message_wrap")[-max(limit * 2, 20):]:
|
||
if msg.select_one("div.tgme_widget_message_service"):
|
||
continue
|
||
text_el = msg.select_one("div.tgme_widget_message_text")
|
||
if not text_el:
|
||
continue
|
||
text = text_el.get_text("\n", strip=True)
|
||
text = re.sub(r"\n{2,}", "\n", text).strip()
|
||
# лив-каналы пишут коротко («<домен> 2M users»), им порог ниже
|
||
min_len = 12 if "leak" in src.tags else 25
|
||
min_alpha = 8 if "leak" in src.tags else 15
|
||
if len(text) < min_len:
|
||
continue
|
||
if SERVICE_RE.match(text):
|
||
continue
|
||
if sum(ch.isalpha() for ch in text) < min_alpha: # только ссылки/эмодзи/цифры
|
||
continue
|
||
link_el = msg.select_one("a.tgme_widget_message_date")
|
||
link = str(link_el.get("href") or src.url) if link_el else src.url
|
||
dt = ""
|
||
t_el = msg.select_one("time[datetime]")
|
||
if t_el:
|
||
dt = str(t_el.get("datetime") or "")
|
||
views = ""
|
||
v_el = msg.select_one("span.tgme_widget_message_views")
|
||
if v_el:
|
||
views = v_el.get_text(" ", strip=True)
|
||
via = ""
|
||
vv = msg.select_one("span.tgme_widget_message_via")
|
||
if vv:
|
||
via = vv.get_text(" ", strip=True)
|
||
lines = [ln.strip() for ln in text.split("\n")
|
||
if ln.strip() and not ln.strip().startswith(("http", "t.co/"))]
|
||
lines = [ln for ln in lines if sum(ch.isalpha() for ch in ln) >= 12]
|
||
if not lines:
|
||
continue
|
||
first = [ln for ln in lines[:6] if len(ln) >= 20]
|
||
head = (first[0] if first else max(lines[:6], key=len)).strip(' ,.;:-——')
|
||
out.append(_item(src, "post", link, head[:140], text, via, dt[:10],
|
||
{"views": views, "handle": src.url.rsplit("/", 1)[-1], "lang": ""}))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
def _html_listing(src, limit):
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
inc = src.extra.get("include_prefixes") or []
|
||
inc_c = src.extra.get("include_contains") or []
|
||
exc = set(src.extra.get("exclude_exact") or [])
|
||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||
seen, links = set(), []
|
||
for a in soup.find_all("a", href=True):
|
||
href = urllib.parse.urljoin(base, str(a["href"]))
|
||
path = urllib.parse.urlsplit(href).path
|
||
if path in exc or len(path) < 6:
|
||
continue
|
||
if inc and not any(path.startswith(p) for p in inc):
|
||
continue
|
||
if inc_c and not any(c in path for c in inc_c):
|
||
continue
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
links.append((href, a.get_text(" ", strip=True)))
|
||
if len(links) >= limit:
|
||
break
|
||
out = []
|
||
for href, title in links:
|
||
text = ""
|
||
if src.detail:
|
||
d = net.get(href, headers=UA)
|
||
if d.status == 200:
|
||
ds = BeautifulSoup(d.text, "lxml")
|
||
for bad in ds(["script", "style", "nav", "footer", "header"]):
|
||
bad.decompose()
|
||
paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
|
||
text = " ".join(paras)[:4000]
|
||
if not title:
|
||
t = ds.find("h1") or ds.title
|
||
title = t.get_text(" ", strip=True) if t else href
|
||
out.append(_item(src, "entry", href, title or href, text))
|
||
return out
|
||
|
||
|
||
def _json_of(url, timeout=35):
|
||
"""JSON без зависимостей от конкретного клиента: net.get + json.loads."""
|
||
try:
|
||
r = net.get(url, json_mode=True, timeout=timeout)
|
||
except Exception:
|
||
return None
|
||
if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
|
||
return None
|
||
try:
|
||
return json.loads(r.text)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _generic_json(src, limit):
|
||
"""Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам."""
|
||
d = _json_of(src.url)
|
||
rows = []
|
||
if isinstance(d, list):
|
||
rows = d
|
||
elif isinstance(d, dict):
|
||
for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"):
|
||
v = d.get(key)
|
||
if isinstance(v, list):
|
||
rows = v
|
||
break
|
||
if not rows:
|
||
for v in d.values():
|
||
if isinstance(v, list) and v and isinstance(v[0], dict):
|
||
rows = v
|
||
break
|
||
out = []
|
||
for r in rows[:limit]:
|
||
if not isinstance(r, dict):
|
||
continue
|
||
title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id")
|
||
url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id")
|
||
if isinstance(url, str) and url and not url.startswith("http"):
|
||
if "github.com" in src.url:
|
||
url = "https://github.com/" + url.lstrip("/")
|
||
elif "huggingface.co" in src.url:
|
||
url = "https://huggingface.co/" + url.lstrip("/")
|
||
text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or ""
|
||
if isinstance(text, dict):
|
||
text = json.dumps(text, ensure_ascii=False)[:400]
|
||
published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or ""
|
||
if not title:
|
||
continue
|
||
out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800],
|
||
str(r.get("author") or r.get("owner") or ""), str(published)[:24]))
|
||
return out
|
||
|
||
|
||
def _or_models(src, limit):
|
||
d = (_json_of(src.url) or {}).get("data", [])
|
||
out = []
|
||
for m in d[:limit]:
|
||
mid = m.get("id") or ""
|
||
pricing = m.get("pricing") or {}
|
||
price = None
|
||
try:
|
||
price = round(float(pricing.get("prompt", 0)) * 1_000_000, 3)
|
||
except Exception:
|
||
price = None
|
||
name = m.get("name") or mid
|
||
ctx = m.get("context_length") or 0
|
||
out.append(_item(src, "model", f"https://openrouter.ai/models/{mid}", f"{name}",
|
||
f"context {ctx} · prompt ${price}/1M", "", str(m.get("created") or ""),
|
||
extra={"price": price}))
|
||
return out
|
||
|
||
|
||
def _cisa_kev(src, limit):
|
||
d = _json_of(src.url, timeout=45) or {}
|
||
vulns = d.get("vulnerabilities", [])
|
||
vulns = sorted(vulns, key=lambda v: v.get("dateAdded") or "", reverse=True)
|
||
out = []
|
||
for v in vulns[:limit]:
|
||
cve = v.get("cveID") or ""
|
||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cve}",
|
||
f"{cve} · {v.get('vendorProject','')} {v.get('product','')}".strip(),
|
||
(v.get("shortDescription") or "")[:600],
|
||
v.get("vendorProject") or "", v.get("dateAdded") or ""))
|
||
return out
|
||
|
||
|
||
def _nvd(src, limit):
|
||
d = _json_of(src.url, timeout=45) or {}
|
||
out = []
|
||
for v in d.get("vulnerabilities", [])[:limit]:
|
||
c = v.get("cve", {})
|
||
cid = c.get("id") or ""
|
||
desc = ""
|
||
for dd in c.get("descriptions", []):
|
||
if dd.get("lang") == "en":
|
||
desc = dd.get("value") or ""
|
||
break
|
||
sev = ""
|
||
metrics = c.get("metrics", {})
|
||
for key in ("cvssMetricV31", "cvssMetricV40", "cvssMetricV2"):
|
||
if metrics.get(key):
|
||
try:
|
||
sev = str(metrics[key][0]["cvssData"].get("baseScore"))
|
||
except Exception:
|
||
sev = ""
|
||
break
|
||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cid}",
|
||
f"{cid}{(' · CVSS ' + sev) if sev else ''}", desc[:600],
|
||
"", c.get("published") or ""))
|
||
return out
|
||
|
||
|
||
|
||
def _biorxiv(src, limit):
|
||
import json as _j
|
||
d = _json_of(src.url)
|
||
rows = d.get("collection") if isinstance(d, dict) else d
|
||
out = []
|
||
for r in (rows or [])[:limit]:
|
||
title = r.get("title") or ""
|
||
doi = r.get("doi") or ""
|
||
if not title:
|
||
continue
|
||
ver = r.get("version") or "1"
|
||
host = "medrxiv" if "medrxiv" in src.url else "biorxiv"
|
||
url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url
|
||
out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800],
|
||
str(r.get("authors") or ""), str(r.get("date") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _dblp(src, limit):
|
||
d = _json_of(src.url)
|
||
hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or []
|
||
out = []
|
||
for h in hits[:limit]:
|
||
info = h.get("info") or {}
|
||
title = info.get("title") or ""
|
||
if not title:
|
||
continue
|
||
auth = info.get("authors") or {}
|
||
if isinstance(auth, dict):
|
||
a = auth.get("author")
|
||
if isinstance(a, list):
|
||
auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a)
|
||
elif isinstance(a, dict):
|
||
auth = a.get("text", "")
|
||
out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""),
|
||
str(auth or ""), str(info.get("year") or "")))
|
||
return out
|
||
|
||
|
||
def _hal(src, limit):
|
||
d = _json_of(src.url)
|
||
docs = ((d or {}).get("response") or {}).get("docs") or []
|
||
out = []
|
||
for r in docs[:limit]:
|
||
t = r.get("title_s")
|
||
title = (t[0] if isinstance(t, list) and t else t) or ""
|
||
if not title:
|
||
continue
|
||
ab = r.get("abstract_s")
|
||
if isinstance(ab, list):
|
||
ab = ab[0] if ab else ""
|
||
out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "",
|
||
str(r.get("producedDateY_i") or "")))
|
||
return out
|
||
|
||
|
||
def _zenodo(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = ((d or {}).get("hits") or {}).get("hits") or []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
md = r.get("metadata") or {}
|
||
title = md.get("title") or ""
|
||
if not title:
|
||
continue
|
||
url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or ""
|
||
out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800],
|
||
str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""),
|
||
str(md.get("publication_date") or "")))
|
||
return out
|
||
|
||
|
||
def _cve_circl(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = d if isinstance(d, list) else []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
cid = r.get("id") or ""
|
||
if not cid:
|
||
continue
|
||
out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140],
|
||
str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _ghsa(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
gid = r.get("ghsa_id") or ""
|
||
title = r.get("summary") or gid
|
||
if not title:
|
||
continue
|
||
sev = r.get("severity") or ""
|
||
out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title),
|
||
(sev + " | " + (r.get("description") or ""))[:800], "",
|
||
str(r.get("published_at") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _models_dev(src, limit):
|
||
d = _json_of(src.url)
|
||
out = []
|
||
if not isinstance(d, dict):
|
||
return out
|
||
for pid, prov in d.items():
|
||
if not isinstance(prov, dict):
|
||
continue
|
||
pname = prov.get("name") or pid
|
||
models = prov.get("models") or {}
|
||
if isinstance(models, dict):
|
||
iterable = models.items()
|
||
elif isinstance(models, list):
|
||
iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)]
|
||
else:
|
||
continue
|
||
for mid, m in iterable:
|
||
if not isinstance(m, dict):
|
||
continue
|
||
name = m.get("name") or mid
|
||
out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name),
|
||
"ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"),
|
||
(m.get("cost") or {}).get("input", "?"),
|
||
(m.get("cost") or {}).get("output", "?")),
|
||
str(pid), ""))
|
||
if len(out) >= limit:
|
||
return out
|
||
return out
|
||
|
||
|
||
def _x_user(src, limit=12):
|
||
"""Лента X-аккаунта через api.fxtwitter.com (без ключей)."""
|
||
from .providers import x as xp
|
||
handle = (src.extra or {}).get("handle") or src.url.rsplit("/", 1)[-1]
|
||
return xp.profile(handle, limit=max(4, min(limit, 25)))
|
||
|
||
|
||
ADAPTERS = {
|
||
"x_user": _x_user,
|
||
"rss": _rss,
|
||
"generic_json": _generic_json,
|
||
"or_models": _or_models,
|
||
"cisa_kev": _cisa_kev,
|
||
"nvd": _nvd,
|
||
"hn_top": _hn_top,
|
||
"reddit_listing": _reddit_listing,
|
||
"hf_trending": _hf_generic,
|
||
"hf_generic": _hf_generic,
|
||
"hf_models": _hf_models,
|
||
"hf_daily_papers": _hf_daily_papers,
|
||
"hf_rows": _hf_rows,
|
||
"openalex": _openalex,
|
||
"s2": _s2,
|
||
"samsung": _samsung,
|
||
"lg_ai": _lg_ai,
|
||
"github_org": _github_org,
|
||
"discourse": _discourse,
|
||
"xenforo_latest": _xenforo_latest,
|
||
"tg_web": _tg_web,
|
||
"html_listing": _html_listing,
|
||
"biorxiv": _biorxiv,
|
||
"dblp": _dblp,
|
||
"hal": _hal,
|
||
"zenodo": _zenodo,
|
||
"cve_circl": _cve_circl,
|
||
"ghsa": _ghsa,
|
||
"models_dev": _models_dev,
|
||
|
||
}
|
||
|
||
|
||
def fetch(src, limit=None):
|
||
fn = ADAPTERS.get(src.adapter)
|
||
if not fn:
|
||
return []
|
||
n = limit or src.limit
|
||
try:
|
||
return fn(src, n) or []
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
|
||
|
||
|
||
def fetch_many(sources, limit=None, workers=8):
|
||
"""Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
|
||
fast = [s for s in sources if s not in slow]
|
||
items, report = [], []
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
futs = [(ex.submit(fetch, s, limit), s) for s in fast]
|
||
for f, s in futs:
|
||
got = f.result()
|
||
report.append((s.name, len(got)))
|
||
items.extend(got)
|
||
for s in slow:
|
||
got = fetch(s, limit)
|
||
if not got and "reddit.com" in s.url:
|
||
time.sleep(7)
|
||
got = fetch(s, limit)
|
||
report.append((s.name, len(got)))
|
||
items.extend(got)
|
||
time.sleep(3.5 if "reddit.com" in s.url else 1.0)
|
||
return items, report |