reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

omni: реестр 113 источников (papers/company/cn/kr/community/security/forum/models/benchmark) + команды sources и feed

Browse Source
This commit is contained in:
dev
2026-10-25 18:54:33 +03:00
parent 02816cacfb
commit c4d188520f
4 changed files with 683 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

413
omni/feeds.py Normal file
View File

@@ -0,0 +1,413 @@
"""Забор источников из реестра sources.py.
Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
"""
import html
import json
import re
import urllib.parse
import xml.etree.ElementTree as ET
from bs4 import BeautifulSoup
from . import config, net
from .model import Item
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
"Accept-Language": "en,ru;q=0.8"}
def _txt(x):
if x is None:
return ""
return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()
def _item(src, kind, url, title, text="", author="", published="", extra=None):
ex = dict(extra or {})
ex["tags"] = list(src.tags)
return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
author=_txt(author)[:80], published=str(published or ""), source=src.category,
extra=ex)
# ---------------- rss ----------------
def _rss(src, limit):
r = net.get(src.url, headers=UA)
if r.status in (429, 503):
import time
time.sleep(2.5)
r = net.get(src.url, headers=UA)
if r.status != 200 or not r.text.strip():
return []
out = []
try:
root = ET.fromstring(r.text.encode("utf-8", "ignore"))
except Exception:
return _rss_regex(src, r.text, limit)
ns = {"a": "http://www.w3.org/2005/Atom"}
entries = root.findall(".//item") or root.findall(".//a:entry", ns)
for e in entries[:limit]:
g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
link = g("link")
if not link:
le = e.find("a:link", ns) or e.find("link")
link = le.get("href") if le is not None and le.get("href") else ""
body = g("description") or g("summary") or g("content")
out.append(_item(src, "entry", link, g("title"), body,
g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
return out
def _rss_regex(src, text, limit):
out = []
for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
blob = m.group(1)
g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
if len(out) >= limit:
break
return out
# ---------------- json adapters ----------------
def _hn_top(src, limit):
ids = net.get(src.url, headers=UA, json_mode=True).json() or []
out = []
for sid in ids[:limit]:
d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
if not d.get("title"):
continue
out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
d.get("title"), d.get("text", ""), d.get("by", ""),
str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
"hn": f"https://news.ycombinator.com/item?id={sid}"}))
return out
def _reddit_listing(src, limit):
"""Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
import time
children = []
if rss_first:
rr = net.get(rss_first, headers=UA)
if rr.status == 200 and "<entry" in (rr.text or ""):
return _reddit_atom(src, rr.text, limit)
if rr.status in (429, 503):
time.sleep(4)
rr = net.get(rss_first, headers=UA)
if rr.status == 200 and "<entry" in (rr.text or ""):
return _reddit_atom(src, rr.text, limit)
r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
out = []
for ch in children[:limit]:
d = ch.get("data") or {}
if not d.get("title"):
continue
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
d.get("title"), d.get("selftext", ""), d.get("author", ""),
str(d.get("created_utc", "")),
{"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
if out:
return out
rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
r2 = net.get(rss_url, headers=UA) if not rss_first else None
if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
return _reddit_atom(src, r2.text, limit)
sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
if sub:
alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
f"&size={limit}&sort=desc", headers=UA, json_mode=True)
for d in ((alt.json() or {}).get("data") or [])[:limit]:
if d.get("title"):
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
d.get("title"), d.get("selftext", ""), d.get("author", ""),
str(d.get("created_utc", "")), {"ups": d.get("score")}))
return out
def _reddit_atom(src, text, limit):
out = []
try:
root = ET.fromstring(text.encode("utf-8", "ignore"))
except Exception:
return out
ns = {"a": "http://www.w3.org/2005/Atom"}
for e in root.findall(".//a:entry", ns)[:limit]:
title = e.findtext("a:title", namespaces=ns) or ""
link_el = e.find("a:link", ns)
link = link_el.get("href") if link_el is not None else ""
content = e.findtext("a:content", namespaces=ns) or ""
author = e.findtext("a:author/a:name", namespaces=ns) or ""
upd = e.findtext("a:updated", namespaces=ns) or ""
if title:
out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
return out
def _hf_generic(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
or d.get("spaces") or d.get("datasets") or [])
out = []
for x in rows[:limit]:
if not isinstance(x, dict):
continue
rid = x.get("id") or x.get("repoId") or x.get("name") or ""
if not rid:
continue
kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
"", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
return out
def _hf_models(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or []
out = []
for x in (d if isinstance(d, list) else [])[:limit]:
out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
"", "", x.get("createdAt", ""),
{"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
return out
def _hf_daily_papers(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or []
out = []
for x in (d if isinstance(d, list) else [])[:limit]:
p = x.get("paper") or x
pid = p.get("id") or ""
out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
return out
def _hf_rows(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
out = []
for row in (d.get("rows") or [])[:limit]:
r = row.get("row") or {}
name = r.get("fullname") or r.get("model") or r.get("Model") or ""
if not name:
continue
out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
"", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
return out
def _openalex(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
out = []
for w in (d.get("results") or [])[:limit]:
out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
{"cited": w.get("cited_by_count")}))
return out
def _s2(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
out = []
for p in (d.get("data") or [])[:limit]:
out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
str(p.get("year", "")), {"citations": p.get("citationCount")}))
return out
def _samsung(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
out = []
for x in rows[:limit]:
if not isinstance(x, dict):
continue
out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
x.get("title", ""), x.get("summary") or x.get("content", ""),
"", str(x.get("regDate") or x.get("date") or ""), {}))
return out
def _lg_ai(src, limit):
d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
out = []
for x in rows[:limit]:
if not isinstance(x, dict):
continue
out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
return out
def _github_org(src, limit):
hdr = dict(UA)
tok = config.KEYS.get("GITHUB_TOKEN")
if tok:
hdr["Authorization"] = f"token {tok}"
url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
d = net.get(url, headers=hdr, json_mode=True).json() or []
out = []
for x in (d if isinstance(d, list) else [])[:limit]:
out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
x.get("description", ""), x.get("owner", {}).get("login", ""),
x.get("pushed_at", ""),
{"stars": x.get("stargazers_count"), "lang": x.get("language")}))
return out
def _discourse(src, limit):
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
base = src.url.split("/latest.json")[0]
out = []
for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
"", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
return out
def _xenforo_latest(src, limit):
r = net.get(src.url, headers=UA)
if r.status != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
out = []
for a in soup.select("a[href*='/threads/']"):
title = a.get_text(" ", strip=True)
href = str(a.get("href", "") or "")
if len(title) < 12 or href.rstrip("/").endswith("/threads"):
continue
if not href.startswith("http"):
href = "https://www.blackhatworld.com" + href
out.append(_item(src, "thread", href, title))
if len(out) >= limit:
break
return out
def _tg_web(src, limit):
r = net.get(src.url, headers=UA)
if r.status != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
out = []
for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]:
text_el = msg.select_one("div.tgme_widget_message_text")
link_el = msg.select_one("a.tgme_widget_message_date")
text = text_el.get_text("\n", strip=True) if text_el else ""
link = str(link_el.get("href") or src.url) if link_el else src.url
if not text:
continue
out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {}))
return out
def _html_listing(src, limit):
r = net.get(src.url, headers=UA)
if r.status != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
inc = src.extra.get("include_prefixes") or []
inc_c = src.extra.get("include_contains") or []
exc = set(src.extra.get("exclude_exact") or [])
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
seen, links = set(), []
for a in soup.find_all("a", href=True):
href = urllib.parse.urljoin(base, str(a["href"]))
path = urllib.parse.urlsplit(href).path
if path in exc or len(path) < 6:
continue
if inc and not any(path.startswith(p) for p in inc):
continue
if inc_c and not any(c in path for c in inc_c):
continue
if href in seen:
continue
seen.add(href)
links.append((href, a.get_text(" ", strip=True)))
if len(links) >= limit:
break
out = []
for href, title in links:
text = ""
if src.detail:
d = net.get(href, headers=UA)
if d.status == 200:
ds = BeautifulSoup(d.text, "lxml")
for bad in ds(["script", "style", "nav", "footer", "header"]):
bad.decompose()
paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
text = " ".join(paras)[:4000]
if not title:
t = ds.find("h1") or ds.title
title = t.get_text(" ", strip=True) if t else href
out.append(_item(src, "entry", href, title or href, text))
return out
ADAPTERS = {
"rss": _rss,
"hn_top": _hn_top,
"reddit_listing": _reddit_listing,
"hf_trending": _hf_generic,
"hf_generic": _hf_generic,
"hf_models": _hf_models,
"hf_daily_papers": _hf_daily_papers,
"hf_rows": _hf_rows,
"openalex": _openalex,
"s2": _s2,
"samsung": _samsung,
"lg_ai": _lg_ai,
"github_org": _github_org,
"discourse": _discourse,
"xenforo_latest": _xenforo_latest,
"tg_web": _tg_web,
"html_listing": _html_listing,
}
def fetch(src, limit=None):
fn = ADAPTERS.get(src.adapter)
if not fn:
return []
n = limit or src.limit
try:
return fn(src, n) or []
except Exception:
return []
SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
def fetch_many(sources, limit=None, workers=8):
"""Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
import time
from concurrent.futures import ThreadPoolExecutor
slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
fast = [s for s in sources if s not in slow]
items, report = [], []
with ThreadPoolExecutor(max_workers=workers) as ex:
futs = [(ex.submit(fetch, s, limit), s) for s in fast]
for f, s in futs:
got = f.result()
report.append((s.name, len(got)))
items.extend(got)
for s in slow:
got = fetch(s, limit)
if not got and "reddit.com" in s.url:
time.sleep(7)
got = fetch(s, limit)
report.append((s.name, len(got)))
items.extend(got)
time.sleep(3.5 if "reddit.com" in s.url else 1.0)
return items, report