omni: веб-панель (serve + /api), дайджест, arxiv API-fallback, HF trending unwrap

omni/feeds.py

@@ -33,6 +33,33 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
 
 
 # ---------------- rss ----------------
+def _arxiv_api(src, limit):
+    """arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
+    m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
+    cat = m.group(1) if m else "cs.AI"
+    url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
+           "&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
+    r = net.get(url, headers=UA)
+    if r.status != 200 or "<entry" not in (r.text or ""):
+        return []
+    out = []
+    try:
+        root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
+    except Exception:
+        return []
+    ns = {"a": "http://www.w3.org/2005/Atom"}
+    for e in root.findall(".//a:entry", ns)[:limit]:
+        title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
+        summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
+        link = e.findtext("a:id", namespaces=ns) or ""
+        pub = e.findtext("a:published", namespaces=ns) or ""
+        auth = e.findtext("a:author/a:name", namespaces=ns) or ""
+        if title:
+            out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
+                             {"cat": cat}))
+    return out
+
+
 def _rss(src, limit):
     r = net.get(src.url, headers=UA)
     if r.status in (429, 503):
@@ -40,7 +67,7 @@ def _rss(src, limit):
         time.sleep(2.5)
         r = net.get(src.url, headers=UA)
     if r.status != 200 or not r.text.strip():
-        return []
+        return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
     out = []
     try:
         root = ET.fromstring(r.text.encode("utf-8", "ignore"))
@@ -57,6 +84,8 @@ def _rss(src, limit):
         body = g("description") or g("summary") or g("content")
         out.append(_item(src, "entry", link, g("title"), body,
                          g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
+    if not out and "arxiv.org" in src.url:
+        return _arxiv_api(src, limit)
     return out
 
 
@@ -103,6 +132,8 @@ def _reddit_listing(src, limit):
                 return _reddit_atom(src, rr.text, limit)
     r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
     out = []
+    if r.status == 200 and isinstance(r.json(), dict):
+        children = (((r.json() or {}).get("data") or {}).get("children") or [])
     for ch in children[:limit]:
         d = ch.get("data") or {}
         if not d.get("title"):
@@ -158,6 +189,8 @@ def _hf_generic(src, limit):
     for x in rows[:limit]:
         if not isinstance(x, dict):
             continue
+        if "repoData" in x:  # trending: {repoType, repoData:{...}}
+            x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
         rid = x.get("id") or x.get("repoId") or x.get("name") or ""
         if not rid:
             continue