omni: редизайн панели + 450 источников (341 живых), health-аудит, чистка html/дат, per-source учёт

omni/feeds.py

@@ -28,7 +28,7 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
     ex = dict(extra or {})
     ex["tags"] = list(src.tags)
     return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
-                author=_txt(author)[:80], published=str(published or ""), source=src.category,
+                author=_txt(author)[:80], published=str(published or ""), source=src.name,
                 extra=ex)
 
 
@@ -515,6 +515,138 @@ def _nvd(src, limit):
     return out
 
 
+
+def _biorxiv(src, limit):
+    import json as _j
+    d = _json_of(src.url)
+    rows = d.get("collection") if isinstance(d, dict) else d
+    out = []
+    for r in (rows or [])[:limit]:
+        title = r.get("title") or ""
+        doi = r.get("doi") or ""
+        if not title:
+            continue
+        ver = r.get("version") or "1"
+        host = "medrxiv" if "medrxiv" in src.url else "biorxiv"
+        url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url
+        out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800],
+                         str(r.get("authors") or ""), str(r.get("date") or "")[:24]))
+    return out
+
+
+def _dblp(src, limit):
+    d = _json_of(src.url)
+    hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or []
+    out = []
+    for h in hits[:limit]:
+        info = h.get("info") or {}
+        title = info.get("title") or ""
+        if not title:
+            continue
+        auth = info.get("authors") or {}
+        if isinstance(auth, dict):
+            a = auth.get("author")
+            if isinstance(a, list):
+                auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a)
+            elif isinstance(a, dict):
+                auth = a.get("text", "")
+        out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""),
+                         str(auth or ""), str(info.get("year") or "")))
+    return out
+
+
+def _hal(src, limit):
+    d = _json_of(src.url)
+    docs = ((d or {}).get("response") or {}).get("docs") or []
+    out = []
+    for r in docs[:limit]:
+        t = r.get("title_s")
+        title = (t[0] if isinstance(t, list) and t else t) or ""
+        if not title:
+            continue
+        ab = r.get("abstract_s")
+        if isinstance(ab, list):
+            ab = ab[0] if ab else ""
+        out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "",
+                         str(r.get("producedDateY_i") or "")))
+    return out
+
+
+def _zenodo(src, limit):
+    d = _json_of(src.url)
+    rows = ((d or {}).get("hits") or {}).get("hits") or []
+    out = []
+    for r in rows[:limit]:
+        md = r.get("metadata") or {}
+        title = md.get("title") or ""
+        if not title:
+            continue
+        url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or ""
+        out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800],
+                         str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""),
+                         str(md.get("publication_date") or "")))
+    return out
+
+
+def _cve_circl(src, limit):
+    d = _json_of(src.url)
+    rows = d if isinstance(d, list) else []
+    out = []
+    for r in rows[:limit]:
+        cid = r.get("id") or ""
+        if not cid:
+            continue
+        out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140],
+                         str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24]))
+    return out
+
+
+def _ghsa(src, limit):
+    d = _json_of(src.url)
+    rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else []
+    out = []
+    for r in rows[:limit]:
+        gid = r.get("ghsa_id") or ""
+        title = r.get("summary") or gid
+        if not title:
+            continue
+        sev = r.get("severity") or ""
+        out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title),
+                         (sev + " | " + (r.get("description") or ""))[:800], "",
+                         str(r.get("published_at") or "")[:24]))
+    return out
+
+
+def _models_dev(src, limit):
+    d = _json_of(src.url)
+    out = []
+    if not isinstance(d, dict):
+        return out
+    for pid, prov in d.items():
+        if not isinstance(prov, dict):
+            continue
+        pname = prov.get("name") or pid
+        models = prov.get("models") or {}
+        if isinstance(models, dict):
+            iterable = models.items()
+        elif isinstance(models, list):
+            iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)]
+        else:
+            continue
+        for mid, m in iterable:
+            if not isinstance(m, dict):
+                continue
+            name = m.get("name") or mid
+            out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name),
+                             "ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"),
+                                                            (m.get("cost") or {}).get("input", "?"),
+                                                            (m.get("cost") or {}).get("output", "?")),
+                             str(pid), ""))
+            if len(out) >= limit:
+                return out
+    return out
+
+
 ADAPTERS = {
     "rss": _rss,
     "generic_json": _generic_json,
@@ -537,6 +669,14 @@ ADAPTERS = {
     "xenforo_latest": _xenforo_latest,
     "tg_web": _tg_web,
     "html_listing": _html_listing,
+    "biorxiv": _biorxiv,
+    "dblp": _dblp,
+    "hal": _hal,
+    "zenodo": _zenodo,
+    "cve_circl": _cve_circl,
+    "ghsa": _ghsa,
+    "models_dev": _models_dev,
+
 }