omni: менеджер источников в панели (338 источников, вкл/выкл, добавление, health-check, фоновые джобы)

omni/feeds.py

@@ -60,8 +60,24 @@ def _arxiv_api(src, limit):
     return out
 
 
+_FEED_CACHE = None
+
+
+def _cache_url(src):
+    global _FEED_CACHE
+    if _FEED_CACHE is None:
+        try:
+            import pathlib as _pl
+            f = _pl.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
+            _FEED_CACHE = __import__("json").loads(f.read_text(encoding="utf-8")) if f.exists() else {}
+        except Exception:
+            _FEED_CACHE = {}
+    hit = _FEED_CACHE.get(src.name)
+    return hit.get("url") if isinstance(hit, dict) and hit.get("url") else src.url
+
+
 def _rss(src, limit):
-    r = net.get(src.url, headers=UA)
+    r = net.get(_cache_url(src), headers=UA)
     if r.status in (429, 503):
         import time
         time.sleep(2.5)
@@ -387,8 +403,124 @@ def _html_listing(src, limit):
     return out
 
 
+def _json_of(url, timeout=35):
+    """JSON без зависимостей от конкретного клиента: net.get + json.loads."""
+    try:
+        r = net.get(url, json_mode=True, timeout=timeout)
+    except Exception:
+        return None
+    if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
+        return None
+    try:
+        return json.loads(r.text)
+    except Exception:
+        return None
+
+
+def _generic_json(src, limit):
+    """Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам."""
+    d = _json_of(src.url)
+    rows = []
+    if isinstance(d, list):
+        rows = d
+    elif isinstance(d, dict):
+        for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"):
+            v = d.get(key)
+            if isinstance(v, list):
+                rows = v
+                break
+        if not rows:
+            for v in d.values():
+                if isinstance(v, list) and v and isinstance(v[0], dict):
+                    rows = v
+                    break
+    out = []
+    for r in rows[:limit]:
+        if not isinstance(r, dict):
+            continue
+        title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id")
+        url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id")
+        if isinstance(url, str) and url and not url.startswith("http"):
+            if "github.com" in src.url:
+                url = "https://github.com/" + url.lstrip("/")
+            elif "huggingface.co" in src.url:
+                url = "https://huggingface.co/" + url.lstrip("/")
+        text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or ""
+        if isinstance(text, dict):
+            text = json.dumps(text, ensure_ascii=False)[:400]
+        published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or ""
+        if not title:
+            continue
+        out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800],
+                         str(r.get("author") or r.get("owner") or ""), str(published)[:24]))
+    return out
+
+
+def _or_models(src, limit):
+    d = (_json_of(src.url) or {}).get("data", [])
+    out = []
+    for m in d[:limit]:
+        mid = m.get("id") or ""
+        pricing = m.get("pricing") or {}
+        price = None
+        try:
+            price = round(float(pricing.get("prompt", 0)) * 1_000_000, 3)
+        except Exception:
+            price = None
+        name = m.get("name") or mid
+        ctx = m.get("context_length") or 0
+        out.append(_item(src, "model", f"https://openrouter.ai/models/{mid}", f"{name}",
+                         f"context {ctx} · prompt ${price}/1M", "", str(m.get("created") or ""),
+                         extra={"price": price}))
+    return out
+
+
+def _cisa_kev(src, limit):
+    d = _json_of(src.url, timeout=45) or {}
+    vulns = d.get("vulnerabilities", [])
+    vulns = sorted(vulns, key=lambda v: v.get("dateAdded") or "", reverse=True)
+    out = []
+    for v in vulns[:limit]:
+        cve = v.get("cveID") or ""
+        out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cve}",
+                         f"{cve} · {v.get('vendorProject','')} {v.get('product','')}".strip(),
+                         (v.get("shortDescription") or "")[:600],
+                         v.get("vendorProject") or "", v.get("dateAdded") or ""))
+    return out
+
+
+def _nvd(src, limit):
+    d = _json_of(src.url, timeout=45) or {}
+    out = []
+    for v in d.get("vulnerabilities", [])[:limit]:
+        c = v.get("cve", {})
+        cid = c.get("id") or ""
+        desc = ""
+        for dd in c.get("descriptions", []):
+            if dd.get("lang") == "en":
+                desc = dd.get("value") or ""
+                break
+        sev = ""
+        metrics = c.get("metrics", {})
+        for key in ("cvssMetricV31", "cvssMetricV40", "cvssMetricV2"):
+            if metrics.get(key):
+                try:
+                    sev = str(metrics[key][0]["cvssData"].get("baseScore"))
+                except Exception:
+                    sev = ""
+                break
+        out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cid}",
+                         f"{cid}{(' · CVSS ' + sev) if sev else ''}", desc[:600],
+                         "", c.get("published") or ""))
+    return out
+
+
 ADAPTERS = {
     "rss": _rss,
+    "generic_json": _generic_json,
+    "or_models": _or_models,
+    "cisa_kev": _cisa_kev,
+    "nvd": _nvd,
     "hn_top": _hn_top,
     "reddit_listing": _reddit_listing,
     "hf_trending": _hf_generic,