omni: веб-панель (serve + /api), дайджест, arxiv API-fallback, HF trending unwrap

2026-10-25 19:11:52 +03:00
parent c4d188520f
commit e1368f2a46
6 changed files with 446 additions and 2 deletions
--- a/omni/cli.py
+++ b/omni/cli.py
@@ -1,5 +1,6 @@
 """CLI: providers / search / fanout / cards / top / find / stats / export."""
 import argparse
+import pathlib
 import concurrent.futures
 import json
 import sys
@@ -175,6 +176,26 @@ def cmd_feed(args):
    return 0


+def cmd_digest(args):
+    """Сводка по базе: топ записей по категориям за прогон, в markdown."""
+    rows = store.top(limit=args.limit * 8)
+    by_cat = {}
+    for prov, kind, src, title, sc, price, cur, url in rows:
+        by_cat.setdefault(src or prov, []).append((sc, title, url))
+    out = [f"# дайджест omni · {__import__('datetime').date.today()}", ""]
+    for cat, entries in sorted(by_cat.items(), key=lambda x: -len(x[1])):
+        out.append(f"## {cat} · {len(entries)}")
+        for sc, title, url in entries[:args.limit]:
+            out.append(f"- [{title[:110]}]({url}) · {sc:.2f}")
+        out.append("")
+    text = chr(10).join(out)
+    path = args.out
+    pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True)
+    pathlib.Path(path).write_text(text, encoding="utf-8")
+    print(f"дайджест: {len(by_cat)} категорий, {len(rows)} записей -> {path}")
+    return 0
+
+
 def cmd_export(args):
    n, path = store.export(args.out, args.fmt, args.limit)
    print(f"выгружено {n} записей -> {path}")
@@ -246,6 +267,15 @@ def main(argv=None):
    sp.add_argument("--verbose", action="store_true")
    sp.set_defaults(fn=cmd_feed)

+    sp = sub.add_parser("serve", help="веб-панель + JSON API")
+    sp.add_argument("--port", type=int, default=8077)
+    sp.set_defaults(fn=lambda a: __import__("omni.serve", fromlist=["serve"]).serve(a.port))
+
+    sp = sub.add_parser("digest", help="markdown-дайджест из базы")
+    sp.add_argument("--limit", type=int, default=8)
+    sp.add_argument("--out", default="data/digest.md")
+    sp.set_defaults(fn=cmd_digest)
+
    sp = sub.add_parser("export", help="выгрузка jsonl/csv")
    sp.add_argument("--out", default="data/top.jsonl")
    sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"])
--- a/omni/feeds.py
+++ b/omni/feeds.py
@@ -33,6 +33,33 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):


 # ---------------- rss ----------------
+def _arxiv_api(src, limit):
+    """arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
+    m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
+    cat = m.group(1) if m else "cs.AI"
+    url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
+           "&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
+    r = net.get(url, headers=UA)
+    if r.status != 200 or "<entry" not in (r.text or ""):
+        return []
+    out = []
+    try:
+        root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
+    except Exception:
+        return []
+    ns = {"a": "http://www.w3.org/2005/Atom"}
+    for e in root.findall(".//a:entry", ns)[:limit]:
+        title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
+        summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
+        link = e.findtext("a:id", namespaces=ns) or ""
+        pub = e.findtext("a:published", namespaces=ns) or ""
+        auth = e.findtext("a:author/a:name", namespaces=ns) or ""
+        if title:
+            out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
+                             {"cat": cat}))
+    return out
+
+
 def _rss(src, limit):
    r = net.get(src.url, headers=UA)
    if r.status in (429, 503):
@@ -40,7 +67,7 @@ def _rss(src, limit):
        time.sleep(2.5)
        r = net.get(src.url, headers=UA)
    if r.status != 200 or not r.text.strip():
-        return []
+        return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
    out = []
    try:
        root = ET.fromstring(r.text.encode("utf-8", "ignore"))
@@ -57,6 +84,8 @@ def _rss(src, limit):
        body = g("description") or g("summary") or g("content")
        out.append(_item(src, "entry", link, g("title"), body,
                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
+    if not out and "arxiv.org" in src.url:
+        return _arxiv_api(src, limit)
    return out


@@ -103,6 +132,8 @@ def _reddit_listing(src, limit):
                return _reddit_atom(src, rr.text, limit)
    r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
    out = []
+    if r.status == 200 and isinstance(r.json(), dict):
+        children = (((r.json() or {}).get("data") or {}).get("children") or [])
    for ch in children[:limit]:
        d = ch.get("data") or {}
        if not d.get("title"):
@@ -158,6 +189,8 @@ def _hf_generic(src, limit):
    for x in rows[:limit]:
        if not isinstance(x, dict):
            continue
+        if "repoData" in x:  # trending: {repoType, repoData:{...}}
+            x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
        rid = x.get("id") or x.get("repoId") or x.get("name") or ""
        if not rid:
            continue
--- a/omni/serve.py
+++ b/omni/serve.py
@@ -0,0 +1,111 @@
+"""Веб-панель omni: статика + JSON API поверх той же SQLite.
+
+Запуск: python -m omni serve --port 8077
+"""
+import json
+import pathlib
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from . import feeds, sources, store
+
+WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
+JOBS = {}
+_LOCK = threading.Lock()
+
+
+def _run_feed(job_id, category, limit, tags):
+    picked = sources.pick(category, None, tags)
+    try:
+        items, report = feeds.fetch_many(picked, limit=limit, workers=8)
+        new, dup = store.add(items)
+        with _LOCK:
+            JOBS[job_id] = {"state": "done", "sources": len(picked), "fetched": len(items),
+                            "new": new, "dup": dup, "empty": sum(1 for _, c in report if not c),
+                            "report": sorted(report, key=lambda x: -x[1])[:40]}
+    except Exception as e:
+        with _LOCK:
+            JOBS[job_id] = {"state": "error", "error": f"{type(e).__name__}: {e}"[:200]}
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "omni/1.0"
+
+    def log_message(self, *a):
+        pass
+
+    def _send(self, code, body, ctype="application/json; charset=utf-8"):
+        data = body if isinstance(body, bytes) else str(body).encode("utf-8")
+        self.send_response(code)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def _json(self, obj, code=200):
+        self._send(code, json.dumps(obj, ensure_ascii=False))
+
+    def do_GET(self):
+        u = urllib.parse.urlsplit(self.path)
+        q = urllib.parse.parse_qs(u.query)
+        path = u.path
+
+        if path in ("/", "/index.html"):
+            f = WEB / "index.html"
+            if not f.exists():
+                return self._send(404, "no ui", "text/plain")
+            return self._send(200, f.read_bytes(), "text/html; charset=utf-8")
+
+        if path == "/api/stats":
+            s = store.stats()
+            return self._json({"items": s["items"], "priced": s["priced"], "runs": s["runs"],
+                               "runs_ok": s["runs_ok"],
+                               "by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
+                               "by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
+                               "sources": len(sources.SOURCES), "categories": sources.stats()})
+
+        if path == "/api/sources":
+            return self._json({"total": len(sources.SOURCES), "categories": sources.stats(),
+                               "items": [{"name": s.name, "category": s.category, "kind": s.kind,
+                                          "adapter": s.adapter, "url": s.url, "tags": list(s.tags)}
+                                         for s in sources.SOURCES]})
+
+        if path == "/api/items":
+            cat = (q.get("category") or [None])[0]
+            query = (q.get("q") or [""])[0].strip()
+            limit = min(int((q.get("limit") or ["60"])[0]), 300)
+            offset = int((q.get("offset") or ["0"])[0])
+            rows = store.browse(category=cat, q=query, limit=limit, offset=offset)
+            return self._json({"count": len(rows), "items": rows})
+
+        if path.startswith("/api/job/"):
+            jid = path.rsplit("/", 1)[-1]
+            return self._json(JOBS.get(jid) or {"state": "unknown"})
+
+        return self._send(404, "not found", "text/plain")
+
+    def do_POST(self):
+        u = urllib.parse.urlsplit(self.path)
+        n = int(self.headers.get("Content-Length") or 0)
+        body = {}
+        if n:
+            try:
+                body = json.loads(self.rfile.read(n) or b"{}")
+            except Exception:
+                body = {}
+        if u.path == "/api/feed":
+            jid = f"job{len(JOBS) + 1}"
+            with _LOCK:
+                JOBS[jid] = {"state": "running"}
+            threading.Thread(target=_run_feed, args=(jid, body.get("category"), int(body.get("limit") or 12),
+                                                     body.get("tags")), daemon=True).start()
+            return self._json({"job": jid})
+        return self._json({"error": "not found"}, 404)
+
+
+def serve(port=8077):
+    srv = ThreadingHTTPServer(("127.0.0.1", port), Handler)
+    print(f"omni ui: http://127.0.0.1:{port}  (источников {len(sources.SOURCES)})")
+    srv.serve_forever()
--- a/omni/sources.py
+++ b/omni/sources.py
@@ -56,7 +56,8 @@ SOURCES = [
    S("hf_new_models", "models", "json", "https://huggingface.co/api/models?sort=createdAt&direction=-1&limit=25", "hf_models", ("model", "hf")),
    S("hf_trending_spaces", "models", "json", "https://huggingface.co/api/trending?type=space", "hf_generic", ("space", "hf")),
    S("hf_trending_datasets", "models", "json", "https://huggingface.co/api/trending?type=dataset", "hf_generic", ("dataset", "hf")),
-    S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"), detail=False),
+    S("ollama_library", "models", "json", "https://ollama.com/library", "html_listing", ("model", "local"),
+      detail=False, extra={"include_contains": ["/library/"]}),

    # ---------- benchmark ----------
    S("open_llm_leaderboard", "benchmark", "json",
--- a/omni/store.py
+++ b/omni/store.py
@@ -131,6 +131,25 @@ def search(q, limit=20):
            "ORDER BY score DESC LIMIT ?", (like, like, limit)).fetchall()


+def browse(category=None, q="", limit=60, offset=0):
+    """Список записей для веб-панели: фильтр по категории источника и подстроке."""
+    sql = ("SELECT provider,kind,source,title,text,author,published,score,price,currency,url "
+           "FROM items WHERE 1=1 ")
+    args = []
+    if category and category != "all":
+        sql += "AND source=? "
+        args.append(category)
+    if q:
+        sql += "AND (title LIKE ? OR text LIKE ?) "
+        args += [f"%{q}%", f"%{q}%"]
+    sql += "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"
+    args += [limit, offset]
+    with conn() as c:
+        rows = c.execute(sql, args).fetchall()
+    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
+    return [dict(zip(keys, r)) for r in rows]
+
+
 def stats():
    with conn() as c:
        total = c.execute("SELECT COUNT(*) FROM items").fetchone()[0]