omni: веб-панель (serve + /api), дайджест, arxiv API-fallback, HF trending unwrap

omni/serve.py

@@ -0,0 +1,111 @@
+"""Веб-панель omni: статика + JSON API поверх той же SQLite.
+
+Запуск: python -m omni serve --port 8077
+"""
+import json
+import pathlib
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from . import feeds, sources, store
+
+WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
+JOBS = {}
+_LOCK = threading.Lock()
+
+
+def _run_feed(job_id, category, limit, tags):
+    picked = sources.pick(category, None, tags)
+    try:
+        items, report = feeds.fetch_many(picked, limit=limit, workers=8)
+        new, dup = store.add(items)
+        with _LOCK:
+            JOBS[job_id] = {"state": "done", "sources": len(picked), "fetched": len(items),
+                            "new": new, "dup": dup, "empty": sum(1 for _, c in report if not c),
+                            "report": sorted(report, key=lambda x: -x[1])[:40]}
+    except Exception as e:
+        with _LOCK:
+            JOBS[job_id] = {"state": "error", "error": f"{type(e).__name__}: {e}"[:200]}
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "omni/1.0"
+
+    def log_message(self, *a):
+        pass
+
+    def _send(self, code, body, ctype="application/json; charset=utf-8"):
+        data = body if isinstance(body, bytes) else str(body).encode("utf-8")
+        self.send_response(code)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def _json(self, obj, code=200):
+        self._send(code, json.dumps(obj, ensure_ascii=False))
+
+    def do_GET(self):
+        u = urllib.parse.urlsplit(self.path)
+        q = urllib.parse.parse_qs(u.query)
+        path = u.path
+
+        if path in ("/", "/index.html"):
+            f = WEB / "index.html"
+            if not f.exists():
+                return self._send(404, "no ui", "text/plain")
+            return self._send(200, f.read_bytes(), "text/html; charset=utf-8")
+
+        if path == "/api/stats":
+            s = store.stats()
+            return self._json({"items": s["items"], "priced": s["priced"], "runs": s["runs"],
+                               "runs_ok": s["runs_ok"],
+                               "by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
+                               "by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
+                               "sources": len(sources.SOURCES), "categories": sources.stats()})
+
+        if path == "/api/sources":
+            return self._json({"total": len(sources.SOURCES), "categories": sources.stats(),
+                               "items": [{"name": s.name, "category": s.category, "kind": s.kind,
+                                          "adapter": s.adapter, "url": s.url, "tags": list(s.tags)}
+                                         for s in sources.SOURCES]})
+
+        if path == "/api/items":
+            cat = (q.get("category") or [None])[0]
+            query = (q.get("q") or [""])[0].strip()
+            limit = min(int((q.get("limit") or ["60"])[0]), 300)
+            offset = int((q.get("offset") or ["0"])[0])
+            rows = store.browse(category=cat, q=query, limit=limit, offset=offset)
+            return self._json({"count": len(rows), "items": rows})
+
+        if path.startswith("/api/job/"):
+            jid = path.rsplit("/", 1)[-1]
+            return self._json(JOBS.get(jid) or {"state": "unknown"})
+
+        return self._send(404, "not found", "text/plain")
+
+    def do_POST(self):
+        u = urllib.parse.urlsplit(self.path)
+        n = int(self.headers.get("Content-Length") or 0)
+        body = {}
+        if n:
+            try:
+                body = json.loads(self.rfile.read(n) or b"{}")
+            except Exception:
+                body = {}
+        if u.path == "/api/feed":
+            jid = f"job{len(JOBS) + 1}"
+            with _LOCK:
+                JOBS[jid] = {"state": "running"}
+            threading.Thread(target=_run_feed, args=(jid, body.get("category"), int(body.get("limit") or 12),
+                                                     body.get("tags")), daemon=True).start()
+            return self._json({"job": jid})
+        return self._json({"error": "not found"}, 404)
+
+
+def serve(port=8077):
+    srv = ThreadingHTTPServer(("127.0.0.1", port), Handler)
+    print(f"omni ui: http://127.0.0.1:{port}  (источников {len(sources.SOURCES)})")
+    srv.serve_forever()