reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

omni: редизайн панели + 450 источников (341 живых), health-аудит, чистка html/дат, per-source учёт

Browse Source
This commit is contained in:
Reform
2026-10-26 09:41:04 +03:00
parent 35be863977
commit 81f3e556c4
8 changed files with 1032 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
omni/serve.py
View File

@@ -4,7 +4,9 @@
"""
import json
import pathlib
import re
import threading
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -14,6 +16,13 @@ from . import feeds, net, sources, source_state, store
WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
JOBS = {}
_LOCK = threading.Lock()
SEQ = [0]
def _jid(prefix):
with _LOCK:
SEQ[0] += 1
return "%s%d" % (prefix, SEQ[0])
def _job(jid, **kw):
@@ -23,20 +32,79 @@ def _job(jid, **kw):
JOBS[jid] = cur
def _cat_health():
"""Сводка по категориям: всего/включено/живых/мёртвых/непроверенных."""
fl = source_state.flags()
out = {}
for s in sources.all_sources():
f = fl.get(s.name, {})
c = out.setdefault(s.category, {"name": s.category, "total": 0, "enabled": 0,
"alive": 0, "dead": 0, "untested": 0})
c["total"] += 1
if f.get("enabled", True):
c["enabled"] += 1
if not f.get("checked"):
c["untested"] += 1
elif (f.get("items") or 0) > 0:
c["alive"] += 1
else:
c["dead"] += 1
return sorted(out.values(), key=lambda x: -x["total"])
def _clean_txt(v, n=0):
"""Снимает html-теги/энтити из title/text, схлопывает пробелы."""
import html as _h
if not v:
return ""
t = str(v)
t = re.sub(r"<[^>]{1,200}>", " ", t)
for _ in range(2):
t = _h.unescape(t)
t = re.sub(r"\s+", " ", t).strip()
return t[:n].rstrip() if n else t
def _fmt_date(v):
"""unix-таймстамп -> YYYY-MM-DD, rfc822 оставляет как есть."""
if not v:
return ""
s = str(v).strip()
if s.isdigit() and len(s) >= 9:
import datetime as _dt
try:
return _dt.datetime.utcfromtimestamp(int(s[:10])).strftime("%Y-%m-%d")
except Exception:
return s
m = re.search(r"(\d{1,2})\s+([A-Za-z]{3})\s+(\d{4})", s)
if m:
return "%s %s %s" % (m.group(1), m.group(2), m.group(3))
return s[:19]
def _names_in_category(cat):
if not cat or cat in ("all", ""):
return None
return [s.name for s in sources.all_sources() if s.category == cat]
def _run_feed(job_id, category, limit, tags):
picked = sources.pick(category, None, tags)
_job(job_id, state="running", total=len(picked), done=0)
try:
items, report = feeds.fetch_many(picked, limit=limit, workers=8)
new, dup = store.add(items)
_job(job_id, state="done", sources=len(picked), fetched=len(items), new=new, dup=dup,
empty=sum(1 for _, c in report if not c),
report=sorted(report, key=lambda x: -x[1])[:40])
empty=sum(1 for _, c in report if not c), done=len(picked),
report=sorted(report, key=lambda x: -x[1])[:60])
except Exception as e:
_job(job_id, state="error", error=f"{type(e).__name__}: {e}"[:200])
def _check_one(src, limit=5):
note = ""
t0 = time.time()
try:
items = feeds.fetch(src, limit)
except Exception as e:
@@ -75,7 +143,7 @@ def _run_check(job_id, names=None, category=None):
class Handler(BaseHTTPRequestHandler):
server_version = "omni/1.1"
server_version = "omni/1.2"
def log_message(self, *a):
pass
@@ -105,52 +173,75 @@ class Handler(BaseHTTPRequestHandler):
if path == "/api/stats":
s = store.stats()
per_source = store.count_by_source()
fl = source_state.flags()
all_s = sources.all_sources()
cats = _cat_health()
for c in cats:
c["stored"] = sum(per_source.get(x.name, 0) for x in all_s if x.category == c["name"])
return self._json({
"items": s["items"], "priced": s["priced"], "runs": s["runs"], "runs_ok": s["runs_ok"],
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"][:12]],
"by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
"sources": len(all_s),
"sources_enabled": sum(1 for x in all_s if fl.get(x.name, {}).get("enabled", True)),
"categories": sources.stats(),
"alive": sum(1 for x in all_s if (fl.get(x.name, {}).get("items") or 0) > 0),
"tested": sum(1 for x in all_s if fl.get(x.name, {}).get("checked")),
"categories": cats,
})
if path == "/api/sources":
cat = (q.get("category") or [None])[0]
query = (q.get("q") or [""])[0].strip().lower()
state = (q.get("state") or [""])[0]
fl = source_state.flags()
per_source = store.count_by_source()
custom_names = {d["name"] for d in source_state.custom()}
items = []
for s in sources.all_sources():
f = fl.get(s.name, {})
alive = (f.get("items") or 0) > 0
if cat and cat not in ("all", "") and s.category != cat:
continue
if query and query not in s.name.lower() and query not in s.url.lower():
continue
if state == "alive" and not alive:
continue
if state == "dead" and not (f.get("checked") and not alive):
continue
if state == "untested" and f.get("checked"):
continue
if state == "off" and f.get("enabled", True):
continue
items.append({
"name": s.name, "category": s.category, "kind": s.kind, "adapter": s.adapter,
"url": s.url, "tags": list(s.tags),
"enabled": f.get("enabled", True), "status": f.get("status"),
"items": f.get("items"), "note": f.get("note"), "checked": f.get("checked"),
"custom": s.name in custom_names,
"stored": per_source.get(s.name, 0), "custom": s.name in custom_names,
})
items.sort(key=lambda x: (-(x["stored"] or 0), -(x["items"] or 0), x["name"]))
return self._json({
"total": len(sources.all_sources()), "shown": len(items),
"enabled": sum(1 for x in items if x["enabled"]),
"alive": sum(1 for x in items if (x["items"] or 0) > 0),
"dead": sum(1 for x in items if x["checked"] and not (x["items"] or 0)),
"untested": sum(1 for x in items if not x["checked"]),
"categories": sources.stats(), "items": items,
"categories": _cat_health(), "items": items,
})
if path == "/api/items":
cat = (q.get("category") or [None])[0]
query = (q.get("q") or [""])[0].strip()
limit = min(int((q.get("limit") or ["60"])[0]), 300)
limit = min(int((q.get("limit") or ["40"])[0]), 200)
offset = int((q.get("offset") or ["0"])[0])
rows = store.browse(category=cat, q=query, limit=limit, offset=offset)
return self._json({"count": len(rows), "items": rows})
sort = (q.get("sort") or ["score"])[0]
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort)
for _r in rows:
_r["title"] = _clean_txt(_r.get("title"), 180)
_r["text"] = _clean_txt(_r.get("text"), 260)
_r["published"] = _fmt_date(_r.get("published"))
return self._json({"count": len(rows), "total": total, "offset": offset, "items": rows})
if path.startswith("/api/job/"):
jid = path.rsplit("/", 1)[-1]
@@ -169,7 +260,7 @@ class Handler(BaseHTTPRequestHandler):
body = {}
if u.path == "/api/feed":
jid = f"feed{len(JOBS)}"
jid = _jid("feed")
_job(jid, state="running")
threading.Thread(target=_run_feed,
args=(jid, body.get("category"), int(body.get("limit") or 12), body.get("tags")),
@@ -182,6 +273,17 @@ class Handler(BaseHTTPRequestHandler):
return self._json({"error": "name required"}, 400)
return self._json(source_state.set_enabled(name, bool(body.get("enabled", True))))
if u.path == "/api/sources/toggle_all":
on = bool(body.get("enabled", True))
cat = body.get("category")
changed = 0
for s in sources.all_sources():
if cat and cat not in ("all", "") and s.category != cat:
continue
source_state.set_enabled(s.name, on)
changed += 1
return self._json({"ok": True, "changed": changed, "enabled": on})
if u.path == "/api/sources/add":
try:
res = source_state.add_custom(body.get("name"), body.get("category"), body.get("url"),
@@ -200,8 +302,8 @@ class Handler(BaseHTTPRequestHandler):
if u.path == "/api/sources/check":
names = body.get("names") or ([body["name"]] if body.get("name") else None)
category = None if body.get("all") else body.get("category")
jid = f"check{len(JOBS)}"
category = body.get("category")
jid = _jid("check")
_job(jid, state="running")
threading.Thread(target=_run_check, args=(jid, names, category), daemon=True).start()
return self._json({"job": jid})