reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

omni: редизайн панели + 450 источников (341 живых), health-аудит, чистка html/дат, per-source учёт

Browse Source
This commit is contained in:
Reform
2026-10-26 09:41:04 +03:00
parent 35be863977
commit 81f3e556c4
8 changed files with 1032 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
omni/cli.py
View File

@@ -164,6 +164,39 @@ def cmd_sources(args):
return 0
def cmd_audit(args):
"""Сводка здоровья реестра: по категориям и список мёртвых с причиной."""
from . import source_state
fl = source_state.flags()
all_s = sources.all_sources()
agg = {}
dead = []
for s in all_s:
f = fl.get(s.name, {})
a = agg.setdefault(s.category, {"t": 0, "alive": 0, "dead": 0, "untested": 0})
a["t"] += 1
if not f.get("checked"):
a["untested"] += 1
elif (f.get("items") or 0) > 0:
a["alive"] += 1
else:
a["dead"] += 1
dead.append((s.category, s.name, f.get("note") or "empty"))
print("источников: %d | живых: %d | мёртвых: %d | не проверено: %d" % (
len(all_s), sum(a["alive"] for a in agg.values()),
sum(a["dead"] for a in agg.values()), sum(a["untested"] for a in agg.values())))
for cat, a in sorted(agg.items(), key=lambda x: -x[1]["t"]):
pct = round(a["alive"] / a["t"] * 100) if a["t"] else 0
print(" %-12s всего %3d живых %3d мёртвых %3d не проверено %3d (%d%%)"
% (cat, a["t"], a["alive"], a["dead"], a["untested"], pct))
if args.dead:
print("")
print("мёртвые:")
for cat, name, note in sorted(dead):
print(" %-12s %-28s %s" % (cat, name, note[:50]))
return 0
def cmd_feed(args):
picked = sources.pick(args.category, args.only, args.tags)
if not picked:
@@ -265,6 +298,10 @@ def main(argv=None):
sp.add_argument("--all", action="store_true", help="показать и выключенные")
sp.set_defaults(fn=cmd_sources)
sp = sub.add_parser("audit", help="здоровье реестра источников")
sp.add_argument("--dead", action="store_true", help="показать мёртвые с причиной")
sp.set_defaults(fn=cmd_audit)
sp = sub.add_parser("feed", help="обход реестра источников без запроса")
sp.add_argument("--category", default=None)
sp.add_argument("--only", default=None)

142
omni/feeds.py
View File

@@ -28,7 +28,7 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
ex = dict(extra or {})
ex["tags"] = list(src.tags)
return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
author=_txt(author)[:80], published=str(published or ""), source=src.category,
author=_txt(author)[:80], published=str(published or ""), source=src.name,
extra=ex)
@@ -515,6 +515,138 @@ def _nvd(src, limit):
return out
def _biorxiv(src, limit):
import json as _j
d = _json_of(src.url)
rows = d.get("collection") if isinstance(d, dict) else d
out = []
for r in (rows or [])[:limit]:
title = r.get("title") or ""
doi = r.get("doi") or ""
if not title:
continue
ver = r.get("version") or "1"
host = "medrxiv" if "medrxiv" in src.url else "biorxiv"
url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url
out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800],
str(r.get("authors") or ""), str(r.get("date") or "")[:24]))
return out
def _dblp(src, limit):
d = _json_of(src.url)
hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or []
out = []
for h in hits[:limit]:
info = h.get("info") or {}
title = info.get("title") or ""
if not title:
continue
auth = info.get("authors") or {}
if isinstance(auth, dict):
a = auth.get("author")
if isinstance(a, list):
auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a)
elif isinstance(a, dict):
auth = a.get("text", "")
out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""),
str(auth or ""), str(info.get("year") or "")))
return out
def _hal(src, limit):
d = _json_of(src.url)
docs = ((d or {}).get("response") or {}).get("docs") or []
out = []
for r in docs[:limit]:
t = r.get("title_s")
title = (t[0] if isinstance(t, list) and t else t) or ""
if not title:
continue
ab = r.get("abstract_s")
if isinstance(ab, list):
ab = ab[0] if ab else ""
out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "",
str(r.get("producedDateY_i") or "")))
return out
def _zenodo(src, limit):
d = _json_of(src.url)
rows = ((d or {}).get("hits") or {}).get("hits") or []
out = []
for r in rows[:limit]:
md = r.get("metadata") or {}
title = md.get("title") or ""
if not title:
continue
url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or ""
out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800],
str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""),
str(md.get("publication_date") or "")))
return out
def _cve_circl(src, limit):
d = _json_of(src.url)
rows = d if isinstance(d, list) else []
out = []
for r in rows[:limit]:
cid = r.get("id") or ""
if not cid:
continue
out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140],
str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24]))
return out
def _ghsa(src, limit):
d = _json_of(src.url)
rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else []
out = []
for r in rows[:limit]:
gid = r.get("ghsa_id") or ""
title = r.get("summary") or gid
if not title:
continue
sev = r.get("severity") or ""
out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title),
(sev + " | " + (r.get("description") or ""))[:800], "",
str(r.get("published_at") or "")[:24]))
return out
def _models_dev(src, limit):
d = _json_of(src.url)
out = []
if not isinstance(d, dict):
return out
for pid, prov in d.items():
if not isinstance(prov, dict):
continue
pname = prov.get("name") or pid
models = prov.get("models") or {}
if isinstance(models, dict):
iterable = models.items()
elif isinstance(models, list):
iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)]
else:
continue
for mid, m in iterable:
if not isinstance(m, dict):
continue
name = m.get("name") or mid
out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name),
"ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"),
(m.get("cost") or {}).get("input", "?"),
(m.get("cost") or {}).get("output", "?")),
str(pid), ""))
if len(out) >= limit:
return out
return out
ADAPTERS = {
"rss": _rss,
"generic_json": _generic_json,
@@ -537,6 +669,14 @@ ADAPTERS = {
"xenforo_latest": _xenforo_latest,
"tg_web": _tg_web,
"html_listing": _html_listing,
"biorxiv": _biorxiv,
"dblp": _dblp,
"hal": _hal,
"zenodo": _zenodo,
"cve_circl": _cve_circl,
"ghsa": _ghsa,
"models_dev": _models_dev,
}

128
omni/serve.py
View File

@@ -4,7 +4,9 @@
"""
import json
import pathlib
import re
import threading
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -14,6 +16,13 @@ from . import feeds, net, sources, source_state, store
WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
JOBS = {}
_LOCK = threading.Lock()
SEQ = [0]
def _jid(prefix):
with _LOCK:
SEQ[0] += 1
return "%s%d" % (prefix, SEQ[0])
def _job(jid, **kw):
@@ -23,20 +32,79 @@ def _job(jid, **kw):
JOBS[jid] = cur
def _cat_health():
"""Сводка по категориям: всего/включено/живых/мёртвых/непроверенных."""
fl = source_state.flags()
out = {}
for s in sources.all_sources():
f = fl.get(s.name, {})
c = out.setdefault(s.category, {"name": s.category, "total": 0, "enabled": 0,
"alive": 0, "dead": 0, "untested": 0})
c["total"] += 1
if f.get("enabled", True):
c["enabled"] += 1
if not f.get("checked"):
c["untested"] += 1
elif (f.get("items") or 0) > 0:
c["alive"] += 1
else:
c["dead"] += 1
return sorted(out.values(), key=lambda x: -x["total"])
def _clean_txt(v, n=0):
"""Снимает html-теги/энтити из title/text, схлопывает пробелы."""
import html as _h
if not v:
return ""
t = str(v)
t = re.sub(r"<[^>]{1,200}>", " ", t)
for _ in range(2):
t = _h.unescape(t)
t = re.sub(r"\s+", " ", t).strip()
return t[:n].rstrip() if n else t
def _fmt_date(v):
"""unix-таймстамп -> YYYY-MM-DD, rfc822 оставляет как есть."""
if not v:
return ""
s = str(v).strip()
if s.isdigit() and len(s) >= 9:
import datetime as _dt
try:
return _dt.datetime.utcfromtimestamp(int(s[:10])).strftime("%Y-%m-%d")
except Exception:
return s
m = re.search(r"(\d{1,2})\s+([A-Za-z]{3})\s+(\d{4})", s)
if m:
return "%s %s %s" % (m.group(1), m.group(2), m.group(3))
return s[:19]
def _names_in_category(cat):
if not cat or cat in ("all", ""):
return None
return [s.name for s in sources.all_sources() if s.category == cat]
def _run_feed(job_id, category, limit, tags):
picked = sources.pick(category, None, tags)
_job(job_id, state="running", total=len(picked), done=0)
try:
items, report = feeds.fetch_many(picked, limit=limit, workers=8)
new, dup = store.add(items)
_job(job_id, state="done", sources=len(picked), fetched=len(items), new=new, dup=dup,
empty=sum(1 for _, c in report if not c),
report=sorted(report, key=lambda x: -x[1])[:40])
empty=sum(1 for _, c in report if not c), done=len(picked),
report=sorted(report, key=lambda x: -x[1])[:60])
except Exception as e:
_job(job_id, state="error", error=f"{type(e).__name__}: {e}"[:200])
def _check_one(src, limit=5):
note = ""
t0 = time.time()
try:
items = feeds.fetch(src, limit)
except Exception as e:
@@ -75,7 +143,7 @@ def _run_check(job_id, names=None, category=None):
class Handler(BaseHTTPRequestHandler):
server_version = "omni/1.1"
server_version = "omni/1.2"
def log_message(self, *a):
pass
@@ -105,52 +173,75 @@ class Handler(BaseHTTPRequestHandler):
if path == "/api/stats":
s = store.stats()
per_source = store.count_by_source()
fl = source_state.flags()
all_s = sources.all_sources()
cats = _cat_health()
for c in cats:
c["stored"] = sum(per_source.get(x.name, 0) for x in all_s if x.category == c["name"])
return self._json({
"items": s["items"], "priced": s["priced"], "runs": s["runs"], "runs_ok": s["runs_ok"],
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"][:12]],
"by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
"sources": len(all_s),
"sources_enabled": sum(1 for x in all_s if fl.get(x.name, {}).get("enabled", True)),
"categories": sources.stats(),
"alive": sum(1 for x in all_s if (fl.get(x.name, {}).get("items") or 0) > 0),
"tested": sum(1 for x in all_s if fl.get(x.name, {}).get("checked")),
"categories": cats,
})
if path == "/api/sources":
cat = (q.get("category") or [None])[0]
query = (q.get("q") or [""])[0].strip().lower()
state = (q.get("state") or [""])[0]
fl = source_state.flags()
per_source = store.count_by_source()
custom_names = {d["name"] for d in source_state.custom()}
items = []
for s in sources.all_sources():
f = fl.get(s.name, {})
alive = (f.get("items") or 0) > 0
if cat and cat not in ("all", "") and s.category != cat:
continue
if query and query not in s.name.lower() and query not in s.url.lower():
continue
if state == "alive" and not alive:
continue
if state == "dead" and not (f.get("checked") and not alive):
continue
if state == "untested" and f.get("checked"):
continue
if state == "off" and f.get("enabled", True):
continue
items.append({
"name": s.name, "category": s.category, "kind": s.kind, "adapter": s.adapter,
"url": s.url, "tags": list(s.tags),
"enabled": f.get("enabled", True), "status": f.get("status"),
"items": f.get("items"), "note": f.get("note"), "checked": f.get("checked"),
"custom": s.name in custom_names,
"stored": per_source.get(s.name, 0), "custom": s.name in custom_names,
})
items.sort(key=lambda x: (-(x["stored"] or 0), -(x["items"] or 0), x["name"]))
return self._json({
"total": len(sources.all_sources()), "shown": len(items),
"enabled": sum(1 for x in items if x["enabled"]),
"alive": sum(1 for x in items if (x["items"] or 0) > 0),
"dead": sum(1 for x in items if x["checked"] and not (x["items"] or 0)),
"untested": sum(1 for x in items if not x["checked"]),
"categories": sources.stats(), "items": items,
"categories": _cat_health(), "items": items,
})
if path == "/api/items":
cat = (q.get("category") or [None])[0]
query = (q.get("q") or [""])[0].strip()
limit = min(int((q.get("limit") or ["60"])[0]), 300)
limit = min(int((q.get("limit") or ["40"])[0]), 200)
offset = int((q.get("offset") or ["0"])[0])
rows = store.browse(category=cat, q=query, limit=limit, offset=offset)
return self._json({"count": len(rows), "items": rows})
sort = (q.get("sort") or ["score"])[0]
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort)
for _r in rows:
_r["title"] = _clean_txt(_r.get("title"), 180)
_r["text"] = _clean_txt(_r.get("text"), 260)
_r["published"] = _fmt_date(_r.get("published"))
return self._json({"count": len(rows), "total": total, "offset": offset, "items": rows})
if path.startswith("/api/job/"):
jid = path.rsplit("/", 1)[-1]
@@ -169,7 +260,7 @@ class Handler(BaseHTTPRequestHandler):
body = {}
if u.path == "/api/feed":
jid = f"feed{len(JOBS)}"
jid = _jid("feed")
_job(jid, state="running")
threading.Thread(target=_run_feed,
args=(jid, body.get("category"), int(body.get("limit") or 12), body.get("tags")),
@@ -182,6 +273,17 @@ class Handler(BaseHTTPRequestHandler):
return self._json({"error": "name required"}, 400)
return self._json(source_state.set_enabled(name, bool(body.get("enabled", True))))
if u.path == "/api/sources/toggle_all":
on = bool(body.get("enabled", True))
cat = body.get("category")
changed = 0
for s in sources.all_sources():
if cat and cat not in ("all", "") and s.category != cat:
continue
source_state.set_enabled(s.name, on)
changed += 1
return self._json({"ok": True, "changed": changed, "enabled": on})
if u.path == "/api/sources/add":
try:
res = source_state.add_custom(body.get("name"), body.get("category"), body.get("url"),
@@ -200,8 +302,8 @@ class Handler(BaseHTTPRequestHandler):
if u.path == "/api/sources/check":
names = body.get("names") or ([body["name"]] if body.get("name") else None)
category = None if body.get("all") else body.get("category")
jid = f"check{len(JOBS)}"
category = body.get("category")
jid = _jid("check")
_job(jid, state="running")
threading.Thread(target=_run_check, args=(jid, names, category), daemon=True).start()
return self._json({"job": jid})

13
omni/sources.py
View File

@@ -11,6 +11,7 @@
from dataclasses import dataclass, field
from .sources_ext import EXTRA
from .sources_ext2 import EXTRA2
@dataclass
@@ -177,7 +178,17 @@ SOURCES_BASE = [
S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
]
SOURCES = SOURCES_BASE + [Source(**d) for d in EXTRA]
def _dedup(rows):
seen, out = set(), []
for s in rows:
if s.url in seen:
continue
seen.add(s.url)
out.append(s)
return out
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2])
BY_NAME = {s.name: s for s in SOURCES}
CATEGORIES = sorted({s.category for s in SOURCES})

118
omni/sources_ext2.py Normal file
View File

@@ -0,0 +1,118 @@
"""Волна 2+3: источники, прошедшие живой фетч (items > 0). Сгенерировано probe_w23.py."""
EXTRA2 = [
dict(name="gaia_bench", category="benchmark", kind="html", url="https://huggingface.co/spaces/gaia-benchmark/leaderboard", adapter="html_listing", tags=["bench"], limit=20, detail=False),
dict(name="osworld_leaderboard", category="benchmark", kind="html", url="https://os-world.github.io/", adapter="html_listing", tags=["bench"], limit=20, detail=False),
dict(name="swe_rebench", category="benchmark", kind="html", url="https://swe-rebench.com/", adapter="html_listing", tags=["bench"], limit=20, detail=False),
dict(name="terminal_bench", category="benchmark", kind="html", url="https://www.tbench.ai/leaderboard", adapter="html_listing", tags=["bench"], limit=20, detail=False),
dict(name="coding_horror", category="community", kind="rss", url="https://blog.codinghorror.com/rss/", adapter="rss", tags=["blog"], limit=20, detail=False),
dict(name="danluu_blog", category="community", kind="rss", url="https://danluu.com/atom.xml", adapter="rss", tags=["blog"], limit=20, detail=False),
dict(name="jay_alammar", category="community", kind="rss", url="https://jalammar.github.io/feed.xml", adapter="rss", tags=["ml", "explainer"], limit=20, detail=False),
dict(name="jvns_blog", category="community", kind="rss", url="https://jvns.ca/atom.xml", adapter="rss", tags=["blog"], limit=20, detail=False),
dict(name="kentcdodds", category="community", kind="rss", url="https://kentcdodds.com/blog/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="lilianweng", category="community", kind="rss", url="https://lilianweng.github.io/index.xml", adapter="rss", tags=["ml", "research"], limit=20, detail=False),
dict(name="lobsters_databases", category="community", kind="rss", url="https://lobste.rs/t/databases.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False),
dict(name="lobsters_linux", category="community", kind="rss", url="https://lobste.rs/t/linux.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False),
dict(name="lobsters_ml", category="community", kind="rss", url="https://lobste.rs/t/ml.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False),
dict(name="lobsters_performance", category="community", kind="rss", url="https://lobste.rs/t/performance.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False),
dict(name="lobsters_web", category="community", kind="rss", url="https://lobste.rs/t/web.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False),
dict(name="lwn_headlines", category="community", kind="rss", url="https://lwn.net/headlines/rss", adapter="rss", tags=["linux"], limit=20, detail=False),
dict(name="martinfowler", category="community", kind="rss", url="https://martinfowler.com/feed.atom", adapter="rss", tags=["arch"], limit=20, detail=False),
dict(name="neuromancer_ai", category="community", kind="rss", url="https://www.interconnects.ai/feed", adapter="rss", tags=["llm", "analysis"], limit=20, detail=False),
dict(name="overreacted", category="community", kind="rss", url="https://overreacted.io/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="phoronix", category="community", kind="rss", url="https://www.phoronix.com/rss.php", adapter="rss", tags=["linux", "hardware"], limit=20, detail=False),
dict(name="reddit_cpp", category="community", kind="reddit", url="https://www.reddit.com/r/cpp/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_docker", category="community", kind="reddit", url="https://www.reddit.com/r/docker/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_flutter", category="community", kind="reddit", url="https://www.reddit.com/r/FlutterDev/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_gamedev", category="community", kind="reddit", url="https://www.reddit.com/r/gamedev/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_gcp", category="community", kind="reddit", url="https://www.reddit.com/r/googlecloud/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_homelab", category="community", kind="reddit", url="https://www.reddit.com/r/homelab/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_llm", category="community", kind="reddit", url="https://www.reddit.com/r/LLM/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_localllm", category="community", kind="reddit", url="https://www.reddit.com/r/LocalLLM/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_php", category="community", kind="reddit", url="https://www.reddit.com/r/PHP/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="reddit_swift", category="community", kind="reddit", url="https://www.reddit.com/r/swift/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False),
dict(name="scott_helme", category="community", kind="rss", url="https://scotthelme.co.uk/rss/", adapter="rss", tags=["tls"], limit=20, detail=False),
dict(name="sebastian_raschka", category="community", kind="rss", url="https://magazine.sebastianraschka.com/feed", adapter="rss", tags=["llm", "ml"], limit=20, detail=False),
dict(name="simonwillison", category="community", kind="rss", url="https://simonwillison.net/atom/everything/", adapter="rss", tags=["llm", "blog"], limit=20, detail=False),
dict(name="tg_addmetana", category="community", kind="tg", url="https://t.me/s/addmeto", adapter="tg_web", tags=["tg", "ru"], limit=20, detail=False),
dict(name="the_gradient", category="community", kind="rss", url="https://thegradient.pub/rss/", adapter="rss", tags=["ml"], limit=20, detail=False),
dict(name="astro_blog", category="company", kind="rss", url="https://astro.build/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="bun_blog", category="company", kind="rss", url="https://bun.sh/rss.xml", adapter="rss", tags=["runtime"], limit=20, detail=False),
dict(name="dagster_blog", category="company", kind="rss", url="https://dagster.io/blog/rss.xml", adapter="rss", tags=["data"], limit=20, detail=False),
dict(name="datadog_blog", category="company", kind="rss", url="https://www.datadoghq.com/blog/index.xml", adapter="rss", tags=["obs"], limit=20, detail=False),
dict(name="dbt_blog", category="company", kind="rss", url="https://www.getdbt.com/blog/rss.xml", adapter="rss", tags=["data"], limit=20, detail=False),
dict(name="deno_blog", category="company", kind="rss", url="https://deno.com/feed", adapter="rss", tags=["runtime"], limit=20, detail=False),
dict(name="fal_blog", category="company", kind="rss", url="https://blog.fal.ai/rss/", adapter="rss", tags=["inference", "image"], limit=20, detail=False),
dict(name="gh_axolotl", category="company", kind="github", url="axolotl-ai-cloud", adapter="github_org", tags=["finetune"], limit=20, detail=False),
dict(name="gh_crewai", category="company", kind="github", url="crewAIInc", adapter="github_org", tags=["agents"], limit=20, detail=False),
dict(name="gh_litellm", category="company", kind="github", url="BerriAI", adapter="github_org", tags=["gateway"], limit=20, detail=False),
dict(name="gh_llamaindex", category="company", kind="github", url="run-llama", adapter="github_org", tags=["rag"], limit=20, detail=False),
dict(name="gh_mlc", category="company", kind="github", url="mlc-ai", adapter="github_org", tags=["local"], limit=20, detail=False),
dict(name="gh_triton", category="company", kind="github", url="triton-inference-server", adapter="github_org", tags=["inference"], limit=20, detail=False),
dict(name="go_blog", category="company", kind="rss", url="https://go.dev/blog/feed.atom", adapter="rss", tags=["lang"], limit=20, detail=False),
dict(name="grafana_blog", category="company", kind="rss", url="https://grafana.com/blog/index.xml", adapter="rss", tags=["obs"], limit=20, detail=False),
dict(name="nextjs_blog", category="company", kind="rss", url="https://nextjs.org/feed.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="node_blog", category="company", kind="rss", url="https://nodejs.org/en/feed/blog.xml", adapter="rss", tags=["runtime"], limit=20, detail=False),
dict(name="prisma_blog", category="company", kind="rss", url="https://www.prisma.io/blog/rss.xml", adapter="rss", tags=["db"], limit=20, detail=False),
dict(name="python_insider", category="company", kind="rss", url="https://pythoninsider.blogspot.com/feeds/posts/default", adapter="rss", tags=["python"], limit=20, detail=False),
dict(name="react_blog", category="company", kind="rss", url="https://react.dev/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="rust_blog", category="company", kind="rss", url="https://blog.rust-lang.org/feed.xml", adapter="rss", tags=["lang"], limit=20, detail=False),
dict(name="svelte_blog", category="company", kind="rss", url="https://svelte.dev/blog/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False),
dict(name="tg_anthropic", category="company", kind="tg", url="https://t.me/s/AnthropicAI", adapter="tg_web", tags=["tg"], limit=20, detail=False),
dict(name="tg_googleai", category="company", kind="tg", url="https://t.me/s/googleai", adapter="tg_web", tags=["tg"], limit=20, detail=False),
dict(name="timescale_blog", category="company", kind="rss", url="https://www.timescale.com/blog/rss/", adapter="rss", tags=["db"], limit=20, detail=False),
dict(name="coupang_tech", category="company_kr", kind="rss", url="https://medium.com/feed/coupang-tech", adapter="rss", tags=["kr"], limit=20, detail=False),
dict(name="bubble_forum", category="forum", kind="json", url="https://forum.bubble.io/latest.json", adapter="discourse", tags=["nocode"], limit=20, detail=False),
dict(name="discourse_meta", category="forum", kind="json", url="https://meta.discourse.org/latest.json", adapter="discourse", tags=["meta"], limit=20, detail=False),
dict(name="discuss_jupyter", category="forum", kind="json", url="https://discourse.jupyter.org/latest.json", adapter="discourse", tags=["data"], limit=20, detail=False),
dict(name="discuss_kotlin", category="forum", kind="json", url="https://discuss.kotlinlang.org/latest.json", adapter="discourse", tags=["kotlin"], limit=20, detail=False),
dict(name="discuss_kubernetes", category="forum", kind="json", url="https://discuss.kubernetes.io/latest.json", adapter="discourse", tags=["infra"], limit=20, detail=False),
dict(name="fastai_forum", category="forum", kind="json", url="https://forums.fast.ai/latest.json", adapter="discourse", tags=["ml"], limit=20, detail=False),
dict(name="macrumors_forum", category="forum", kind="rss", url="https://forums.macrumors.com/forums/-/index.rss", adapter="rss", tags=["apple"], limit=20, detail=False),
dict(name="make_community", category="forum", kind="json", url="https://community.make.com/latest.json", adapter="discourse", tags=["automation"], limit=20, detail=False),
dict(name="pixls_forum", category="forum", kind="json", url="https://discuss.pixls.us/latest.json", adapter="discourse", tags=["photo"], limit=20, detail=False),
dict(name="proxmox_forum", category="forum", kind="rss", url="https://forum.proxmox.com/forums/-/index.rss", adapter="rss", tags=["virt"], limit=20, detail=False),
dict(name="streamlit_forum", category="forum", kind="json", url="https://discuss.streamlit.io/latest.json", adapter="discourse", tags=["data"], limit=20, detail=False),
dict(name="xda_forum", category="forum", kind="rss", url="https://xdaforums.com/forums/-/index.rss", adapter="rss", tags=["mobile"], limit=20, detail=False),
dict(name="hf_datasets_recent", category="models", kind="json", url="https://huggingface.co/api/datasets?sort=lastModified&limit=20", adapter="hf_generic", tags=["hf", "dataset"], limit=20, detail=False),
dict(name="hf_models_audio", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=automatic-speech-recognition&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "audio"], limit=20, detail=False),
dict(name="hf_models_ocr", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=image-text-to-text&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "vlm"], limit=20, detail=False),
dict(name="hf_models_video", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=text-to-video&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "video"], limit=20, detail=False),
dict(name="arxiv_cs_cc", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CC", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_cs_ce", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CE", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_cs_cg", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CG", adapter="rss", tags=["paper", "arxiv", "graphics"], limit=20, detail=False),
dict(name="arxiv_cs_dm", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.DM", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_cs_gl", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.GL", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_cs_oh", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.OH", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_econ_gn", category="papers", kind="rss", url="https://rss.arxiv.org/rss/econ.GN", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_eess_sy", category="papers", kind="rss", url="https://rss.arxiv.org/rss/eess.SY", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_math_st", category="papers", kind="rss", url="https://rss.arxiv.org/rss/math.ST", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_physics_socph", category="papers", kind="rss", url="https://rss.arxiv.org/rss/physics.soc-ph", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_qbio_qm", category="papers", kind="rss", url="https://rss.arxiv.org/rss/q-bio.QM", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_qfin_cp", category="papers", kind="rss", url="https://rss.arxiv.org/rss/q-fin.CP", adapter="rss", tags=["paper", "arxiv", "finance"], limit=20, detail=False),
dict(name="arxiv_stat_co", category="papers", kind="rss", url="https://rss.arxiv.org/rss/stat.CO", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="arxiv_stat_me", category="papers", kind="rss", url="https://rss.arxiv.org/rss/stat.ME", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False),
dict(name="biorxiv_recent", category="papers", kind="json", url="https://api.biorxiv.org/details/biorxiv/2026-09-01/2026-10-25/0", adapter="biorxiv", tags=["paper", "bio"], limit=20, detail=False),
dict(name="hal_ai", category="papers", kind="json", url="https://api.archives-ouvertes.fr/search/?q=llm&fl=title_s,uri_s,abstract_s,producedDateY_i&rows=25&wt=json", adapter="hal", tags=["paper", "hal"], limit=20, detail=False),
dict(name="medrxiv_recent", category="papers", kind="json", url="https://api.medrxiv.org/details/medrxiv/2026-09-01/2026-10-25/0", adapter="biorxiv", tags=["paper", "med"], limit=20, detail=False),
dict(name="zenodo_ai", category="papers", kind="json", url="https://zenodo.org/api/records?q=large+language+model&sort=newest&size=20", adapter="zenodo", tags=["paper", "zenodo"], limit=20, detail=False),
dict(name="zenodo_security", category="papers", kind="json", url="https://zenodo.org/api/records?q=vulnerability&sort=newest&size=20", adapter="zenodo", tags=["paper", "zenodo"], limit=20, detail=False),
dict(name="0xdf_blog", category="security", kind="rss", url="https://0xdf.gitlab.io/feed.xml", adapter="rss", tags=["bb", "htb"], limit=20, detail=False),
dict(name="ctftime_upcoming", category="security", kind="rss", url="https://ctftime.org/event/list/upcoming/rss/", adapter="rss", tags=["ctf"], limit=20, detail=False),
dict(name="cve_circl", category="security", kind="json", url="https://cve.circl.lu/api/last/30", adapter="cve_circl", tags=["cve"], limit=20, detail=False),
dict(name="dfir_report", category="security", kind="rss", url="https://thedfirreport.com/feed/", adapter="rss", tags=["dfir"], limit=20, detail=False),
dict(name="gitguardian_blog", category="security", kind="rss", url="https://blog.gitguardian.com/rss/", adapter="rss", tags=["secrets"], limit=20, detail=False),
dict(name="hackread", category="security", kind="rss", url="https://hackread.com/feed/", adapter="rss", tags=["news"], limit=20, detail=False),
dict(name="huntress_blog", category="security", kind="rss", url="https://www.huntress.com/blog/rss.xml", adapter="rss", tags=["dfir"], limit=20, detail=False),
dict(name="intigriti_blog", category="security", kind="rss", url="https://blog.intigriti.com/feed/", adapter="rss", tags=["bb"], limit=20, detail=False),
dict(name="jfrog_blog", category="security", kind="rss", url="https://jfrog.com/blog/feed/", adapter="rss", tags=["supply"], limit=20, detail=False),
dict(name="medium_bugbounty", category="security", kind="rss", url="https://medium.com/feed/tag/bug-bounty", adapter="rss", tags=["bb", "writeups"], limit=20, detail=False),
dict(name="medium_pentest", category="security", kind="rss", url="https://medium.com/feed/tag/penetration-testing", adapter="rss", tags=["pentest"], limit=20, detail=False),
dict(name="medium_redteam", category="security", kind="rss", url="https://medium.com/feed/tag/red-team", adapter="rss", tags=["redteam"], limit=20, detail=False),
dict(name="risky_business", category="security", kind="rss", url="https://risky.biz/feeds/risky-business-news/", adapter="rss", tags=["news"], limit=20, detail=False),
dict(name="schneier_blog", category="security", kind="rss", url="https://www.schneier.com/blog/atom.xml", adapter="rss", tags=["crypto", "privacy"], limit=20, detail=False),
dict(name="seclists_oss", category="security", kind="rss", url="https://seclists.org/rss/oss-sec.rss", adapter="rss", tags=["oss", "cve"], limit=20, detail=False),
dict(name="sonatype_blog", category="security", kind="rss", url="https://blog.sonatype.com/rss.xml", adapter="rss", tags=["supply"], limit=20, detail=False),
dict(name="troy_hunt", category="security", kind="rss", url="https://www.troyhunt.com/rss/", adapter="rss", tags=["breach"], limit=20, detail=False),
dict(name="watchtowr_blog", category="security", kind="rss", url="https://labs.watchtowr.com/rss/", adapter="rss", tags=["research"], limit=20, detail=False),
]

23
omni/store.py
View File

@@ -150,6 +150,29 @@ def browse(category=None, q="", limit=60, offset=0):
return [dict(zip(keys, r)) for r in rows]
def browse2(q="", names=None, limit=60, offset=0, sort="score"):
"""Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
where = "WHERE 1=1 "
args = []
if names:
where += "AND source IN (%s) " % ",".join("?" * len(names))
args += list(names)
if q:
where += "AND (title LIKE ? OR text LIKE ?) "
args += [f"%{q}%", f"%{q}%"]
with conn() as c:
total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url "
"FROM items " + where + ("ORDER BY published DESC, id DESC LIMIT ? OFFSET ?" if sort == "fresh" else "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"),
args + [limit, offset]).fetchall()
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
return [dict(zip(keys, r)) for r in rows], total
def count_by_source():
with conn() as c:
return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall())
def stats():
with conn() as c:
total = c.execute("SELECT COUNT(*) FROM items").fetchone()[0]