diff --git a/README.md b/README.md index ff89f89..b4e6e6c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ | `firecrawl` | Firecrawl search | `FIRECRAWL_API_KEY` | | `cards` | карточки товаров: JSON-LD Product/Offer, OpenGraph, microdata, цена/валюта/наличие | нет | | `treg` | treg-поиск | `TREG_TOKEN` | -| `feed` | обход реестра из 338 лент/API без запроса (см. ниже) | нет | +| `feed` | обход реестра из 450 лент/API без запроса (см. ниже) | нет | `python -m omni providers` печатает, что реально поднялось в этом окружении. @@ -44,7 +44,7 @@ python -m omni search "scraping twitter without api" --providers reddit,forums,d python -m omni sources python -m omni sources --category papers,security --list -# обход реестра БЕЗ запроса: 338 источников -> база +# обход реестра БЕЗ запроса: 450 источников -> база python -m omni feed --category community --limit 8 python -m omni feed --limit 15 # всё подряд python -m omni feed --only arxiv_cs_ai,hn_top,reddit_localllama @@ -66,7 +66,7 @@ python -m omni export --format jsonl --out data/dump.jsonl ## Реестр источников -`omni/sources.py` + `omni/sources_ext.py` — **338 источников в 9 категориях**, у каждого свой адаптер и флаг вкл/выкл: +`omni/sources.py` + `omni/sources_ext.py` — **450 источников в 9 категориях**, у каждого свой адаптер и флаг вкл/выкл: - **papers 49** — arxiv по 25+ разделам (cs.AI/LG/CL/CV/RO/IR/CR/SE/NE/MA/DC/DB/HC/SD/CY, stat.ML, eess.AS/SP/IV, math.OC, q-bio.NC, cs.GR, cs.AR, cs.PF), HF daily papers, OpenAlex, Semantic Scholar, Papers with Code, Connected Papers - **company 95** — ленты OpenAI, Google AI/Research/DeepMind, Microsoft Research, NVIDIA, Apple ML, Amazon Science, HF, Meta AI, Mistral, Cohere, Together, Perplexity, Groq, Cerebras, Stability, Databricks, Anyscale, Modal, W&B, LangChain, LlamaIndex, vLLM, Replicate, Fireworks, SambaNova, Scale, xAI, Salesforce, AWS ML, Azure AI, Anthropic, AI2, EleutherAI, LAION, Runway, ElevenLabs, Deepgram, AssemblyAI, Pinecone, Weaviate, Qdrant, Chroma, Unsloth, Axolotl @@ -93,7 +93,7 @@ python -m omni digest --out data/digest.md 1. **лента** — счётчики (записей, с ценой, прогонов, источников, включённых), поиск по базе, фильтр по категориям, кнопка «собрать ленту» с прогрессом и разбивкой fetched/new/dup/empty. 2. **источники** — менеджер реестра: - - таблица всех 338 источников: переключатель вкл/выкл, категория, тип, url, статус (жив / мёртв / не проверен), число записей; + - таблица всех 450 источников: переключатель вкл/выкл, категория, тип, url, статус (жив / мёртв / не проверен), число записей; - фильтр по имени/url и по категории; - **подключить источник** — форма (имя, url, категория): тип и адаптер определяются по url автоматически (`/rss` → rss, `.json`/`/api/` → generic_json, `github.com/org` → github_org, `t.me/s/` → tg_web), запись живёт в SQLite и сразу участвует в обходах; - **проверить всё / проверить категорию** — фоновой джоб: реальный фетч каждого источника, запись `status/items/note/checked`, прогресс `done/total, живых/мёртвых`; diff --git a/omni/cli.py b/omni/cli.py index f9db31c..8eae735 100644 --- a/omni/cli.py +++ b/omni/cli.py @@ -164,6 +164,39 @@ def cmd_sources(args): return 0 +def cmd_audit(args): + """Сводка здоровья реестра: по категориям и список мёртвых с причиной.""" + from . import source_state + fl = source_state.flags() + all_s = sources.all_sources() + agg = {} + dead = [] + for s in all_s: + f = fl.get(s.name, {}) + a = agg.setdefault(s.category, {"t": 0, "alive": 0, "dead": 0, "untested": 0}) + a["t"] += 1 + if not f.get("checked"): + a["untested"] += 1 + elif (f.get("items") or 0) > 0: + a["alive"] += 1 + else: + a["dead"] += 1 + dead.append((s.category, s.name, f.get("note") or "empty")) + print("источников: %d | живых: %d | мёртвых: %d | не проверено: %d" % ( + len(all_s), sum(a["alive"] for a in agg.values()), + sum(a["dead"] for a in agg.values()), sum(a["untested"] for a in agg.values()))) + for cat, a in sorted(agg.items(), key=lambda x: -x[1]["t"]): + pct = round(a["alive"] / a["t"] * 100) if a["t"] else 0 + print(" %-12s всего %3d живых %3d мёртвых %3d не проверено %3d (%d%%)" + % (cat, a["t"], a["alive"], a["dead"], a["untested"], pct)) + if args.dead: + print("") + print("мёртвые:") + for cat, name, note in sorted(dead): + print(" %-12s %-28s %s" % (cat, name, note[:50])) + return 0 + + def cmd_feed(args): picked = sources.pick(args.category, args.only, args.tags) if not picked: @@ -265,6 +298,10 @@ def main(argv=None): sp.add_argument("--all", action="store_true", help="показать и выключенные") sp.set_defaults(fn=cmd_sources) + sp = sub.add_parser("audit", help="здоровье реестра источников") + sp.add_argument("--dead", action="store_true", help="показать мёртвые с причиной") + sp.set_defaults(fn=cmd_audit) + sp = sub.add_parser("feed", help="обход реестра источников без запроса") sp.add_argument("--category", default=None) sp.add_argument("--only", default=None) diff --git a/omni/feeds.py b/omni/feeds.py index 7e17586..81b5381 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -28,7 +28,7 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None): ex = dict(extra or {}) ex["tags"] = list(src.tags) return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000], - author=_txt(author)[:80], published=str(published or ""), source=src.category, + author=_txt(author)[:80], published=str(published or ""), source=src.name, extra=ex) @@ -515,6 +515,138 @@ def _nvd(src, limit): return out + +def _biorxiv(src, limit): + import json as _j + d = _json_of(src.url) + rows = d.get("collection") if isinstance(d, dict) else d + out = [] + for r in (rows or [])[:limit]: + title = r.get("title") or "" + doi = r.get("doi") or "" + if not title: + continue + ver = r.get("version") or "1" + host = "medrxiv" if "medrxiv" in src.url else "biorxiv" + url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url + out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800], + str(r.get("authors") or ""), str(r.get("date") or "")[:24])) + return out + + +def _dblp(src, limit): + d = _json_of(src.url) + hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or [] + out = [] + for h in hits[:limit]: + info = h.get("info") or {} + title = info.get("title") or "" + if not title: + continue + auth = info.get("authors") or {} + if isinstance(auth, dict): + a = auth.get("author") + if isinstance(a, list): + auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a) + elif isinstance(a, dict): + auth = a.get("text", "") + out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""), + str(auth or ""), str(info.get("year") or ""))) + return out + + +def _hal(src, limit): + d = _json_of(src.url) + docs = ((d or {}).get("response") or {}).get("docs") or [] + out = [] + for r in docs[:limit]: + t = r.get("title_s") + title = (t[0] if isinstance(t, list) and t else t) or "" + if not title: + continue + ab = r.get("abstract_s") + if isinstance(ab, list): + ab = ab[0] if ab else "" + out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "", + str(r.get("producedDateY_i") or ""))) + return out + + +def _zenodo(src, limit): + d = _json_of(src.url) + rows = ((d or {}).get("hits") or {}).get("hits") or [] + out = [] + for r in rows[:limit]: + md = r.get("metadata") or {} + title = md.get("title") or "" + if not title: + continue + url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or "" + out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800], + str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""), + str(md.get("publication_date") or ""))) + return out + + +def _cve_circl(src, limit): + d = _json_of(src.url) + rows = d if isinstance(d, list) else [] + out = [] + for r in rows[:limit]: + cid = r.get("id") or "" + if not cid: + continue + out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140], + str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24])) + return out + + +def _ghsa(src, limit): + d = _json_of(src.url) + rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else [] + out = [] + for r in rows[:limit]: + gid = r.get("ghsa_id") or "" + title = r.get("summary") or gid + if not title: + continue + sev = r.get("severity") or "" + out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title), + (sev + " | " + (r.get("description") or ""))[:800], "", + str(r.get("published_at") or "")[:24])) + return out + + +def _models_dev(src, limit): + d = _json_of(src.url) + out = [] + if not isinstance(d, dict): + return out + for pid, prov in d.items(): + if not isinstance(prov, dict): + continue + pname = prov.get("name") or pid + models = prov.get("models") or {} + if isinstance(models, dict): + iterable = models.items() + elif isinstance(models, list): + iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)] + else: + continue + for mid, m in iterable: + if not isinstance(m, dict): + continue + name = m.get("name") or mid + out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name), + "ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"), + (m.get("cost") or {}).get("input", "?"), + (m.get("cost") or {}).get("output", "?")), + str(pid), "")) + if len(out) >= limit: + return out + return out + + ADAPTERS = { "rss": _rss, "generic_json": _generic_json, @@ -537,6 +669,14 @@ ADAPTERS = { "xenforo_latest": _xenforo_latest, "tg_web": _tg_web, "html_listing": _html_listing, + "biorxiv": _biorxiv, + "dblp": _dblp, + "hal": _hal, + "zenodo": _zenodo, + "cve_circl": _cve_circl, + "ghsa": _ghsa, + "models_dev": _models_dev, + } diff --git a/omni/serve.py b/omni/serve.py index 57c1f11..5781999 100644 --- a/omni/serve.py +++ b/omni/serve.py @@ -4,7 +4,9 @@ """ import json import pathlib +import re import threading +import time import urllib.parse from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -14,6 +16,13 @@ from . import feeds, net, sources, source_state, store WEB = pathlib.Path(__file__).resolve().parent.parent / "web" JOBS = {} _LOCK = threading.Lock() +SEQ = [0] + + +def _jid(prefix): + with _LOCK: + SEQ[0] += 1 + return "%s%d" % (prefix, SEQ[0]) def _job(jid, **kw): @@ -23,20 +32,79 @@ def _job(jid, **kw): JOBS[jid] = cur +def _cat_health(): + """Сводка по категориям: всего/включено/живых/мёртвых/непроверенных.""" + fl = source_state.flags() + out = {} + for s in sources.all_sources(): + f = fl.get(s.name, {}) + c = out.setdefault(s.category, {"name": s.category, "total": 0, "enabled": 0, + "alive": 0, "dead": 0, "untested": 0}) + c["total"] += 1 + if f.get("enabled", True): + c["enabled"] += 1 + if not f.get("checked"): + c["untested"] += 1 + elif (f.get("items") or 0) > 0: + c["alive"] += 1 + else: + c["dead"] += 1 + return sorted(out.values(), key=lambda x: -x["total"]) + + + +def _clean_txt(v, n=0): + """Снимает html-теги/энтити из title/text, схлопывает пробелы.""" + import html as _h + if not v: + return "" + t = str(v) + t = re.sub(r"<[^>]{1,200}>", " ", t) + for _ in range(2): + t = _h.unescape(t) + t = re.sub(r"\s+", " ", t).strip() + return t[:n].rstrip() if n else t + + +def _fmt_date(v): + """unix-таймстамп -> YYYY-MM-DD, rfc822 оставляет как есть.""" + if not v: + return "" + s = str(v).strip() + if s.isdigit() and len(s) >= 9: + import datetime as _dt + try: + return _dt.datetime.utcfromtimestamp(int(s[:10])).strftime("%Y-%m-%d") + except Exception: + return s + m = re.search(r"(\d{1,2})\s+([A-Za-z]{3})\s+(\d{4})", s) + if m: + return "%s %s %s" % (m.group(1), m.group(2), m.group(3)) + return s[:19] + + +def _names_in_category(cat): + if not cat or cat in ("all", ""): + return None + return [s.name for s in sources.all_sources() if s.category == cat] + + def _run_feed(job_id, category, limit, tags): picked = sources.pick(category, None, tags) + _job(job_id, state="running", total=len(picked), done=0) try: items, report = feeds.fetch_many(picked, limit=limit, workers=8) new, dup = store.add(items) _job(job_id, state="done", sources=len(picked), fetched=len(items), new=new, dup=dup, - empty=sum(1 for _, c in report if not c), - report=sorted(report, key=lambda x: -x[1])[:40]) + empty=sum(1 for _, c in report if not c), done=len(picked), + report=sorted(report, key=lambda x: -x[1])[:60]) except Exception as e: _job(job_id, state="error", error=f"{type(e).__name__}: {e}"[:200]) def _check_one(src, limit=5): note = "" + t0 = time.time() try: items = feeds.fetch(src, limit) except Exception as e: @@ -75,7 +143,7 @@ def _run_check(job_id, names=None, category=None): class Handler(BaseHTTPRequestHandler): - server_version = "omni/1.1" + server_version = "omni/1.2" def log_message(self, *a): pass @@ -105,52 +173,75 @@ class Handler(BaseHTTPRequestHandler): if path == "/api/stats": s = store.stats() + per_source = store.count_by_source() fl = source_state.flags() all_s = sources.all_sources() + cats = _cat_health() + for c in cats: + c["stored"] = sum(per_source.get(x.name, 0) for x in all_s if x.category == c["name"]) return self._json({ "items": s["items"], "priced": s["priced"], "runs": s["runs"], "runs_ok": s["runs_ok"], - "by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]], + "by_provider": [{"name": n, "count": c} for n, c in s["by_provider"][:12]], "by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]], "sources": len(all_s), "sources_enabled": sum(1 for x in all_s if fl.get(x.name, {}).get("enabled", True)), - "categories": sources.stats(), + "alive": sum(1 for x in all_s if (fl.get(x.name, {}).get("items") or 0) > 0), + "tested": sum(1 for x in all_s if fl.get(x.name, {}).get("checked")), + "categories": cats, }) if path == "/api/sources": cat = (q.get("category") or [None])[0] query = (q.get("q") or [""])[0].strip().lower() + state = (q.get("state") or [""])[0] fl = source_state.flags() + per_source = store.count_by_source() custom_names = {d["name"] for d in source_state.custom()} items = [] for s in sources.all_sources(): f = fl.get(s.name, {}) + alive = (f.get("items") or 0) > 0 if cat and cat not in ("all", "") and s.category != cat: continue if query and query not in s.name.lower() and query not in s.url.lower(): continue + if state == "alive" and not alive: + continue + if state == "dead" and not (f.get("checked") and not alive): + continue + if state == "untested" and f.get("checked"): + continue + if state == "off" and f.get("enabled", True): + continue items.append({ "name": s.name, "category": s.category, "kind": s.kind, "adapter": s.adapter, "url": s.url, "tags": list(s.tags), "enabled": f.get("enabled", True), "status": f.get("status"), "items": f.get("items"), "note": f.get("note"), "checked": f.get("checked"), - "custom": s.name in custom_names, + "stored": per_source.get(s.name, 0), "custom": s.name in custom_names, }) + items.sort(key=lambda x: (-(x["stored"] or 0), -(x["items"] or 0), x["name"])) return self._json({ "total": len(sources.all_sources()), "shown": len(items), "enabled": sum(1 for x in items if x["enabled"]), "alive": sum(1 for x in items if (x["items"] or 0) > 0), "dead": sum(1 for x in items if x["checked"] and not (x["items"] or 0)), "untested": sum(1 for x in items if not x["checked"]), - "categories": sources.stats(), "items": items, + "categories": _cat_health(), "items": items, }) if path == "/api/items": cat = (q.get("category") or [None])[0] query = (q.get("q") or [""])[0].strip() - limit = min(int((q.get("limit") or ["60"])[0]), 300) + limit = min(int((q.get("limit") or ["40"])[0]), 200) offset = int((q.get("offset") or ["0"])[0]) - rows = store.browse(category=cat, q=query, limit=limit, offset=offset) - return self._json({"count": len(rows), "items": rows}) + sort = (q.get("sort") or ["score"])[0] + rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort) + for _r in rows: + _r["title"] = _clean_txt(_r.get("title"), 180) + _r["text"] = _clean_txt(_r.get("text"), 260) + _r["published"] = _fmt_date(_r.get("published")) + return self._json({"count": len(rows), "total": total, "offset": offset, "items": rows}) if path.startswith("/api/job/"): jid = path.rsplit("/", 1)[-1] @@ -169,7 +260,7 @@ class Handler(BaseHTTPRequestHandler): body = {} if u.path == "/api/feed": - jid = f"feed{len(JOBS)}" + jid = _jid("feed") _job(jid, state="running") threading.Thread(target=_run_feed, args=(jid, body.get("category"), int(body.get("limit") or 12), body.get("tags")), @@ -182,6 +273,17 @@ class Handler(BaseHTTPRequestHandler): return self._json({"error": "name required"}, 400) return self._json(source_state.set_enabled(name, bool(body.get("enabled", True)))) + if u.path == "/api/sources/toggle_all": + on = bool(body.get("enabled", True)) + cat = body.get("category") + changed = 0 + for s in sources.all_sources(): + if cat and cat not in ("all", "") and s.category != cat: + continue + source_state.set_enabled(s.name, on) + changed += 1 + return self._json({"ok": True, "changed": changed, "enabled": on}) + if u.path == "/api/sources/add": try: res = source_state.add_custom(body.get("name"), body.get("category"), body.get("url"), @@ -200,8 +302,8 @@ class Handler(BaseHTTPRequestHandler): if u.path == "/api/sources/check": names = body.get("names") or ([body["name"]] if body.get("name") else None) - category = None if body.get("all") else body.get("category") - jid = f"check{len(JOBS)}" + category = body.get("category") + jid = _jid("check") _job(jid, state="running") threading.Thread(target=_run_check, args=(jid, names, category), daemon=True).start() return self._json({"job": jid}) diff --git a/omni/sources.py b/omni/sources.py index bbd7400..3c1c129 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -11,6 +11,7 @@ from dataclasses import dataclass, field from .sources_ext import EXTRA +from .sources_ext2 import EXTRA2 @dataclass @@ -177,7 +178,17 @@ SOURCES_BASE = [ S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)), ] -SOURCES = SOURCES_BASE + [Source(**d) for d in EXTRA] +def _dedup(rows): + seen, out = set(), [] + for s in rows: + if s.url in seen: + continue + seen.add(s.url) + out.append(s) + return out + + +SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2]) BY_NAME = {s.name: s for s in SOURCES} CATEGORIES = sorted({s.category for s in SOURCES}) diff --git a/omni/sources_ext2.py b/omni/sources_ext2.py new file mode 100644 index 0000000..ef13a5d --- /dev/null +++ b/omni/sources_ext2.py @@ -0,0 +1,118 @@ +"""Волна 2+3: источники, прошедшие живой фетч (items > 0). Сгенерировано probe_w23.py.""" + +EXTRA2 = [ + dict(name="gaia_bench", category="benchmark", kind="html", url="https://huggingface.co/spaces/gaia-benchmark/leaderboard", adapter="html_listing", tags=["bench"], limit=20, detail=False), + dict(name="osworld_leaderboard", category="benchmark", kind="html", url="https://os-world.github.io/", adapter="html_listing", tags=["bench"], limit=20, detail=False), + dict(name="swe_rebench", category="benchmark", kind="html", url="https://swe-rebench.com/", adapter="html_listing", tags=["bench"], limit=20, detail=False), + dict(name="terminal_bench", category="benchmark", kind="html", url="https://www.tbench.ai/leaderboard", adapter="html_listing", tags=["bench"], limit=20, detail=False), + dict(name="coding_horror", category="community", kind="rss", url="https://blog.codinghorror.com/rss/", adapter="rss", tags=["blog"], limit=20, detail=False), + dict(name="danluu_blog", category="community", kind="rss", url="https://danluu.com/atom.xml", adapter="rss", tags=["blog"], limit=20, detail=False), + dict(name="jay_alammar", category="community", kind="rss", url="https://jalammar.github.io/feed.xml", adapter="rss", tags=["ml", "explainer"], limit=20, detail=False), + dict(name="jvns_blog", category="community", kind="rss", url="https://jvns.ca/atom.xml", adapter="rss", tags=["blog"], limit=20, detail=False), + dict(name="kentcdodds", category="community", kind="rss", url="https://kentcdodds.com/blog/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="lilianweng", category="community", kind="rss", url="https://lilianweng.github.io/index.xml", adapter="rss", tags=["ml", "research"], limit=20, detail=False), + dict(name="lobsters_databases", category="community", kind="rss", url="https://lobste.rs/t/databases.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False), + dict(name="lobsters_linux", category="community", kind="rss", url="https://lobste.rs/t/linux.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False), + dict(name="lobsters_ml", category="community", kind="rss", url="https://lobste.rs/t/ml.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False), + dict(name="lobsters_performance", category="community", kind="rss", url="https://lobste.rs/t/performance.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False), + dict(name="lobsters_web", category="community", kind="rss", url="https://lobste.rs/t/web.rss", adapter="rss", tags=["lobsters"], limit=20, detail=False), + dict(name="lwn_headlines", category="community", kind="rss", url="https://lwn.net/headlines/rss", adapter="rss", tags=["linux"], limit=20, detail=False), + dict(name="martinfowler", category="community", kind="rss", url="https://martinfowler.com/feed.atom", adapter="rss", tags=["arch"], limit=20, detail=False), + dict(name="neuromancer_ai", category="community", kind="rss", url="https://www.interconnects.ai/feed", adapter="rss", tags=["llm", "analysis"], limit=20, detail=False), + dict(name="overreacted", category="community", kind="rss", url="https://overreacted.io/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="phoronix", category="community", kind="rss", url="https://www.phoronix.com/rss.php", adapter="rss", tags=["linux", "hardware"], limit=20, detail=False), + dict(name="reddit_cpp", category="community", kind="reddit", url="https://www.reddit.com/r/cpp/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_docker", category="community", kind="reddit", url="https://www.reddit.com/r/docker/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_flutter", category="community", kind="reddit", url="https://www.reddit.com/r/FlutterDev/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_gamedev", category="community", kind="reddit", url="https://www.reddit.com/r/gamedev/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_gcp", category="community", kind="reddit", url="https://www.reddit.com/r/googlecloud/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_homelab", category="community", kind="reddit", url="https://www.reddit.com/r/homelab/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_llm", category="community", kind="reddit", url="https://www.reddit.com/r/LLM/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_localllm", category="community", kind="reddit", url="https://www.reddit.com/r/LocalLLM/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_php", category="community", kind="reddit", url="https://www.reddit.com/r/PHP/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="reddit_swift", category="community", kind="reddit", url="https://www.reddit.com/r/swift/", adapter="reddit_listing", tags=["reddit"], limit=20, detail=False), + dict(name="scott_helme", category="community", kind="rss", url="https://scotthelme.co.uk/rss/", adapter="rss", tags=["tls"], limit=20, detail=False), + dict(name="sebastian_raschka", category="community", kind="rss", url="https://magazine.sebastianraschka.com/feed", adapter="rss", tags=["llm", "ml"], limit=20, detail=False), + dict(name="simonwillison", category="community", kind="rss", url="https://simonwillison.net/atom/everything/", adapter="rss", tags=["llm", "blog"], limit=20, detail=False), + dict(name="tg_addmetana", category="community", kind="tg", url="https://t.me/s/addmeto", adapter="tg_web", tags=["tg", "ru"], limit=20, detail=False), + dict(name="the_gradient", category="community", kind="rss", url="https://thegradient.pub/rss/", adapter="rss", tags=["ml"], limit=20, detail=False), + dict(name="astro_blog", category="company", kind="rss", url="https://astro.build/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="bun_blog", category="company", kind="rss", url="https://bun.sh/rss.xml", adapter="rss", tags=["runtime"], limit=20, detail=False), + dict(name="dagster_blog", category="company", kind="rss", url="https://dagster.io/blog/rss.xml", adapter="rss", tags=["data"], limit=20, detail=False), + dict(name="datadog_blog", category="company", kind="rss", url="https://www.datadoghq.com/blog/index.xml", adapter="rss", tags=["obs"], limit=20, detail=False), + dict(name="dbt_blog", category="company", kind="rss", url="https://www.getdbt.com/blog/rss.xml", adapter="rss", tags=["data"], limit=20, detail=False), + dict(name="deno_blog", category="company", kind="rss", url="https://deno.com/feed", adapter="rss", tags=["runtime"], limit=20, detail=False), + dict(name="fal_blog", category="company", kind="rss", url="https://blog.fal.ai/rss/", adapter="rss", tags=["inference", "image"], limit=20, detail=False), + dict(name="gh_axolotl", category="company", kind="github", url="axolotl-ai-cloud", adapter="github_org", tags=["finetune"], limit=20, detail=False), + dict(name="gh_crewai", category="company", kind="github", url="crewAIInc", adapter="github_org", tags=["agents"], limit=20, detail=False), + dict(name="gh_litellm", category="company", kind="github", url="BerriAI", adapter="github_org", tags=["gateway"], limit=20, detail=False), + dict(name="gh_llamaindex", category="company", kind="github", url="run-llama", adapter="github_org", tags=["rag"], limit=20, detail=False), + dict(name="gh_mlc", category="company", kind="github", url="mlc-ai", adapter="github_org", tags=["local"], limit=20, detail=False), + dict(name="gh_triton", category="company", kind="github", url="triton-inference-server", adapter="github_org", tags=["inference"], limit=20, detail=False), + dict(name="go_blog", category="company", kind="rss", url="https://go.dev/blog/feed.atom", adapter="rss", tags=["lang"], limit=20, detail=False), + dict(name="grafana_blog", category="company", kind="rss", url="https://grafana.com/blog/index.xml", adapter="rss", tags=["obs"], limit=20, detail=False), + dict(name="nextjs_blog", category="company", kind="rss", url="https://nextjs.org/feed.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="node_blog", category="company", kind="rss", url="https://nodejs.org/en/feed/blog.xml", adapter="rss", tags=["runtime"], limit=20, detail=False), + dict(name="prisma_blog", category="company", kind="rss", url="https://www.prisma.io/blog/rss.xml", adapter="rss", tags=["db"], limit=20, detail=False), + dict(name="python_insider", category="company", kind="rss", url="https://pythoninsider.blogspot.com/feeds/posts/default", adapter="rss", tags=["python"], limit=20, detail=False), + dict(name="react_blog", category="company", kind="rss", url="https://react.dev/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="rust_blog", category="company", kind="rss", url="https://blog.rust-lang.org/feed.xml", adapter="rss", tags=["lang"], limit=20, detail=False), + dict(name="svelte_blog", category="company", kind="rss", url="https://svelte.dev/blog/rss.xml", adapter="rss", tags=["frontend"], limit=20, detail=False), + dict(name="tg_anthropic", category="company", kind="tg", url="https://t.me/s/AnthropicAI", adapter="tg_web", tags=["tg"], limit=20, detail=False), + dict(name="tg_googleai", category="company", kind="tg", url="https://t.me/s/googleai", adapter="tg_web", tags=["tg"], limit=20, detail=False), + dict(name="timescale_blog", category="company", kind="rss", url="https://www.timescale.com/blog/rss/", adapter="rss", tags=["db"], limit=20, detail=False), + dict(name="coupang_tech", category="company_kr", kind="rss", url="https://medium.com/feed/coupang-tech", adapter="rss", tags=["kr"], limit=20, detail=False), + dict(name="bubble_forum", category="forum", kind="json", url="https://forum.bubble.io/latest.json", adapter="discourse", tags=["nocode"], limit=20, detail=False), + dict(name="discourse_meta", category="forum", kind="json", url="https://meta.discourse.org/latest.json", adapter="discourse", tags=["meta"], limit=20, detail=False), + dict(name="discuss_jupyter", category="forum", kind="json", url="https://discourse.jupyter.org/latest.json", adapter="discourse", tags=["data"], limit=20, detail=False), + dict(name="discuss_kotlin", category="forum", kind="json", url="https://discuss.kotlinlang.org/latest.json", adapter="discourse", tags=["kotlin"], limit=20, detail=False), + dict(name="discuss_kubernetes", category="forum", kind="json", url="https://discuss.kubernetes.io/latest.json", adapter="discourse", tags=["infra"], limit=20, detail=False), + dict(name="fastai_forum", category="forum", kind="json", url="https://forums.fast.ai/latest.json", adapter="discourse", tags=["ml"], limit=20, detail=False), + dict(name="macrumors_forum", category="forum", kind="rss", url="https://forums.macrumors.com/forums/-/index.rss", adapter="rss", tags=["apple"], limit=20, detail=False), + dict(name="make_community", category="forum", kind="json", url="https://community.make.com/latest.json", adapter="discourse", tags=["automation"], limit=20, detail=False), + dict(name="pixls_forum", category="forum", kind="json", url="https://discuss.pixls.us/latest.json", adapter="discourse", tags=["photo"], limit=20, detail=False), + dict(name="proxmox_forum", category="forum", kind="rss", url="https://forum.proxmox.com/forums/-/index.rss", adapter="rss", tags=["virt"], limit=20, detail=False), + dict(name="streamlit_forum", category="forum", kind="json", url="https://discuss.streamlit.io/latest.json", adapter="discourse", tags=["data"], limit=20, detail=False), + dict(name="xda_forum", category="forum", kind="rss", url="https://xdaforums.com/forums/-/index.rss", adapter="rss", tags=["mobile"], limit=20, detail=False), + dict(name="hf_datasets_recent", category="models", kind="json", url="https://huggingface.co/api/datasets?sort=lastModified&limit=20", adapter="hf_generic", tags=["hf", "dataset"], limit=20, detail=False), + dict(name="hf_models_audio", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=automatic-speech-recognition&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "audio"], limit=20, detail=False), + dict(name="hf_models_ocr", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=image-text-to-text&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "vlm"], limit=20, detail=False), + dict(name="hf_models_video", category="models", kind="json", url="https://huggingface.co/api/models?pipeline_tag=text-to-video&sort=downloads&limit=20", adapter="hf_models", tags=["hf", "video"], limit=20, detail=False), + dict(name="arxiv_cs_cc", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CC", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_cs_ce", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CE", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_cs_cg", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.CG", adapter="rss", tags=["paper", "arxiv", "graphics"], limit=20, detail=False), + dict(name="arxiv_cs_dm", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.DM", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_cs_gl", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.GL", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_cs_oh", category="papers", kind="rss", url="https://rss.arxiv.org/rss/cs.OH", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_econ_gn", category="papers", kind="rss", url="https://rss.arxiv.org/rss/econ.GN", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_eess_sy", category="papers", kind="rss", url="https://rss.arxiv.org/rss/eess.SY", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_math_st", category="papers", kind="rss", url="https://rss.arxiv.org/rss/math.ST", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_physics_socph", category="papers", kind="rss", url="https://rss.arxiv.org/rss/physics.soc-ph", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_qbio_qm", category="papers", kind="rss", url="https://rss.arxiv.org/rss/q-bio.QM", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_qfin_cp", category="papers", kind="rss", url="https://rss.arxiv.org/rss/q-fin.CP", adapter="rss", tags=["paper", "arxiv", "finance"], limit=20, detail=False), + dict(name="arxiv_stat_co", category="papers", kind="rss", url="https://rss.arxiv.org/rss/stat.CO", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="arxiv_stat_me", category="papers", kind="rss", url="https://rss.arxiv.org/rss/stat.ME", adapter="rss", tags=["paper", "arxiv"], limit=20, detail=False), + dict(name="biorxiv_recent", category="papers", kind="json", url="https://api.biorxiv.org/details/biorxiv/2026-09-01/2026-10-25/0", adapter="biorxiv", tags=["paper", "bio"], limit=20, detail=False), + dict(name="hal_ai", category="papers", kind="json", url="https://api.archives-ouvertes.fr/search/?q=llm&fl=title_s,uri_s,abstract_s,producedDateY_i&rows=25&wt=json", adapter="hal", tags=["paper", "hal"], limit=20, detail=False), + dict(name="medrxiv_recent", category="papers", kind="json", url="https://api.medrxiv.org/details/medrxiv/2026-09-01/2026-10-25/0", adapter="biorxiv", tags=["paper", "med"], limit=20, detail=False), + dict(name="zenodo_ai", category="papers", kind="json", url="https://zenodo.org/api/records?q=large+language+model&sort=newest&size=20", adapter="zenodo", tags=["paper", "zenodo"], limit=20, detail=False), + dict(name="zenodo_security", category="papers", kind="json", url="https://zenodo.org/api/records?q=vulnerability&sort=newest&size=20", adapter="zenodo", tags=["paper", "zenodo"], limit=20, detail=False), + dict(name="0xdf_blog", category="security", kind="rss", url="https://0xdf.gitlab.io/feed.xml", adapter="rss", tags=["bb", "htb"], limit=20, detail=False), + dict(name="ctftime_upcoming", category="security", kind="rss", url="https://ctftime.org/event/list/upcoming/rss/", adapter="rss", tags=["ctf"], limit=20, detail=False), + dict(name="cve_circl", category="security", kind="json", url="https://cve.circl.lu/api/last/30", adapter="cve_circl", tags=["cve"], limit=20, detail=False), + dict(name="dfir_report", category="security", kind="rss", url="https://thedfirreport.com/feed/", adapter="rss", tags=["dfir"], limit=20, detail=False), + dict(name="gitguardian_blog", category="security", kind="rss", url="https://blog.gitguardian.com/rss/", adapter="rss", tags=["secrets"], limit=20, detail=False), + dict(name="hackread", category="security", kind="rss", url="https://hackread.com/feed/", adapter="rss", tags=["news"], limit=20, detail=False), + dict(name="huntress_blog", category="security", kind="rss", url="https://www.huntress.com/blog/rss.xml", adapter="rss", tags=["dfir"], limit=20, detail=False), + dict(name="intigriti_blog", category="security", kind="rss", url="https://blog.intigriti.com/feed/", adapter="rss", tags=["bb"], limit=20, detail=False), + dict(name="jfrog_blog", category="security", kind="rss", url="https://jfrog.com/blog/feed/", adapter="rss", tags=["supply"], limit=20, detail=False), + dict(name="medium_bugbounty", category="security", kind="rss", url="https://medium.com/feed/tag/bug-bounty", adapter="rss", tags=["bb", "writeups"], limit=20, detail=False), + dict(name="medium_pentest", category="security", kind="rss", url="https://medium.com/feed/tag/penetration-testing", adapter="rss", tags=["pentest"], limit=20, detail=False), + dict(name="medium_redteam", category="security", kind="rss", url="https://medium.com/feed/tag/red-team", adapter="rss", tags=["redteam"], limit=20, detail=False), + dict(name="risky_business", category="security", kind="rss", url="https://risky.biz/feeds/risky-business-news/", adapter="rss", tags=["news"], limit=20, detail=False), + dict(name="schneier_blog", category="security", kind="rss", url="https://www.schneier.com/blog/atom.xml", adapter="rss", tags=["crypto", "privacy"], limit=20, detail=False), + dict(name="seclists_oss", category="security", kind="rss", url="https://seclists.org/rss/oss-sec.rss", adapter="rss", tags=["oss", "cve"], limit=20, detail=False), + dict(name="sonatype_blog", category="security", kind="rss", url="https://blog.sonatype.com/rss.xml", adapter="rss", tags=["supply"], limit=20, detail=False), + dict(name="troy_hunt", category="security", kind="rss", url="https://www.troyhunt.com/rss/", adapter="rss", tags=["breach"], limit=20, detail=False), + dict(name="watchtowr_blog", category="security", kind="rss", url="https://labs.watchtowr.com/rss/", adapter="rss", tags=["research"], limit=20, detail=False), +] diff --git a/omni/store.py b/omni/store.py index 5de79ad..e7b1415 100644 --- a/omni/store.py +++ b/omni/store.py @@ -150,6 +150,29 @@ def browse(category=None, q="", limit=60, offset=0): return [dict(zip(keys, r)) for r in rows] +def browse2(q="", names=None, limit=60, offset=0, sort="score"): + """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total.""" + where = "WHERE 1=1 " + args = [] + if names: + where += "AND source IN (%s) " % ",".join("?" * len(names)) + args += list(names) + if q: + where += "AND (title LIKE ? OR text LIKE ?) " + args += [f"%{q}%", f"%{q}%"] + with conn() as c: + total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0] + rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url " + "FROM items " + where + ("ORDER BY published DESC, id DESC LIMIT ? OFFSET ?" if sort == "fresh" else "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"), + args + [limit, offset]).fetchall() + keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url") + return [dict(zip(keys, r)) for r in rows], total + + +def count_by_source(): + with conn() as c: + return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall()) + def stats(): with conn() as c: total = c.execute("SELECT COUNT(*) FROM items").fetchone()[0] diff --git a/web/index.html b/web/index.html index f77f473..f2746b2 100644 --- a/web/index.html +++ b/web/index.html @@ -1,4 +1,4 @@ - + @@ -6,320 +6,630 @@ omni · радар источников -
-
- -
загрузка…
-
- -
- - -
- - -
-
-
- - - +
+
- -
-
-
- - - - - - +
+ +
-
-
-
подключить источник
-
- - - - +
категории
+
+
живыемёртвыене проверены
+ +
загрузка…
+ + +
+
+

Лентасвежие записи со всех источников

+
+
+ ⌕ + + /
+
-
—статус · записей
-
-
+
+ +
+
+
+
+ +
+ + + +
+
-
+ +
+ работаю… +
+ +
+ + + +
\ No newline at end of file