abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)

.local/diag.py

@@ -0,0 +1,28 @@
+import sys, json
+sys.path.insert(0,'.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net
+
+URLS = [
+ "https://darknetlive.com/feed/", "https://darknetlive.com/rss/", "https://darknetlive.com/rss.xml",
+ "https://www.volexity.com/blog/feed/", "https://cxsecurity.com/rss/",
+ "https://www.debian.org/security/", "https://www.openwall.com/lists/oss-security/",
+ "https://www.t00ls.com/", "https://xz.aliyun.com/",
+ "https://t.me/s/TheHackersNews", "https://t.me/s/thehackernews", "https://t.me/s/SecAtor",
+ "https://t.me/s/true_secator", "https://t.me/s/databreaches", "https://t.me/s/cybersecuritynews",
+]
+def one(u):
+    try:
+        r = net.get(u, timeout=25)
+        t = r.text or ""
+        return {"u":u,"st":r.status,"n":len(t),
+                "ct":(r.headers or {}).get("content-type","")[:40],
+                "item":t.count("<item"),"entry":t.count("<entry"),
+                "tgmsg":t.count("tgme_widget_message_wrap"),
+                "hdr":t.count('tgme_channel_info_header_title'),
+                "head": " ".join(t[:200].split())[:120]}
+    except Exception as e:
+        return {"u":u,"st":0,"err":str(e)[:90]}
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(one, URLS):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag2.py

@@ -0,0 +1,33 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net, feeds
+from omni.sources import Source
+
+TG = ["thehackernews", "true_secator", "cybersecuritynews", "ransomwatch", "ransomware_news", "leakbase",
+      "CVEnew", "cvefeed", "hackernews", "cyberdefensemagazine", "securityweek", "therecord",
+      "ciscotalos", "unit42", "projectzero", "huntress", "sentinelone", "recordedfuture",
+      "threatmon", "kalilinux", "darkwebinformer", "darkwebinformer_ru", "breachdetector",
+      "databreach", "ruhack", "expl0it", "infosec_ru", "securitylab_ru", "itsecforu_ru",
+      "xakep", "XakepRU", "bo0om_ru", "mycopypaste", "cnews_ru", "bdu_fstec", "kaspersky",
+      "kaspersky_ru", "talos", "smd_ru", "kernelmode", "hitech_crime", "codeby"]
+
+
+def tg_one(h, url):
+    try:
+        r = net.get(url, timeout=25)
+        t = r.text or ""
+        title = re.search(r"<title>(.*?)</title>", t, re.S)
+        src = Source(name="probe_" + h, category="security", kind="html", url=url, adapter="tg_web",
+                     tags=("tg", "leak"), limit=5)
+        items = feeds.fetch(src, 5)
+        return {"h": h, "st": r.status, "n": len(t), "msg": t.count("tgme_widget_message_wrap"),
+                "items": len(items), "title": (title.group(1).strip()[:60] if title else ""),
+                "first": (items[0].title[:70] if items else "")}
+    except Exception as e:
+        return {"h": h, "st": 0, "err": str(e)[:80]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(lambda h: tg_one(h, "https://t.me/s/" + h), TG):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag3.py

@@ -0,0 +1,41 @@
+import sys, json, re, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net
+from bs4 import BeautifulSoup
+
+URLS = [
+    "https://www.debian.org/security/",
+    "https://www.openwall.com/lists/oss-security/",
+    "https://www.t00ls.com/",
+    "https://www.anquanke.com/",
+    "https://www.freebuf.com/",
+    "https://bbs.kanxue.com/",
+    "https://www.secrss.com/",
+    "https://www.ichunqiu.com/",
+]
+
+
+def one(u):
+    try:
+        r = net.get(u, timeout=25)
+        soup = BeautifulSoup(r.text or "", "lxml")
+        hrefs = []
+        for a in soup.find_all("a", href=True):
+            h = urllib.parse.urljoin(u, str(a["href"]))
+            p = urllib.parse.urlsplit(h).path
+            hrefs.append(p)
+        pat = {}
+        for p in hrefs:
+            seg = re.sub(r"\d+", "N", p)[:60]
+            pat[seg] = pat.get(seg, 0) + 1
+        top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
+        return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
+                "paths": top}
+    except Exception as e:
+        return {"u": u, "st": 0, "err": str(e)[:90]}
+
+
+with ThreadPoolExecutor(4) as ex:
+    for r in ex.map(one, URLS):
+        print(json.dumps(r, ensure_ascii=False))

omni/abuse.py

@@ -167,14 +167,15 @@ def _connect():
     return con
 
 
-def rebuild(full=False, chunk=4000):
+def rebuild(full=False, chunk=4000, auto=False):
     """Инкрементальный индекс абузов: classify по новым items, без полного перещёлкивания.
 
     full=True (или OMNI_ABUSE_REBUILD=1) — пересчитать всю базу после правки правил.
+    auto=True (вызов из чтения) игнорирует env, иначе каждый scan делал бы полный проход.
     Возвращает число записей, попавших в индекс на этом проходе.
     """
     import os
-    if os.environ.get("OMNI_ABUSE_REBUILD") == "1":
+    if not auto and os.environ.get("OMNI_ABUSE_REBUILD") == "1":
         full = True
     con = _connect()
     try:
@@ -207,7 +208,7 @@ def rebuild(full=False, chunk=4000):
 def _rows(limit=400, tag=None, q=None, since_days=None):
     """Читает из индекса abuse_idx; индекс добивается свежими записями перед чтением."""
     try:
-        rebuild()
+        rebuild(auto=True)
     except Exception:
         pass
     sql = ("SELECT i.id, i.source, i.url, i.title, i.text, i.published, a.tags, i.score, a.score "

omni/panel_api.py

@@ -0,0 +1,316 @@
+"""Данные для панели v2: обзор, детали записи, TG-сессия, дайджест, watchlist.
+
+Здесь только чтение и небольшие записи в data/*.json; HTTP-обвязка — в serve.py,
+чтобы роуты оставались тонкими, а логика — тестируемой без поднятия сервера.
+"""
+import datetime as _dt
+import json
+import os
+import re
+import time
+import urllib.parse
+
+from . import abuse as abuse_mod
+from . import config, sources, source_state, store, taxonomy
+
+BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
+          "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
+
+
+def _clean(v, n=0):
+    import html as _h
+    if not v:
+        return ""
+    t = re.sub(r"<[^>]{1,200}>", " ", str(v))
+    for _ in range(2):
+        t = _h.unescape(t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t[:n].rstrip() if n else t
+
+
+# ---------- обзор ----------
+
+def timeline(days=30, cat=None):
+    """Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями."""
+    days = max(1, min(365, int(days or 30)))
+    names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
+    where, args = "", []
+    if names is not None:
+        where = " WHERE source IN (%s)" % ",".join("?" * len(names))
+        args = list(names)
+    sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
+           " SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
+           " FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
+    got = {}
+    with store.db() as c:
+        for b, n, cve, sev in c.execute(sql, args):
+            if b:
+                got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
+    today = _dt.date.today()
+    out = []
+    for i in range(days - 1, -1, -1):
+        d = (today - _dt.timedelta(days=i)).isoformat()
+        n, cve, sev = got.get(d, (0, 0, 0))
+        out.append({"day": d, "n": n, "cve": cve, "sev": sev})
+    return {"ok": True, "days": days, "points": out,
+            "total": sum(p["n"] for p in out),
+            "max": max([p["n"] for p in out] or [0])}
+
+
+def item_detail(iid):
+    """Полная запись + соседи: свежее из того же источника и с того же домена."""
+    iid = int(iid)
+    keys = ("id", "provider", "kind", "source", "url", "title", "text", "author", "published",
+            "lang", "tags", "price", "currency", "score", "types")
+    with store.db() as c:
+        row = c.execute("SELECT " + ",".join(keys) + " FROM items WHERE id=?", (iid,)).fetchone()
+        if not row:
+            return {"error": "not found"}
+        it: dict = dict(zip(keys, row))
+        host = ""
+        try:
+            host = urllib.parse.urlsplit(it.get("url") or "").netloc.lower()
+        except Exception:
+            host = ""
+        same_src = c.execute("SELECT id, title, source, published FROM items WHERE source=? AND id!=? "
+                             "ORDER BY id DESC LIMIT 6", (it["source"], iid)).fetchall()
+        same_host = []
+        if host:
+            same_host = c.execute("SELECT id, title, source, published FROM items "
+                                  "WHERE url LIKE ? AND id!=? ORDER BY id DESC LIMIT 6",
+                                  ("%//" + host + "/%", iid)).fetchall()
+    it["title"] = _clean(it.get("title"), 400)
+    it["text"] = _clean(it.get("text"))
+    it["types"] = [t for t in (it.get("types") or "").split(",") if t]
+    it["types_labels"] = taxonomy.labels(it["types"])
+    it["tags"] = [t for t in (it.get("tags") or "").split(",") if t] if isinstance(it.get("tags"), str) else (it.get("tags") or [])
+    it["related_same_source"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
+                                 for r in same_src]
+    it["related_same_host"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
+                               for r in same_host]
+    return {"ok": True, "item": it}
+
+
+def suggest(q, limit=8):
+    """Подсказки поиска: заголовки из базы + имена источников + категории."""
+    q = (q or "").strip()
+    out = {"titles": [], "sources": [], "categories": []}
+    if len(q) < 2:
+        return out
+    like = "%" + q + "%"
+    with store.db() as c:
+        for iid, title, src in c.execute("SELECT id,title,source FROM items WHERE title LIKE ? "
+                                         "ORDER BY id DESC LIMIT ?", (like, limit)):
+            out["titles"].append({"id": iid, "title": _clean(title, 120), "source": src})
+    ql = q.lower()
+    for s in sources.all_sources():
+        if ql in s.name.lower() or ql in (s.url or "").lower():
+            out["sources"].append({"name": s.name, "category": s.category})
+            if len(out["sources"]) >= limit:
+                break
+    cats = {}
+    for s in sources.all_sources():
+        cats[s.category] = cats.get(s.category, 0) + 1
+    out["categories"] = [{"name": k, "n": v} for k, v in sorted(cats.items()) if ql in k.lower()][:5]
+    return out
+
+
+def health():
+    st = store.stats()
+    fl = source_state.flags()
+    all_s = sources.all_sources()
+    return {"ok": True, "items": st["items"], "sources": len(all_s),
+            "alive": sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0),
+            "runs": st["runs"], "uptime": int(time.time() - _START),
+            "db_mb": round(config.DB.stat().st_size / 1048576, 1) if config.DB.exists() else 0}
+
+
+_START = time.time()
+
+
+# ---------- telegram ----------
+
+def tg_status():
+    from . import tgsess
+    out = {"enabled": bool(tgsess.enabled()), "limit": 100, "me": None, "error": None}
+    try:
+        cfg = config.tg_cfg()
+        out["limit"] = int(cfg.get("limit") or 100)
+        out["session"] = os.path.basename(str(cfg.get("session") or "")) or ""
+    except Exception:
+        pass
+    if out["enabled"]:
+        try:
+            out["me"] = tgsess.me()
+        except Exception as e:
+            out["error"] = "%s: %s" % (type(e).__name__, str(e)[:160])
+    return out
+
+
+def tg_search(q, limit=40, min_members=0):
+    from . import tgsess
+    if not tgsess.enabled():
+        return {"ok": False, "error": "сессия не настроена", "channels": []}
+    rows = tgsess.search(q, limit=limit)
+    rows = [r for r in rows if int(r.get("participants") or 0) >= int(min_members or 0)]
+    rows.sort(key=lambda r: -(r.get("participants") or 0))
+    return {"ok": True, "query": q, "channels": rows}
+
+
+def tg_pull(handles, limit=100):
+    """Тянет посты через сессию и складывает в общий склад. Возвращает счётчики."""
+    from . import tgsess
+    from .model import Item
+    if not tgsess.enabled():
+        return {"ok": False, "error": "сессия не настроена", "new": 0}
+    chans = tgsess.fetch([h for h in handles if h], limit=limit)
+    items, errors = [], []
+    for ch in chans:
+        if ch.get("error"):
+            errors.append({"handle": ch.get("handle"), "error": ch["error"]})
+        for p in ch.get("posts") or []:
+            items.append(Item("tg_sess", "post", url=p.get("url") or "",
+                              title=p.get("title") or "", text=p.get("text") or "",
+                              source="tg_sess_" + str(ch.get("handle") or ""),
+                              published=(p.get("date") or "")[:10],
+                              extra={"views": p.get("views"), "handle": ch.get("handle")}))
+    new, dup = store.add(items) if items else (0, 0)
+    return {"ok": True, "channels": len(chans), "fetched": len(items), "new": new, "dup": dup,
+            "errors": errors[:20]}
+
+
+# ---------- дайджест ----------
+
+def digest(days=1, limit=30):
+    """Markdown-дайджест: главное за период по score, уязвимости, сливы, здоровье реестра."""
+    days = max(1, min(30, int(days or 1)))
+    since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
+    rows, total = store.browse2(limit=limit, offset=0, sort="score", since=since)
+    cves, n_cve = store.browse2(limit=12, offset=0, sort="fresh", typ="cve", since=since)
+    leaks, n_leak = store.browse2(limit=10, offset=0, sort="fresh", typ="leak", since=since)
+    fl = source_state.flags()
+    all_s = sources.all_sources()
+    alive = sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0)
+    ab = abuse_mod.scan(limit=8)
+    L = []
+    L.append("# omni · дайджест за %d дн. (%s)" % (days, _dt.date.today().isoformat()))
+    L.append("")
+    L.append("Записей за период: **%d**. Реестр: %d источников, живых %d." % (total, len(all_s), alive))
+    L.append("")
+    L.append("## Главное по score")
+    for r in rows[:15]:
+        t = _clean(r.get("title"), 160)
+        L.append("- [%s](%s) — `%s` · %s" % (t, r.get("url") or "#", r.get("source"), r.get("published") or ""))
+    L.append("")
+    L.append("## Уязвимости (%d)" % n_cve)
+    for r in cves:
+        L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
+    L.append("")
+    L.append("## Сливы и утечки (%d)" % n_leak)
+    for r in leaks:
+        L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
+    L.append("")
+    L.append("## Абузы: промо, триалы, ключи")
+    for it in (ab.get("items") or [])[:8]:
+        L.append("- [%s](%s) — `%s` · score %s" % (_clean(it.get("title"), 140), it.get("url") or "#",
+                                                   it.get("source"), it.get("abuse_score")))
+    md = "\n".join(L)
+    return {"ok": True, "days": days, "md": md, "n": total, "path": str(_write_digest(md))}
+
+
+def _write_digest(md):
+    p = config.DATA / "digest_latest.md"
+    try:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(md, encoding="utf-8")
+    except Exception:
+        pass
+    return p
+
+
+# ---------- watchlist ----------
+
+WATCH = config.DATA / "watch.json"
+
+
+def _watch_load():
+    try:
+        d = json.loads(WATCH.read_text(encoding="utf-8"))
+        if isinstance(d, list):
+            return d
+    except Exception:
+        pass
+    return []
+
+
+def _watch_save(rows):
+    try:
+        WATCH.parent.mkdir(parents=True, exist_ok=True)
+        WATCH.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
+    except Exception:
+        pass
+
+
+def watch_list(hits=3):
+    """Правила + сколько записей подошло после добавления и 3 последних примера."""
+    rows = _watch_load()
+    out = []
+    for r in rows:
+        q = (r.get("q") or "").strip()
+        typ = r.get("type") or "all"
+        since = _dt.datetime.fromtimestamp(int(r.get("added") or 0), _dt.timezone.utc).date().isoformat()
+        items, n = store.browse2(q=q, limit=hits, offset=0, sort="fresh", typ=typ, since=since)
+        out.append({**r, "count": n,
+                    "latest": [{"id": x.get("id"), "title": _clean(x.get("title"), 120),
+                                "source": x.get("source"), "url": x.get("url")} for x in items]})
+    return {"ok": True, "rules": out}
+
+
+def watch_add(q, typ="all"):
+    q = (q or "").strip()
+    if len(q) < 2:
+        return {"error": "нужно минимум 2 символа"}
+    rows = _watch_load()
+    for r in rows:
+        if r.get("q", "").lower() == q.lower() and r.get("type", "all") == (typ or "all"):
+            return {"ok": True, "rule": r, "existed": True}
+    rule = {"id": "%x" % int(time.time() * 1000), "q": q, "type": typ or "all", "added": int(time.time())}
+    rows.append(rule)
+    _watch_save(rows)
+    return {"ok": True, "rule": rule}
+
+
+def watch_remove(rid):
+    rows = [r for r in _watch_load() if r.get("id") != rid]
+    _watch_save(rows)
+    return {"ok": True, "left": len(rows)}
+
+
+# ---------- источник подробно ----------
+
+def source_detail(name):
+    fl = source_state.flags()
+    src = next((s for s in sources.all_sources() if s.name == name), None)
+    if not src:
+        return {"error": "not found"}
+    with store.db() as c:
+        stored = c.execute("SELECT COUNT(*) FROM items WHERE source=?", (name,)).fetchone()[0]
+        rows = c.execute("SELECT id,title,url,types,published,score FROM items WHERE source=? "
+                         "ORDER BY id DESC LIMIT 12", (name,)).fetchall()
+        runs = c.execute("SELECT ts, got, kept, status, query FROM runs WHERE source=? "
+                         "ORDER BY id DESC LIMIT 15", (name,)).fetchall()
+    return {"ok": True,
+            "source": {"name": src.name, "category": src.category, "kind": src.kind, "url": src.url,
+                       "adapter": src.adapter, "tags": list(src.tags), "limit": src.limit},
+            "state": fl.get(name, {}), "stored": stored,
+            "latest": [{"id": r[0], "title": _clean(r[1], 160), "url": r[2], "types": r[3],
+                        "published": r[4], "score": r[5]} for r in rows],
+            "runs": [{"when": time.strftime("%Y-%m-%d %H:%M", time.localtime(r[0] or 0)),
+                      "got": r[1], "kept": r[2], "status": r[3], "query": r[4]} for r in runs]}
+
+
+# ---------- экспорт ----------
+
+def export_json(**kw):
+    rows, total = store.browse2(**kw)
+    return {"ok": True, "total": total, "count": len(rows), "items": rows}

omni/serve.py

@@ -12,7 +12,7 @@ import urllib.parse
 from concurrent.futures import ThreadPoolExecutor
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
-from . import feeds, net, sources, source_state, store
+from . import feeds, net, panel_api, sources, source_state, store
 
 WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
 JOBS = {}
@@ -353,8 +353,14 @@ class Handler(BaseHTTPRequestHandler):
             sort = (q.get("sort") or ["score"])[0]
             priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
             typ = (q.get("type") or ["all"])[0]
+            src = (q.get("src") or [""])[0].strip() or None
+            days = _intv(q, "days", 0, 0, 3650)
+            since = None
+            if days:
+                import datetime as _dt
+                since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
             rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
-                                        sort=sort, priced=priced, typ=typ)
+                                        sort=sort, priced=priced, typ=typ, since=since, src=src)
             for _r in rows:
                 _r["title"] = _clean_txt(_r.get("title"), 180)
                 _r["text"] = _clean_txt(_r.get("text"), 260)
@@ -387,6 +393,71 @@ class Handler(BaseHTTPRequestHandler):
             self.wfile.write(data)
             return
 
+        if path.startswith("/api/item/"):
+            try:
+                iid = int(path.rsplit("/", 1)[-1])
+            except ValueError:
+                return self._json({"error": "bad id"}, 400)
+            return self._json(panel_api.item_detail(iid))
+
+        if path == "/api/timeline":
+            days = _intv(q, "days", 30, 1, 365)
+            cat = (q.get("category") or [None])[0]
+            return self._json(panel_api.timeline(days=days, cat=cat))
+
+        if path == "/api/suggest":
+            return self._json(panel_api.suggest((q.get("q") or [""])[0], _intv(q, "limit", 8, 1, 20)))
+
+        if path == "/api/health":
+            return self._json(panel_api.health())
+
+        if path == "/api/tg/status":
+            return self._json(panel_api.tg_status())
+
+        if path == "/api/tg/search":
+            term = (q.get("q") or [""])[0].strip()
+            if len(term) < 2:
+                return self._json({"ok": False, "error": "нужно минимум 2 символа", "channels": []})
+            return self._json(panel_api.tg_search(term, limit=_intv(q, "limit", 40, 1, 100),
+                                                  min_members=_intv(q, "min", 0, 0, 10_000_000)))
+
+        if path == "/api/source":
+            name = (q.get("name") or [""])[0].strip()
+            return self._json(panel_api.source_detail(name) if name else {"error": "name required"})
+
+        if path == "/api/watch":
+            return self._json(panel_api.watch_list(hits=_intv(q, "hits", 3, 1, 10)))
+
+        if path == "/api/digest":
+            days = _intv(q, "days", 1, 1, 30)
+            res = panel_api.digest(days=days, limit=_intv(q, "limit", 30, 5, 100))
+            if (q.get("download") or ["0"])[0] in ("1", "true"):
+                data = res["md"].encode("utf-8")
+                self.send_response(200)
+                self.send_header("Content-Type", "text/markdown; charset=utf-8")
+                self.send_header("Content-Disposition", 'attachment; filename="omni-digest.md"')
+                self.send_header("Content-Length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+                return
+            return self._json(res)
+
+        if path == "/api/export.json":
+            cat = (q.get("category") or [None])[0]
+            rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(),
+                                        names=_names_in_category(cat),
+                                        limit=_intv(q, "limit", 500, 1, 5000),
+                                        sort=(q.get("sort") or ["score"])[0],
+                                        typ=(q.get("type") or ["all"])[0])
+            data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Content-Disposition", 'attachment; filename="omni.json"')
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+
         if path.startswith("/api/job/"):
             jid = path.rsplit("/", 1)[-1]
             return self._json(JOBS.get(jid) or {"state": "unknown"})
@@ -414,6 +485,25 @@ class Handler(BaseHTTPRequestHandler):
             res = _d.discover(limit_hosts=hosts, workers=8, max_add=add, dry=dry)
             return self._json(res)
 
+        if u.path == "/api/tg/pull":
+            handles = [h for h in (body.get("handles") or []) if isinstance(h, str)][:20]
+            if not handles:
+                return self._json({"error": "handles required"}, 400)
+            jid = _jid("tg")
+            _job(jid, state="running")
+            threading.Thread(target=lambda: _job(jid, **panel_api.tg_pull(handles, int(body.get("limit") or 100))),
+                             daemon=True).start()
+            return self._json({"job": jid})
+
+        if u.path == "/api/watch/add":
+            return self._json(panel_api.watch_add(body.get("q"), body.get("type") or "all"))
+
+        if u.path == "/api/watch/remove":
+            rid = body.get("id")
+            if not rid:
+                return self._json({"error": "id required"}, 400)
+            return self._json(panel_api.watch_remove(str(rid)))
+
         if u.path == "/api/feed":
             jid = _jid("feed")
             _job(jid, state="running")

omni/store.py

@@ -182,13 +182,24 @@ def browse(category=None, q="", limit=60, offset=0):
     return [dict(zip(keys, r)) for r in rows]
 
 
-def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None):
-    """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
+def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None,
+            since=None, src=None):
+    """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total.
+
+    since — дата YYYY-MM-DD (включительно), src — точное имя источника.
+    """
     where = "WHERE 1=1 "
     args = []
     if names:
         where += "AND source IN (%s) " % ",".join("?" * len(names))
         args += list(names)
+    if src:
+        where += "AND source = ? "
+        args += [src]
+    if since:
+        where += ("AND (CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
+                  "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END) >= ? ")
+        args += [str(since)[:10]]
     if q:
         where += "AND (title LIKE ? OR text LIKE ?) "
         args += [f"%{q}%", f"%{q}%"]
@@ -202,10 +213,10 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty
              "old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
     with db() as c:
         total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
-        rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
+        rows = c.execute("SELECT id,provider,kind,source,title,text,author,published,score,price,currency,url,types "
                          "FROM items " + where + order + " LIMIT ? OFFSET ?",
                          args + [limit, offset]).fetchall()
-    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
+    keys = ("id", "provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
     return [dict(zip(keys, r)) for r in rows], total
 
 

omni/tgsess.py

@@ -132,3 +132,43 @@ def search(query, limit=40):
     api_id = int(os.environ["OMNI_TG_API_ID"])
     api_hash = os.environ["OMNI_TG_API_HASH"]
     return asyncio.run(_search(query, limit, session, api_id, api_hash))
+
+
+_ME = {}
+
+
+async def _me(session, api_id, api_hash, timeout=40):
+    from telethon import TelegramClient
+
+    client = TelegramClient(session, api_id, api_hash, connection_retries=1, timeout=timeout / 2)
+    await asyncio.wait_for(client.connect(), timeout)
+    try:
+        u = await asyncio.wait_for(client.get_me(), timeout)
+        if not u:
+            return {}
+        return {"id": getattr(u, "id", None), "username": getattr(u, "username", None),
+                "first_name": getattr(u, "first_name", None), "premium": bool(getattr(u, "premium", False)),
+                "phone_tail": (str(getattr(u, "phone", "") or "")[-4:] or None)}
+    finally:
+        try:
+            await asyncio.wait_for(client.disconnect(), 10)
+        except Exception:
+            pass
+
+
+def me():
+    """Аккаунт сессии: {id, username, first_name, premium}. Кэш на процесс."""
+    if _ME:
+        return _ME
+    if not enabled():
+        return {}
+    try:
+        from . import config
+        config.tg_cfg()
+    except Exception:
+        pass
+    session = os.environ["OMNI_TG_SESSION"]
+    api_id = int(os.environ["OMNI_TG_API_ID"])
+    api_hash = os.environ["OMNI_TG_API_HASH"]
+    _ME.update(asyncio.run(_me(session, api_id, api_hash)) or {})
+    return _ME

tools/fresh_candidates.py

@@ -0,0 +1,149 @@
+"""Кандидаты для расширения реестра omni: security/CVE/leak/CN/RU/tg.
+
+Источник истины для tools/probe_fresh.py. Живые строки после прогона
+переезжают в omni/sources_fresh.py (генерируется `--emit`).
+
+Формат dict совпадает с Source(**d). Адаптеры — только из feeds.ADAPTERS.
+"""
+import datetime
+
+
+def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
+    return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
+            "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
+
+
+# NVD recent: окно считается на импорте (API требует обе даты и <=120 дней)
+_NOW = datetime.datetime.utcnow()
+_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
+_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
+
+CANDIDATES = [
+    # ---------------- 1. security news / research (RSS) ----------------
+    d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),
+    d("gbhackers", "security", "rss", "https://gbhackers.com/feed/", "rss", ("news",)),
+    d("cybersecuritynews", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", ("news",)),
+    d("scmedia", "security", "rss", "https://www.scmagazine.com/rss.xml", "rss", ("news",)),
+    d("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ("news",)),
+    d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
+    d("seclists_bugtraq", "security", "rss", "https://seclists.org/rss/bugtraq.rss", "rss", ("mailing", "exploit")),
+    d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
+    d("huntress_blog", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", ("research", "dfir")),
+    d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("research", "cti")),
+    d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("research", "cti")),
+    d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
+    d("sophos_news", "security", "rss", "https://news.sophos.com/en-us/feed/", "rss", ("research",)),
+    d("trendmicro_si", "security", "rss", "https://www.trendmicro.com/rss/security-intelligence.xml", "rss", ("research",)),
+    d("volexity_blog", "security", "rss", "https://www.volexity.com/blog/feed/", "rss", ("research", "dfir")),
+    d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
+    d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
+    d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
+    d("intezer_blog", "security", "rss", "https://intezer.com/blog/feed/", "rss", ("malware",)),
+    d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
+    d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
+    d("zscaler_research", "security", "rss",
+      "https://www.zscaler.com/blogs/security-research/rss.xml", "rss", ("research",)),
+    d("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
+    d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/resources/blog/feed/", "rss", ("cti",)),
+    d("dragos_blog", "security", "rss", "https://www.dragos.com/feed/", "rss", ("ics",)),
+    d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("research", "apt")),
+    d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")),
+    d("jfrog_blog", "security", "rss", "https://jfrog.com/blog/feed/", "rss", ("supply-chain",)),
+    d("gitguardian_blog", "security", "rss", "https://blog.gitguardian.com/rss/", "rss", ("secrets",)),
+    d("truffle_security", "security", "rss", "https://trufflesecurity.com/blog/rss.xml", "rss", ("secrets",)),
+    d("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
+    d("hudsonrock_blog", "security", "rss", "https://www.hudsonrock.com/blog/rss.xml", "rss", ("stealer",)),
+    d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("blog", "writeup")),
+    d("0xdf", "security", "rss", "https://0xdf.gitlab.io/feed.xml", "rss", ("writeup", "htb")),
+    d("hackmag", "security", "rss", "https://hackmag.com/feed/", "rss", ("ru",)),
+    d("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("cloud",)),
+
+    # ---------------- 2. CVE / vulnerabilities ----------------
+    d("nvd_recent", "security", "json",
+      "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
+      % (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")),
+    d("nvd_exploited_kw", "security", "json",
+      "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
+      "nvd", ("cve", "exploited")),
+    d("github_advisories_api", "security", "json",
+      "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
+    d("cve_circl_last", "security", "json", "https://cve.circl.lu/api/last/40", "cve_circl", ("cve",)),
+    d("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
+    d("gentoo_glsa", "security", "rss", "https://security.gentoo.org/glsa/feed.xml", "rss", ("cve", "linux")),
+    d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/feed.atom", "rss", ("cve", "linux")),
+    d("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing", ("cve", "linux"),
+      detail=False, extra={"include_prefixes": ["/security/2026/"], "exclude_exact": ["/security/"]}),
+    d("oss_security_openwall", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
+      ("mailing", "cve"), extra={"include_prefixes": ["/lists/oss-security/2026/"]}),
+    d("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing",
+      ("cve", "browser"), extra={"include_prefixes": ["/en-US/security/advisories/mfsa"]}),
+    d("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default",
+      "rss", ("cve", "browser")),
+    d("cvefeed_latest", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ("cve",)),
+    d("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
+    d("cxsecurity", "security", "rss", "https://cxsecurity.com/rss/", "rss", ("exploit",)),
+    d("vulmon_recent", "security", "rss", "https://vulmon.com/rss/recent", "rss", ("cve",)),
+    d("cisa_ics_advisories", "security", "rss",
+      "https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml", "rss", ("ics", "cve")),
+    d("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
+    d("epss_recent", "security", "json", "https://api.first.org/data/v1/epss?limit=25&order=!epss", "generic_json",
+      ("cve", "epss")),
+
+    # ---------------- 3. leaks / ransomware / underground ----------------
+    d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet", "news")),
+    d("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
+    d("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
+    d("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
+    d("ransomware_live_recent", "abuse", "json", "https://api.ransomware.live/recentvictims", "generic_json",
+      ("ransomware",)),
+    d("ransomlook_recent", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", ("ransomware",)),
+    d("ransomware_live_groups", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", ("ransomware",)),
+    d("shadowserver_news", "abuse", "rss", "https://www.shadowserver.org/news/feed/", "rss", ("infra",)),
+
+    # ---------------- 4. китайские security ----------------
+    d("anquanke_rss", "security", "rss", "https://www.anquanke.com/rss", "rss", ("cn", "news")),
+    d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")),
+    d("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
+    d("seebug_paper", "security", "rss", "https://paper.seebug.org/rss/", "rss", ("cn", "research")),
+    d("xz_aliyun", "security", "html", "https://xz.aliyun.com/", "html_listing", ("cn", "research"),
+      extra={"include_prefixes": ["/t/"]}),
+    d("aqniu", "security", "rss", "https://www.aqniu.com/feed", "rss", ("cn", "news")),
+    d("knownsec_blog", "security", "rss", "https://blog.knownsec.com/feed/", "rss", ("cn", "research")),
+    d("butian_forum", "forum", "html", "https://forum.butian.net/", "html_listing", ("cn", "forum"),
+      extra={"include_prefixes": ["/share/"]}),
+    d("kanxue_news_html", "forum", "html", "https://bbs.kanxue.com/", "html_listing", ("cn", "forum"),
+      extra={"include_prefixes": ["/thread-"]}),
+    d("t00ls_html", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"),
+      extra={"include_prefixes": ["/article-"]}),
+
+    # ---------------- 5. русскоязычные ----------------
+    d("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
+    d("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
+    d("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
+    d("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
+    d("kaspersky_ru_blog", "security", "rss", "https://www.kaspersky.ru/blog/feed/", "rss", ("ru", "research")),
+    d("securitylab_rss", "security", "rss", "https://www.securitylab.ru/rss/all.xml", "rss", ("ru", "news")),
+    d("anti_malware_ru", "security", "rss", "https://www.anti-malware.ru/rss.xml", "rss", ("ru", "news")),
+    d("cnews_security", "security", "rss", "https://www.cnews.ru/inc/rss/news.xml", "rss", ("ru", "news")),
+    d("bdu_fstec", "security", "rss", "https://bdu.fstec.ru/rss/vulxml", "rss", ("ru", "cve")),
+    d("codeby_forum", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", ("ru", "forum")),
+
+    # ---------------- 6. telegram (t.me/s, публичное превью) ----------------
+    d("tg_thehackersnews", "security", "html", "https://t.me/s/TheHackersNews", "tg_web", ("tg", "news")),
+    d("tg_bleepingcomputer", "security", "html", "https://t.me/s/bleepingcomputer", "tg_web", ("tg", "news")),
+    d("tg_vxunderground", "security", "html", "https://t.me/s/vxunderground", "tg_web", ("tg", "malware")),
+    d("tg_secator", "security", "html", "https://t.me/s/SecAtor", "tg_web", ("tg", "ru")),
+    d("tg_itsecforu", "security", "html", "https://t.me/s/itsecforu", "tg_web", ("tg", "ru")),
+    d("tg_hitech_crime", "security", "html", "https://t.me/s/hitech_crime", "tg_web", ("tg", "ru")),
+    d("tg_codeby", "security", "html", "https://t.me/s/codeby", "tg_web", ("tg", "ru")),
+    d("tg_darkwebinformer", "abuse", "html", "https://t.me/s/DarkWebInformer", "tg_web", ("tg", "leak")),
+    d("tg_breachdetector", "abuse", "html", "https://t.me/s/breachdetector", "tg_web", ("tg", "leak")),
+    d("tg_databreaches", "abuse", "html", "https://t.me/s/databreaches", "tg_web", ("tg", "leak")),
+    d("tg_ransomware_live", "abuse", "html", "https://t.me/s/ransomware_live", "tg_web", ("tg", "ransomware")),
+    d("tg_cybersecuritynews", "security", "html", "https://t.me/s/cybersecuritynews", "tg_web", ("tg", "news")),
+    d("tg_ai_newz", "community", "html", "https://t.me/s/ai_newz", "tg_web", ("tg", "ai", "ru")),
+    d("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web",
+      ("tg", "ai", "ru")),
+    d("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
+    d("tg_bo0om", "abuse", "html", "https://t.me/s/bo0om", "tg_web", ("tg", "leak", "ru")),
+]

tools/probe_fresh.py

@@ -0,0 +1,146 @@
+"""Живой прогон кандидатов из tools/fresh_candidates.py через штатные адаптеры omni.
+
+    python tools/probe_fresh.py                 # прогон + data/fresh_probe.json
+    python tools/probe_fresh.py --emit          # + перегенерация omni/sources_fresh.py
+    python tools/probe_fresh.py --only tg_ --workers 4 --limit 5
+
+Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись.
+Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина
+в отчёте была фактической, а не «0 items».
+"""
+import argparse
+import json
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+
+from omni import feeds, net            # noqa: E402
+from omni.sources import Source        # noqa: E402
+from fresh_candidates import CANDIDATES   # noqa: E402
+
+
+def _diag(src):
+    """Почему пусто: код ответа, блокировка, размер тела."""
+    try:
+        r = net.get(src.url, timeout=20)
+        bits = ["http %s" % r.status]
+        if getattr(r, "blocked", False):
+            bits.append("blocked")
+        if getattr(r, "error", ""):
+            bits.append("err:%s" % str(r.error)[:120])
+        bits.append("%db" % len(r.text or ""))
+        return ", ".join(bits)
+    except Exception as e:                      # noqa: BLE001
+        return "exception:%s" % str(e)[:140]
+
+
+def probe_one(src, limit):
+    t0 = time.time()
+    if src.adapter not in feeds.ADAPTERS:
+        return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
+                "url": src.url, "tags": list(src.tags), "ok": False, "n": 0,
+                "err": "no_adapter:%s" % src.adapter, "ms": 0, "samples": []}
+    try:
+        items = feeds.fetch(src, limit) or []
+        err = ""
+    except Exception as e:                      # noqa: BLE001
+        items, err = [], "raise:%s" % str(e)[:140]
+    ms = int((time.time() - t0) * 1000)
+    if not items and not err:
+        err = _diag(src)
+    return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
+            "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items),
+            "err": err, "ms": ms,
+            "samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
+
+
+def emit(rows, dead, path):
+    """omni/sources_fresh.py: только живые, без импортов из omni.sources."""
+    def q(s):
+        return '"%s"' % str(s).replace('"', '\\"')
+
+    lines = [
+        '"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.',
+        "",
+        "Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).",
+        "Отклонённые кандидаты и причины — в хвостовом комментарии.",
+        "Импортов из omni.sources нет намеренно (циклический импорт).",
+        '"""',
+        "",
+        "",
+        "def fresh(S):",
+        '    """S — конструктор Source; возвращает только проверенные сетью источники."""',
+        "    return [",
+    ]
+    for r in rows:
+        extra = ""
+        if r.get("extra"):
+            extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True)
+        limit = r.get("limit") or 15
+        lim = "" if limit == 15 else ", limit=%d" % limit
+        det = ", detail=True" if r.get("detail") else ""
+        tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
+        lines.append('        S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
+            q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
+            tags, lim, det, extra))
+    lines += ["    ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"]
+    for r in dead:
+        lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90]))
+    text = "\n".join(lines) + "\n"
+    path.write_text(text, encoding="utf-8")
+    return len(rows)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=5)
+    ap.add_argument("--workers", type=int, default=6)
+    ap.add_argument("--only", default="")
+    ap.add_argument("--out", default="data/fresh_probe.json")
+    ap.add_argument("--emit", action="store_true")
+    ap.add_argument("--fresh", action="store_true", help="прогонять только то, что уже в sources_fresh.py")
+    a = ap.parse_args()
+
+    cands = [c for c in CANDIDATES if not a.only or a.only in c["name"]]
+    srcs = []
+    for c in cands:
+        try:
+            srcs.append(Source(**c))
+        except Exception as e:                  # noqa: BLE001
+            print("skip %s: %s" % (c.get("name"), e))
+
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=a.workers) as ex:
+        rows = list(ex.map(lambda s: probe_one(s, a.limit), srcs))
+
+    alive = [r for r in rows if r["ok"]]
+    dead = [r for r in rows if not r["ok"]]
+    cats = {}
+    for r in alive:
+        cats[r["category"]] = cats.get(r["category"], 0) + 1
+    out = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "limit": a.limit, "workers": a.workers,
+           "total": len(rows), "alive": len(alive), "dead": len(dead), "seconds": round(time.time() - t0, 1),
+           "by_category": dict(sorted(cats.items(), key=lambda kv: -kv[1])),
+           "results": sorted(rows, key=lambda r: (not r["ok"], r["category"], r["name"]))}
+    p = ROOT / a.out
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
+
+    print("всего %d | живых %d | мёртвых %d | %.1fs" % (len(rows), len(alive), len(dead), time.time() - t0))
+    print("по категориям:", out["by_category"])
+    for r in dead:
+        print("  dead %-24s %s" % (r["name"], r["err"][:110]))
+
+    if a.emit:
+        n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py")
+        print("omni/sources_fresh.py: %d источников" % n)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/ui_smoke.py

@@ -0,0 +1,96 @@
+"""Смоук панели в реальном Chromium: скриншоты + ошибки консоли.
+
+python tools/ui_smoke.py [--url http://127.0.0.1:8078/] [--shots data/shots]
+"""
+import argparse
+import asyncio
+import json
+import pathlib
+import sys
+
+from playwright.async_api import async_playwright
+
+
+async def run(url, out):
+    out.mkdir(parents=True, exist_ok=True)
+    errors, report = [], {}
+    async with async_playwright() as p:
+        b = await p.chromium.launch()
+        pg = await b.new_page(viewport={"width": 1500, "height": 950})
+        pg.on("console", lambda m: errors.append([m.type, m.text]) if m.type in ("error",) else None)
+        pg.on("pageerror", lambda e: errors.append(["pageerror", str(e)]))
+        await pg.goto(url, wait_until="domcontentloaded", timeout=30000)
+        await pg.wait_for_selector(".kpi", timeout=20000)
+        await pg.wait_for_timeout(1200)
+        report["title"] = await pg.title()
+        report["nav"] = await pg.locator(".nav-i").count()
+        report["kpi"] = (await pg.locator(".kpi").all_inner_texts())[:4]
+        await pg.screenshot(path=str(out / "v2_overview.png"), full_page=False)
+
+        await pg.click("[data-nav=feed]")
+        await pg.wait_for_selector("#list .item", timeout=20000)
+        report["feed_rows"] = await pg.locator(".item").count()
+        await pg.screenshot(path=str(out / "v2_feed.png"))
+        await pg.locator(".item").first.click()
+        await pg.wait_for_timeout(1200)
+        report["drawer"] = (await pg.locator("#d-title").inner_text())[:100]
+        await pg.screenshot(path=str(out / "v2_drawer.png"))
+        await pg.keyboard.press("Escape")
+
+        await pg.click("[data-nav=sources]")
+        await pg.wait_for_timeout(1500)
+        report["src_rows"] = await pg.locator("#src-tbl tbody tr").count()
+        await pg.screenshot(path=str(out / "v2_sources.png"))
+
+        await pg.click("[data-nav=abuse]")
+        await pg.wait_for_timeout(1500)
+        report["abuse_rows"] = await pg.locator(".item").count()
+        await pg.screenshot(path=str(out / "v2_abuse.png"))
+
+        await pg.click("[data-nav=runs]")
+        await pg.wait_for_timeout(900)
+        report["run_rows"] = await pg.locator("#r-tbl tbody tr").count()
+        await pg.screenshot(path=str(out / "v2_runs.png"))
+
+        await pg.click("[data-nav=tg]")
+        await pg.wait_for_timeout(2000)
+        report["tg"] = (await pg.locator(".card").first.inner_text())[:160].replace("\n", " | ")
+        await pg.screenshot(path=str(out / "v2_tg.png"))
+
+        await pg.click("[data-nav=keys]")
+        await pg.wait_for_timeout(800)
+        report["key_rows"] = await pg.locator("#k-tbl tbody tr").count()
+        await pg.screenshot(path=str(out / "v2_keys.png"))
+
+        await pg.click("[data-nav=digest]")
+        await pg.wait_for_timeout(500)
+        await pg.click("[data-act=digest]")
+        await pg.wait_for_timeout(6000)
+        report["digest_chars"] = len(await pg.locator("#d-out").inner_text())
+        await pg.screenshot(path=str(out / "v2_digest.png"))
+
+        await pg.click("[data-nav=watch]")
+        await pg.wait_for_timeout(600)
+        await pg.screenshot(path=str(out / "v2_watch.png"))
+
+        # мобильный вид
+        await pg.set_viewport_size({"width": 420, "height": 900})
+        await pg.click("[data-nav=feed]")
+        await pg.wait_for_timeout(1200)
+        await pg.screenshot(path=str(out / "v2_mobile.png"))
+        await b.close()
+    report["console"] = errors[:12]
+    print(json.dumps(report, ensure_ascii=False, indent=1))
+    return 0 if not errors else 1
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--url", default="http://127.0.0.1:8078/")
+    ap.add_argument("--shots", default="data/shots")
+    a = ap.parse_args()
+    sys.exit(asyncio.run(run(a.url, pathlib.Path(a.shots))))
+
+
+if __name__ == "__main__":
+    main()