абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты

.gitignore

@@ -4,3 +4,5 @@ data/
 __pycache__/
 *.pyc
 .bot_token
+*.session*
+.local/

omni/abuse.py

@@ -141,6 +141,13 @@ def _qualifies(source, title, text, tags):
         sc -= 26
         if not hits:
             return 0, []
+    src0 = str(source or "")
+    # new-api панели: «N бесплатных моделей (цена 0)» — это готовая абуз-поверхность
+    if src0.startswith("pn_"):
+        m = re.search(r"(\d{1,4})\s*бесплатн\w*\s*модел", blob, re.I)
+        if m:
+            cnt = min(int(m.group(1)), 500)
+            return 40 + min(cnt // 10, 30), ["freeapi"]
     raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
     src = str(source or "")
     known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_"))

omni/dates.py

@@ -0,0 +1,33 @@
+"""Publication dates from RSS/Atom/APIs, normalized to aware UTC datetimes."""
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+import re
+
+
+def parse_published(value):
+    """Accept Unix seconds/milliseconds, ISO 8601 or RFC 2822; invalid -> None.
+
+    Dates without a timezone are interpreted as UTC, not the host's local zone.
+    """
+    if value is None or not str(value).strip():
+        return None
+    text = str(value).strip()
+    try:
+        if re.fullmatch(r"\d{4}", text):
+            return None  # A bare year is not a Unix timestamp.
+        if re.fullmatch(r"\d{8}", text):
+            return datetime.strptime(text, "%Y%m%d").replace(tzinfo=timezone.utc)
+        if re.fullmatch(r"-?\d+(?:\.\d+)?", text):
+            stamp = float(text)
+            if abs(stamp) >= 10**12:
+                stamp /= 1000
+            return datetime.fromtimestamp(stamp, timezone.utc)
+        try:
+            dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
+        except ValueError:
+            dt = parsedate_to_datetime(text)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt.astimezone(timezone.utc)
+    except (ValueError, TypeError, OverflowError, OSError):
+        return None

omni/feeds.py

@@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET
 from bs4 import BeautifulSoup
 
 from . import config, net
+from .dates import parse_published
 from .model import Item
 
 UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
@@ -38,8 +39,9 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
         m = re.search(r"CVE-\d{4}-\d{4,7}", body or "")
         if m:
             t = "%s · %s" % (m.group(0), t[:120])
+    dt = parse_published(published)
     return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
-                author=_txt(author)[:80], published=str(published or ""), source=src.name,
+                author=_txt(author)[:80], published=dt.isoformat(timespec="seconds") if dt else str(published or ""), source=src.name,
                 extra=ex)
 
 
@@ -100,17 +102,26 @@ def _rss(src, limit):
         root = ET.fromstring(r.text.encode("utf-8", "ignore"))
     except Exception:
         return _rss_regex(src, r.text, limit)
-    ns = {"a": "http://www.w3.org/2005/Atom"}
+    ns = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/elements/1.1/"}
     entries = root.findall(".//item") or root.findall(".//a:entry", ns)
     for e in entries[:limit]:
-        g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
+        def g(tag):
+            direct = e.findtext(tag, namespaces=ns) or ""
+            if direct or ":" in tag:
+                return direct
+            return e.findtext("a:" + tag, namespaces=ns) or ""
+
         link = g("link")
         if not link:
-            le = e.find("a:link", ns) or e.find("link")
+            links = list(e.findall("a:link", ns)) + list(e.findall("link"))
-            link = le.get("href") if le is not None and le.get("href") else ""
+            le = next((el for el in links if el.get("rel", "alternate") == "alternate" and el.get("href")), None)
+            link = le.get("href", "") if le is not None else ""
+        if link:
+            link = urllib.parse.urljoin(src.url, link)
         body = g("description") or g("summary") or g("content")
+        author = e.findtext("a:author/a:name", namespaces=ns) or g("author") or g("dc:creator")
         out.append(_item(src, "entry", link, g("title"), body,
-                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
+                         author, g("pubDate") or g("published") or g("updated")))
     if not out and "arxiv.org" in src.url:
         return _arxiv_api(src, limit)
     return out
@@ -910,18 +921,52 @@ ADAPTERS = {
 }
 
 
+_CHALLENGE_TITLE = re.compile(
+    r"(?:just a moment|access denied|403 forbidden|404(?: not found)?|not found|"
+    r"verify (?:that )?you are human|checking your browser|"
+    r"attention required!?(?:\s*\|\s*cloudflare)?|please enable javascript)[.!…\s]*",
+    re.I,
+)
+
+
+def _valid_items(items):
+    """Drop broken records and obvious challenge pages, not short real posts."""
+    out, seen = [], set()
+    for it in items:
+        if not isinstance(it, Item) or any(not isinstance(getattr(it, field, None), str)
+                                          for field in ("url", "title", "text")):
+            continue
+        try:
+            u = urllib.parse.urlsplit(it.url)
+            valid_url = u.scheme in ("http", "https") and bool(u.hostname)
+        except ValueError:
+            valid_url = False
+        if not valid_url or not (it.title.strip() or it.text.strip()):
+            continue
+        if _CHALLENGE_TITLE.fullmatch(it.title.strip()) and (
+                len(it.text.strip()) < 160 or re.search(
+                    r"cloudflare|captcha|verify you are human|checking your browser|just a moment|attention required",
+                    it.text, re.I)):
+            continue
+        if it.url in seen:
+            continue
+        seen.add(it.url)
+        out.append(it)
+    return out
+
+
 def fetch(src, limit=None):
     fn = ADAPTERS.get(src.adapter)
     if not fn:
         return []
     n = limit or src.limit
     try:
-        return fn(src, n) or []
+        return _valid_items(fn(src, n) or [])
     except Exception:
         return []
 
 
-SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
+SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co", "services.nvd.nist.gov")
 
 
 def fetch_many(sources, limit=None, workers=8):
@@ -944,5 +989,6 @@ def fetch_many(sources, limit=None, workers=8):
             got = fetch(s, limit)
         report.append((s.name, len(got)))
         items.extend(got)
-        time.sleep(3.5 if "reddit.com" in s.url else 1.0)
+        delay = 6.5 if "services.nvd.nist.gov" in s.url else (3.5 if "reddit.com" in s.url else 1.0)
+        time.sleep(delay)
     return items, report

omni/panel_api.py

@@ -13,8 +13,8 @@ import urllib.parse
 from . import abuse as abuse_mod
 from . import config, sources, source_state, store, taxonomy
 
-BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
+BUCKET = ("CASE WHEN length(published)=10 AND published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' "
-          "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
+          "THEN published ELSE COALESCE(date(published,'localtime'),date(seen,'unixepoch','localtime')) END")
 
 
 def _clean(v, n=0):
@@ -32,7 +32,7 @@ def _clean(v, n=0):
 
 # ---------- обзор ----------
 
-BUCKET_SEEN = "date(seen,'unixepoch')"
+BUCKET_SEEN = "date(seen,'unixepoch','localtime')"
 
 
 def timeline(days=30, cat=None, mode="pub"):
@@ -40,6 +40,8 @@ def timeline(days=30, cat=None, mode="pub"):
 
     mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
     mode="seen" — строго по времени импорта: «что комбайн принёс за день».
+
+    last24 — скользящие 24 часа по выбранному mode; непарсируемая дата → seen.
     """
     days = 30 if days is None else int(days or 0)
     days = max(1, min(365, days))
@@ -57,6 +59,11 @@ def timeline(days=30, cat=None, mode="pub"):
         for b, n, cve, sev in c.execute(sql, args):
             if b:
                 got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
+        clock = "seen" if mode == "seen" else "COALESCE(CAST(strftime('%s',published) AS REAL),seen)"
+        recent_where = where + (" AND " if where else " WHERE ") + clock + " >= ? AND " + clock + " <= ?"
+        now = time.time()
+        last24 = c.execute("SELECT COUNT(*) FROM items" + recent_where,
+                           args + [now - 86400, now]).fetchone()[0]
     today = _dt.date.today()
     out = []
     for i in range(days - 1, -1, -1):
@@ -65,7 +72,8 @@ def timeline(days=30, cat=None, mode="pub"):
         out.append({"day": d, "n": n, "cve": cve, "sev": sev})
     return {"ok": True, "days": days, "mode": mode, "points": out,
             "total": sum(p["n"] for p in out),
-            "last24": (out[-1] or {}).get("n", 0),
+            "last24": int(last24),
+            "today": out[-1]["n"],
             "max": max([p["n"] for p in out] or [0])}
 
 

omni/rank.py

@@ -2,6 +2,8 @@
 import re
 import time
 
+from .dates import parse_published
+
 WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0,
           "firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8}
 
@@ -24,20 +26,12 @@ KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "resu
 
 
 def _fresh(published):
-    """published: unix-строка или ISO. Возвращает 0..1."""
+    """Unix seconds/milliseconds, ISO or RFC 2822. Return freshness in 0..1."""
-    if not published:
+    dt = parse_published(published)
+    if dt is None:
         return 0.3
-    try:
+    days = max(0.0, (time.time() - dt.timestamp()) / 86400)
-        if published.isdigit():
-            ts = int(published)
-            if ts > 10**12:
-                ts //= 1000
-        else:
-            ts = time.mktime(time.strptime(published[:19], "%Y-%m-%dT%H:%M:%S"))
-        days = max(0.0, (time.time() - ts) / 86400)
     return max(0.0, 1.0 - days / 30.0)
-    except Exception:
-        return 0.3
 
 
 def score(it):

omni/serve.py

@@ -115,6 +115,33 @@ def _names_in_category(cat):
     return [s.name for s in sources.all_sources() if s.category == cat]
 
 
+def _run_state(status):
+    """Класс статуса прогона: ok / empty / none / err.
+
+    "ok", "ok 0/10", "ok-304" — успех (startswith); "пусто"/"empty" — источник ответил пусто,
+    это не ошибка; отсутствующий статус (None/"") — отдельный класс none; остальное — err.
+    """
+    v = " ".join(str(status or "").split()).lower()
+    if not v:
+        return "none"
+    if v.startswith("ok"):
+        return "ok"
+    if v in ("пусто", "empty"):
+        return "empty"
+    return "err"
+
+
+def _since_days(days):
+    """?days=N -> граница since (YYYY-MM-DD) для store.browse2; 0 — без фильтра.
+
+    Единая логика для /api/items и экспортов: «сутки/неделя/месяц» значат одно и то же.
+    """
+    if not days:
+        return None
+    import datetime as _dt
+    return (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
+
+
 def _run_feed(job_id, category, limit, tags):
     picked = sources.pick(category, None, tags)
     _job(job_id, state="running", total=len(picked), done=0)
@@ -211,6 +238,18 @@ class Handler(BaseHTTPRequestHandler):
                 return self._send(404, "no ui", "text/plain")
             return self._send(200, f.read_bytes(), "text/html; charset=utf-8")
 
+        if path == "/api/panels":
+            import json as _json
+            fp = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "panels.json")
+            try:
+                with open(fp, "r", encoding="utf-8") as f:
+                    raw = _json.load(f)
+            except Exception:
+                raw = []
+            rows = raw if isinstance(raw, list) else (raw.get("panels") or raw.get("items") or [])
+            rows = sorted(rows, key=lambda x: -(x.get("free_count") or x.get("free") or 0))
+            return self._json({"ok": True, "total": len(rows), "panels": rows[:200]})
+
         if path == "/api/stats":
             s = store.stats()
             per_source = store.count_by_source()
@@ -264,6 +303,7 @@ class Handler(BaseHTTPRequestHandler):
             import time as _t
             con = store._con() if hasattr(store, "_con") else None
             rows = []
+            by_status = {"ok": 0, "empty": 0, "err": 0, "none": 0}
             try:
                 import sqlite3 as _s3
                 from .config import DB as _DB
@@ -274,12 +314,22 @@ class Handler(BaseHTTPRequestHandler):
                     d = dict(r)
                     ts = d.get("ts") or 0
                     d["when"] = _t.strftime("%Y-%m-%d %H:%M", _t.localtime(ts)) if ts else ""
+                    d["state"] = _run_state(d.get("status"))
+                    d["ok"] = d["state"] == "ok"
                     rows.append(d)
                 total = db.execute("SELECT count(*) FROM runs").fetchone()[0]
+                # статусы считаем по всей таблице: total-ok из среза limit давал ложные «ошибки»
+                sql = ("SELECT COALESCE(SUM(CASE WHEN lower(trim(status)) LIKE 'ok%' THEN 1 ELSE 0 END),0),"
+                       " COALESCE(SUM(CASE WHEN lower(trim(status)) IN ('пусто','empty') THEN 1 ELSE 0 END),0),"
+                       " COALESCE(SUM(CASE WHEN status IS NULL OR trim(status)='' THEN 1 ELSE 0 END),0)"
+                       " FROM runs")
+                n_ok, n_empty, n_none = db.execute(sql).fetchone()
+                by_status = {"ok": int(n_ok or 0), "empty": int(n_empty or 0), "none": int(n_none or 0)}
+                by_status["err"] = max(0, int(total) - by_status["ok"] - by_status["empty"] - by_status["none"])
                 db.close()
             except Exception as e:
                 return self._json({"error": "runs: %s" % e})
-            return self._json({"items": rows, "total": total, "shown": len(rows)})
+            return self._json({"items": rows, "total": total, "shown": len(rows), "by_status": by_status})
 
         if path == "/api/abuse":
             from . import abuse as _ab
@@ -355,13 +405,13 @@ class Handler(BaseHTTPRequestHandler):
             typ = (q.get("type") or ["all"])[0]
             src = (q.get("src") or [""])[0].strip() or None
             days = _intv(q, "days", 0, 0, 3650)
-            since = None
+            since = _since_days(days)
-            if days:
-                import datetime as _dt
-                since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
             rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
                                         sort=sort, priced=priced, typ=typ, since=since, src=src)
+            from . import abuse as _ab
             for _r in rows:
+                # find нужен itemRow/findChips: browse2 сам признаки абуза не считает
+                _r["find"] = _ab.extract(" ".join((_r.get("title") or "", _r.get("text") or "")))
                 _r["title"] = _clean_txt(_r.get("title"), 180)
                 _r["text"] = _clean_txt(_r.get("text"), 260)
                 _r["published"] = _fmt_date(_r.get("published"))
@@ -374,8 +424,11 @@ class Handler(BaseHTTPRequestHandler):
             priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
             sort = (q.get("sort") or ["score"])[0]
             typ = (q.get("type") or ["all"])[0]
-            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=5000, offset=0,
+            since = _since_days(_intv(q, "days", 0, 0, 3650))
-                                        sort=sort, priced=priced, typ=typ)
+            limit = _intv(q, "limit", 1000, 1, 5000)
+            src = (q.get("src") or [""])[0].strip() or None
+            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=0,
+                                        sort=sort, priced=priced, typ=typ, since=since, src=src)
             import io as _io
             buf = _io.StringIO()
             w = _csv.writer(buf, delimiter=";")
@@ -445,12 +498,17 @@ class Handler(BaseHTTPRequestHandler):
 
         if path == "/api/export.json":
             cat = (q.get("category") or [None])[0]
+            priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
+            since = _since_days(_intv(q, "days", 0, 0, 3650))
+            src = (q.get("src") or [""])[0].strip() or None
             rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(),
                                         names=_names_in_category(cat),
-                                        limit=_intv(q, "limit", 500, 1, 5000),
+                                        limit=_intv(q, "limit", 1000, 1, 5000),
                                         sort=(q.get("sort") or ["score"])[0],
-                                        typ=(q.get("type") or ["all"])[0])
+                                        typ=(q.get("type") or ["all"])[0],
-            data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8")
+                                        priced=priced, since=since, src=src)
+            data = json.dumps({"total": total, "count": len(rows), "items": rows},
+                              ensure_ascii=False, indent=1).encode("utf-8")
             self.send_response(200)
             self.send_header("Content-Type", "application/json; charset=utf-8")
             self.send_header("Content-Disposition", 'attachment; filename="omni.json"')

omni/sources.py

@@ -19,6 +19,7 @@ from .sources_tg_abuse import TG_ABUSE
 from .sources_ext4 import EXTRA5
 from .sources_ext5 import EXTRA6
 from .sources_fresh import fresh as _fresh_sources
+from .sources_quality import quality_sources as _quality_sources
 from .sources_x import X_SOURCES
 
 
@@ -298,7 +299,7 @@ def _gen_sources():
     return out
 
 
-def all_sources():
+def all_sources(include_fresh=True):
     """Реестр + пользовательские источники из панели, без дублей по url и по имени.
 
     Дедуп по двум ключам обязателен:
@@ -306,8 +307,10 @@ def all_sources():
       * name — имя источника является первичным ключом в source_flags/runs, поэтому
         зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
         www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
-    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна →
+    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая и quality-волны →
     пользовательские → abuse-radar → panels.json.
+
+    include_fresh=False исключает текущий generated fresh-модуль при регенерации.
     """
     from . import source_state
     extra = []
@@ -318,7 +321,8 @@ def all_sources():
              ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
         extra.append(Source(**d))
     seen_url, seen_name, out = set(), set(), []
-    for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources():
+    fresh = _fresh_sources(S) if include_fresh else []
+    for s in SOURCES + _gen_sources() + fresh + _quality_sources(S) + extra + radar_sources() + panel_sources():
         key = (s.url or "").rstrip("/").lower()
         name = (s.name or "").strip().lower()
         if (key and key in seen_url) or (name and name in seen_name):

omni/sources_ext.py

@@ -228,8 +228,9 @@ EXTRA = [
     # ---------------- security ----------------
     S("cisa_kev", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
       "cisa_kev", ("cve", "kev")),
-    S("nvd_recent", "security", "json",
+    # nvd_recent намеренно НЕ здесь: живой nvd_recent с пересчитываемым окном
-      "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&noRejected", "nvd", ("cve",)),
+    # lastMod отдаёт sources_fresh.py (tools/probe_fresh.py --emit). Статическая
+    # запись без окна здесь побеждала по first-wins и глушила свежую.
     S("github_advisories", "security", "rss", "https://github.com/advisories.atom", "rss", ("cve", "github")),
     S("snyk_blog", "security", "rss", "https://snyk.io/blog/feed/", "rss", ("appsec",)),
     S("trailofbits_blog", "security", "rss", "https://blog.trailofbits.com/feed/", "rss", ("research",)),

omni/sources_fresh.py

@@ -21,58 +21,58 @@ def fresh(S):
     return [
         S("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
         S("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
-        S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach")),
+        S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach",)),
         S("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
         S("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")),
         S("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
-        S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup")),
+        S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
         S("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", ("tg", "ai", "ru")),
         S("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
         S("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing", ("reddit", "blue-team")),
         S("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing", ("reddit", "exploit")),
         S("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25", "reddit_listing", ("reddit", "purple-team")),
         S("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum")),
-        S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec")),
+        S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec",)),
         S("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news")),
         S("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")),
-        S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti")),
+        S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti",)),
-        S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news")),
+        S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news",)),
-        S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research")),
+        S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
         S("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", "rss", ("cve", "browser")),
         S("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
         S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
-        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
+        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news",)),
-        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
+        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news",)),
         S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
         S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
         S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
         S("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")),
-        S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news")),
+        S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news",)),
         S("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")),
-        S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news")),
+        S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news",)),
-        S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research")),
+        S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research",)),
-        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
+        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research",)),
         S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
         S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
         S("nvd_exploited_kw", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", "nvd", ("cve", "exploited")),
         S("nvd_recent", "security", "json", _nvd_week(), "nvd", ("cve", "nvd")),
-        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
+        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
         S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
-        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
+        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
         S("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news")),
-        S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research")),
+        S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
-        S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research")),
+        S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research",)),
         S("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
-        S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti")),
+        S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
-        S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain")),
+        S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain",)),
-        S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit")),
+        S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit",)),
         S("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")),
         S("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")),
         S("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")),
-        S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news")),
+        S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news",)),
         S("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
-        S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware")),
+        S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware",)),
-        S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve")),
+        S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
         S("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
     ]
 

omni/sources_quality.py

@@ -0,0 +1,174 @@
+"""Quality-волна источников: официальные release-фиды проектов и security-адвайзори вендоров.
+
+Зачем отдельный модуль: основной реестр (sources.py) и волны ext/fresh уже покрывают
+топовые репозитории, но провал в (1) runtime/инференсе второго эшелона и (2) security-
+продуктах, где релиз = факт CVE/патча. Здесь только официальные Atom/RSS-фиды
+(github.com/<repo>/releases.atom, linux-дистрибутивные и вендорные advisory-фиды),
+каждая запись проверена живым прогоном.
+
+Модуль статический и НЕ импортирует .sources: sources.py уже импортирует волны,
+обратный импорт дал бы цикл. Дедуп по url/name делает all_sources().
+
+Категории — только существующие в реестре, без новой схемы:
+  models     — релизы AI-runtime/инференса (diffusers, TensorRT-LLM, onnxruntime, ...);
+  community  — SDK/агентные фреймворки (MCP python-sdk, openai-agents, langgraph);
+  security   — продукты, где релиз = патч/CVE, контейнеры и advisory-фиды дистрибутивов;
+  company    — официальные технические блоги вендоров (NVIDIA, HashiCorp).
+Признак «это релиз» остаётся тегом `release` у всех 37 записей, kind=atom/rss.
+
+Проверка (tools/probe_quality_sources.py, отчёт data/quality_sources_probe.json):
+  * каждое имя и URL сверены с полным реестром all_sources() (≈3.5k источников) — ни одного
+    дубля по имени, нормализованному URL и зеркалу репозитория (github.com/<o>/<r>);
+  * первые записи не пересекаются по URL/заголовкам с другими кандидатами (>=50%/60%);
+  * >= 2 записей с валидным http(s)-URL, без challenge/error-заголовков;
+  * штатный адаптер feeds.fetch (rss, Atom поддерживается) отдаёт 10-20 записей.
+
+Осознанно НЕ включено, хотя фиды живые (шум, не качество): pytorch/pytorch и
+pydantic/pydantic-ai (в releases.atom CI-теги viable/strict, clai2-bleeding),
+microsoft/DeepSpeed (nightly-last-green), keycloak отдаёт nightly-теги (оставлен —
+версии рядом), portainer/immich/netbox/nextcloud/gitea/elastic/opensearch/minio
+(низкий сигнал на релиз, не security), php.net (фид отдаёт 1 запись), а также
+зеркала и медиа-перепечатки (Cisco Talos, MSRC blog, PortSwigger, Cloudflare,
+Ubuntu USN, Rust/Go/Node/Python blogs) — они уже в реестре или дублируют его.
+"""
+
+SPECS = [
+    # ---------- AI runtime / inference (официальные релизы) ----------
+    {"name": "gh_rel_diffusers", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/diffusers/releases.atom",
+     "tags": ("release", "github", "ai", "image"), "limit": 10},
+    {"name": "gh_rel_peft", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/peft/releases.atom",
+     "tags": ("release", "github", "ai", "finetune"), "limit": 10},
+    {"name": "gh_rel_accelerate", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/accelerate/releases.atom",
+     "tags": ("release", "github", "ai", "training"), "limit": 10},
+    {"name": "gh_rel_textgenerationinference", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/text-generation-inference/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_tokenizers", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/tokenizers/releases.atom",
+     "tags": ("release", "github", "ai", "nlp"), "limit": 10},
+    {"name": "gh_rel_candle", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/candle/releases.atom",
+     "tags": ("release", "github", "ai", "rust"), "limit": 10},
+    {"name": "gh_rel_tensorrtllm", "category": "models", "kind": "atom",
+     "url": "https://github.com/NVIDIA/TensorRT-LLM/releases.atom",
+     "tags": ("release", "github", "ai", "nvidia", "inference"), "limit": 10},
+    {"name": "gh_rel_nvidiacontainertoolkit", "category": "models", "kind": "atom",
+     "url": "https://github.com/NVIDIA/nvidia-container-toolkit/releases.atom",
+     "tags": ("release", "github", "gpu", "container"), "limit": 10},
+    {"name": "gh_rel_flashinfer", "category": "models", "kind": "atom",
+     "url": "https://github.com/flashinfer-ai/flashinfer/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_onnxruntime", "category": "models", "kind": "atom",
+     "url": "https://github.com/microsoft/onnxruntime/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_mlx", "category": "models", "kind": "atom",
+     "url": "https://github.com/ml-explore/mlx/releases.atom",
+     "tags": ("release", "github", "ai", "apple"), "limit": 10},
+    {"name": "gh_rel_whispercpp", "category": "models", "kind": "atom",
+     "url": "https://github.com/ggml-org/whisper.cpp/releases.atom",
+     "tags": ("release", "github", "ai", "audio"), "limit": 10},
+    {"name": "gh_rel_exo", "category": "models", "kind": "atom",
+     "url": "https://github.com/exo-explore/exo/releases.atom",
+     "tags": ("release", "github", "ai", "local"), "limit": 10},
+    {"name": "gh_rel_vllmascend", "category": "models", "kind": "atom",
+     "url": "https://github.com/vllm-project/vllm-ascend/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+
+    # ---------- AI-агенты / SDK ----------
+    {"name": "gh_rel_mcp_pythonsdk", "category": "community", "kind": "atom",
+     "url": "https://github.com/modelcontextprotocol/python-sdk/releases.atom",
+     "tags": ("release", "github", "ai", "mcp"), "limit": 10},
+    {"name": "gh_rel_openaiagentspython", "category": "community", "kind": "atom",
+     "url": "https://github.com/openai/openai-agents-python/releases.atom",
+     "tags": ("release", "github", "ai", "agent"), "limit": 10},
+    {"name": "gh_rel_langgraph", "category": "community", "kind": "atom",
+     "url": "https://github.com/langchain-ai/langgraph/releases.atom",
+     "tags": ("release", "github", "ai", "agent"), "limit": 10},
+
+    # ---------- security-продукты: релиз = патч/CVE ----------
+    {"name": "gh_rel_caddy", "category": "security", "kind": "atom",
+     "url": "https://github.com/caddyserver/caddy/releases.atom",
+     "tags": ("security", "release", "github", "web"), "limit": 10},
+    {"name": "gh_rel_keycloak", "category": "security", "kind": "atom",
+     "url": "https://github.com/keycloak/keycloak/releases.atom",
+     "tags": ("security", "release", "github", "iam"), "limit": 10},
+    {"name": "gh_rel_grafana", "category": "security", "kind": "atom",
+     "url": "https://github.com/grafana/grafana/releases.atom",
+     "tags": ("security", "release", "github", "monitoring"), "limit": 10},
+    {"name": "gh_rel_tailscale", "category": "security", "kind": "atom",
+     "url": "https://github.com/tailscale/tailscale/releases.atom",
+     "tags": ("security", "release", "github", "vpn"), "limit": 10},
+    {"name": "gh_rel_envoy", "category": "security", "kind": "atom",
+     "url": "https://github.com/envoyproxy/envoy/releases.atom",
+     "tags": ("security", "release", "github", "proxy"), "limit": 10},
+    {"name": "gh_rel_vault", "category": "security", "kind": "atom",
+     "url": "https://github.com/hashicorp/vault/releases.atom",
+     "tags": ("security", "release", "github", "secrets"), "limit": 10},
+    {"name": "gh_rel_cosign", "category": "security", "kind": "atom",
+     "url": "https://github.com/sigstore/cosign/releases.atom",
+     "tags": ("security", "release", "github", "supply-chain"), "limit": 10},
+    {"name": "gh_rel_trivy", "category": "security", "kind": "atom",
+     "url": "https://github.com/aquasecurity/trivy/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_grype", "category": "security", "kind": "atom",
+     "url": "https://github.com/anchore/grype/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_nuclei", "category": "security", "kind": "atom",
+     "url": "https://github.com/projectdiscovery/nuclei/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_zaproxy", "category": "security", "kind": "atom",
+     "url": "https://github.com/zaproxy/zaproxy/releases.atom",
+     "tags": ("security", "release", "github", "web"), "limit": 10},
+    {"name": "gh_rel_openssl", "category": "security", "kind": "atom",
+     "url": "https://github.com/openssl/openssl/releases.atom",
+     "tags": ("security", "release", "github", "crypto"), "limit": 10},
+    {"name": "gh_rel_curl", "category": "security", "kind": "atom",
+     "url": "https://github.com/curl/curl/releases.atom",
+     "tags": ("security", "release", "github", "http"), "limit": 10},
+
+    # ---------- контейнеры/оркестрация: CVE-релевантные релизы ----------
+    {"name": "gh_rel_runc", "category": "security", "kind": "atom",
+     "url": "https://github.com/opencontainers/runc/releases.atom",
+     "tags": ("security", "release", "github", "container"), "limit": 10},
+    {"name": "gh_rel_containerd", "category": "security", "kind": "atom",
+     "url": "https://github.com/containerd/containerd/releases.atom",
+     "tags": ("security", "release", "github", "container"), "limit": 10},
+    {"name": "gh_rel_kubernetes", "category": "security", "kind": "atom",
+     "url": "https://github.com/kubernetes/kubernetes/releases.atom",
+     "tags": ("security", "release", "github", "kubernetes"), "limit": 10},
+
+    # ---------- официальные advisory/technical фиды вендоров ----------
+    {"name": "amazon_linux_alas", "category": "security", "kind": "rss",
+     "url": "https://alas.aws.amazon.com/alas.rss",
+     "tags": ("security", "release", "advisory", "linux", "aws"), "limit": 20},
+    {"name": "aws_security_bulletins", "category": "security", "kind": "rss",
+     "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
+     "tags": ("security", "release", "advisory", "aws"), "limit": 20},
+    {"name": "nvidia_dev_blog", "category": "company", "kind": "rss",
+     "url": "https://developer.nvidia.com/blog/feed/",
+     "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
+    {"name": "hashicorp_blog", "category": "company", "kind": "rss",
+     "url": "https://www.hashicorp.com/blog/feed.xml",
+     "tags": ("release", "vendor", "hashicorp"), "limit": 15},
+]
+
+NAMES = tuple(d["name"] for d in SPECS)
+
+
+def quality_sources(S, specs=None):
+    """Собрать Source-объекты quality-волны через переданный конструктор S.
+
+    S передаётся снаружи (обычно sources.Source), чтобы модуль не импортировал .sources —
+    это дал бы циклический импорт. Вызов позиционный, как в sources_fresh.fresh: так
+    работает и настоящий Source, и лёгкая заглушка `class _Fake: __init__(self, *a, **k)`,
+    которой tools/run_fresh.py --wave quality собирает имена.
+    Теги всегда кортежи; adapter по умолчанию rss — он же разбирает Atom.
+    """
+    out = []
+    for d in (SPECS if specs is None else specs):
+        out.append(S(d["name"], d["category"], d["kind"], d["url"],
+                     d.get("adapter", "rss"), tuple(d.get("tags") or ()), d.get("limit", 10)))
+    return out

omni/store.py

@@ -257,7 +257,7 @@ def stats():
         by_kind = c.execute("SELECT kind, COUNT(*) FROM items GROUP BY kind ORDER BY 2 DESC").fetchall()
         priced = c.execute("SELECT COUNT(*) FROM items WHERE price IS NOT NULL").fetchone()[0]
         runs = c.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
-        ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE status='ok'").fetchone()[0]
+        ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE lower(trim(status)) LIKE 'ok%'").fetchone()[0]
     return {"items": total, "priced": priced, "runs": runs, "runs_ok": ok_runs,
             "by_provider": by_prov, "by_kind": by_kind}
 

tests/conftest.py

@@ -0,0 +1,35 @@
+"""All tests use disposable storage, including module-import side effects."""
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+# This runs before test collection: abuse builds source metadata at import time.
+_SESSION_DATA = tempfile.TemporaryDirectory(prefix="omni-pytest-")
+os.environ["OMNI_DATA"] = _SESSION_DATA.name
+os.environ["OMNI_DB"] = str(Path(_SESSION_DATA.name) / "collection.sqlite3")
+
+from omni import config
+
+config.DATA = Path(_SESSION_DATA.name)
+config.DB = config.DATA / "collection.sqlite3"
+
+
+@pytest.fixture(autouse=True)
+def sandbox_data(tmp_path, monkeypatch):
+    """Give every test its own database, not only the new panel tests."""
+    monkeypatch.setattr(config, "DATA", tmp_path)
+    monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3")
+    monkeypatch.setenv("OMNI_DATA", str(tmp_path))
+    monkeypatch.setenv("OMNI_DB", str(config.DB))
+    from omni import abuse, panel_api
+    monkeypatch.setattr(abuse, "DB", config.DB)
+    monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json")
+
+
+def pytest_unconfigure(config):
+    _SESSION_DATA.cleanup()

tests/test_ingest_quality.py

@@ -0,0 +1,103 @@
+"""Offline regressions for feed metadata, date ranking and challenge filtering."""
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from omni import feeds, rank
+from omni.dates import parse_published
+from omni.model import Item
+from omni.sources import S
+
+
+@pytest.mark.parametrize("value", [
+    "2026-10-20T12:00:00Z", "2026-10-20T15:00:00+03:00",
+    "Tue, 20 Oct 2026 12:00:00 GMT", "2026-10-20T12:00:00",
+    "1792497600", "1792497600000", 1792497600,
+])
+def test_publication_formats_have_equal_freshness(value, monkeypatch):
+    stamp = datetime(2026, 10, 20, 12, tzinfo=timezone.utc).timestamp()
+    parsed = parse_published(value)
+    assert parsed is not None
+    assert parsed.timestamp() == stamp
+    monkeypatch.setattr(rank.time, "time", lambda: stamp + 86400)
+    assert rank._fresh(value) == pytest.approx(1 - 1 / 30)
+
+
+def test_date_only_and_invalid_dates(monkeypatch):
+    stamp = datetime(2026, 10, 20, tzinfo=timezone.utc).timestamp()
+    monkeypatch.setattr(rank.time, "time", lambda: stamp)
+    assert rank._fresh("2026-10-20") == 1
+    for value in (None, "", "not a date", "99999999999999999999999999", "NaN"):
+        assert parse_published(value) is None
+        assert rank._fresh(value) == 0.3
+    assert rank._fresh("2020-01-01") == 0
+    assert parse_published("2026") is None
+    assert parse_published("20261020") == datetime(2026, 10, 20, tzinfo=timezone.utc)
+
+
+def test_atom_alternate_links_author_and_date(monkeypatch):
+    src = S("atom_test", "models", "rss", "https://example.invalid/feed.atom")
+    xml = '''<feed xmlns="http://www.w3.org/2005/Atom"><entry>
+      <title>Runtime release 2.0</title>
+      <link rel="self" href="https://example.invalid/api/123"/>
+      <link rel="alternate" href="/releases/2.0"/>
+      <author><name>Release team</name></author>
+      <published>2026-10-20T15:00:00+03:00</published>
+      <summary>Faster inference and improved caching.</summary>
+    </entry></feed>'''
+    monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml))
+    rows = feeds.fetch(src)
+    assert len(rows) == 1
+    assert rows[0].url == "https://example.invalid/releases/2.0"
+    assert rows[0].author == "Release team"
+    assert rows[0].published == "2026-10-20T12:00:00+00:00"
+
+
+def test_rss_namespaced_creator_and_no_author(monkeypatch):
+    src = S("rss_test", "security", "rss", "https://example.invalid/feed")
+    xml = '''<rss xmlns:dc="http://purl.org/dc/elements/1.1/"><channel>
+      <item><title>Security advisory</title><link>https://example.invalid/advisory</link>
+      <dc:creator>Security team</dc:creator><pubDate>Tue, 20 Oct 2026 12:00:00 GMT</pubDate></item>
+      <item><title>New patch available</title><link>https://example.invalid/patch</link></item>
+    </channel></rss>'''
+    monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml))
+    rows = feeds.fetch(src)
+    assert len(rows) == 2
+    assert rows[0].author == "Security team"
+    assert rows[0].published == "2026-10-20T12:00:00+00:00"
+    assert rows[1].author == ""
+
+
+def test_fetch_drops_invalid_pages_but_keeps_short_real_posts(monkeypatch):
+    src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter")
+    def item(path, title, text=""):
+        return Item("test", "entry", url=path, title=title, text=text)
+    good = item("https://example.invalid/release", "v1.2")
+    real = item("https://example.invalid/article", "Access denied", "Real analysis of authorization failure. " * 20)
+    rows = [
+        item("", "A missing URL"),
+        item("javascript:alert(1)", "A dangerous URL"),
+        item("https://example.invalid/blank", ""),
+        item("https://example.invalid/challenge", "Just a moment...", "Checking your browser"),
+        item("https://example.invalid/denied", "Attention Required! | Cloudflare", "captcha"),
+        item("https://example.invalid/long-challenge", "Just a moment...",
+             "Checking your browser before accessing this site. " * 20),
+        {"title": "Malformed adapter result"}, None, "not an Item",
+        good, item(good.url, "Duplicate record"), real,
+    ]
+    monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: rows)
+    assert feeds.fetch(src) == [good, real]
+
+
+def test_fetch_bad_record_does_not_discard_valid_siblings(monkeypatch):
+    src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter")
+    good = Item("test", "entry", url="https://example.invalid/release", title="Real release")
+    bad = Item("test", "entry", url="https://example.invalid/bad", title="Malformed")
+    bad.text = None
+    monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: [bad, good])
+    assert feeds.fetch(src) == [good]

tests/test_panel_api.py

@@ -5,11 +5,38 @@ import sys
 import time
 from pathlib import Path
 
+import pytest
+
 ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT))
 os.environ.setdefault("PYTHONIOENCODING", "utf-8")
 
 
+@pytest.fixture(autouse=True)
+def isolated_panel_data(tmp_path, monkeypatch):
+    """Never touch the operator's database, watchlist, digest or TG session."""
+    from omni import abuse, config, panel_api, store, tgsess
+    from omni.model import Item
+
+    monkeypatch.setattr(config, "DATA", tmp_path)
+    monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3")
+    monkeypatch.setattr(abuse, "DB", config.DB)
+    monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json")
+    monkeypatch.setattr(config, "tg_cfg", lambda: {"session": "", "limit": 100})
+    monkeypatch.setattr(tgsess, "enabled", lambda: False)
+    today = time.strftime("%Y-%m-%d")
+    store.add([
+        Item(provider="pytest", kind="post", source="pytest-panel",
+             title="Firecrawl security dataset leak announcement",
+             text="Public incident report for regression coverage.",
+             url="https://example.invalid/seed", published=today),
+        Item(provider="pytest", kind="post", source="pytest-other",
+             title="Guide to observability and performance metrics",
+             text="A completely different article about tracing.",
+             url="https://example.invalid/other", published=today),
+    ])
+
+
 def _tmp_item(tag=None):
     from omni import store
     from omni.model import Item
@@ -124,14 +151,19 @@ def test_health_counts():
 
 
 # ---------- tg ----------
-def test_tg_status_shape_and_flag():
+@pytest.mark.parametrize("enabled", [False, True])
-    from omni import panel_api
+def test_tg_status_shape_and_flag(monkeypatch, enabled):
+    from omni import config, panel_api, tgsess
 
+    monkeypatch.setattr(tgsess, "enabled", lambda: enabled)
+    monkeypatch.setattr(tgsess, "me", lambda: {"id": 123, "first_name": "Test"})
+    monkeypatch.setattr(config, "tg_cfg",
+                        lambda: {"session": "test-account", "limit": 100})
     d = panel_api.tg_status()
-    assert {"enabled", "me", "error", "limit"} <= set(d)
+    assert {"enabled", "me", "error", "limit", "session"} <= set(d)
-    assert isinstance(d["enabled"], bool)
+    assert d["enabled"] is enabled
-    if d["enabled"]:
+    assert d["me"] == ({"id": 123, "first_name": "Test"} if enabled else None)
-        assert d["session"]
+    assert d["error"] is None
 
 
 # ---------- digest ----------
@@ -205,3 +237,170 @@ def test_browse2_filters_since_and_src():
     # src + since вместе
     rows5, total5 = store.browse2(q="", limit=5, src=name, since="2099-01-01")
     assert total5 == 0 and rows5 == []
+
+
+def test_registry_dedupes_url_and_name(monkeypatch):
+    from omni import sources, source_state
+
+    first = sources.S("first", "security", "rss", "https://example.invalid/feed")
+    same_url = sources.S("alias", "security", "rss", first.url + "/")
+    same_name = sources.S("FIRST", "security", "rss", "https://example.invalid/mirror")
+    other = sources.S("other", "security", "rss", "https://example.invalid/other")
+    monkeypatch.setattr(sources, "SOURCES", [first, same_url, same_name, other])
+    monkeypatch.setattr(sources, "_gen_sources", lambda: [])
+    monkeypatch.setattr(sources, "_fresh_sources", lambda factory: [])
+    monkeypatch.setattr(sources, "_quality_sources", lambda factory: [])
+    monkeypatch.setattr(sources, "radar_sources", lambda: [])
+    monkeypatch.setattr(sources, "panel_sources", lambda: [])
+    monkeypatch.setattr(source_state, "custom", lambda: [])
+    assert sources.all_sources() == [first, other]
+
+
+def test_fresh_sources_have_valid_metadata():
+    from omni import feeds, sources
+    from omni.sources_fresh import fresh
+
+    rows = fresh(sources.S)
+    assert rows
+    assert all(isinstance(s.tags, tuple) for s in rows)
+    assert all(s.adapter in feeds.ADAPTERS for s in rows)
+    assert len({s.name for s in rows}) == len(rows)
+
+
+def test_quality_wave_is_integrated_without_collisions():
+    from omni import feeds, sources
+    from omni.sources_quality import quality_sources
+
+    wave = quality_sources(sources.S)
+    registry = sources.all_sources()
+    assert len(wave) >= 30
+    assert all(isinstance(s.tags, tuple) and s.adapter in feeds.ADAPTERS for s in wave)
+    assert {s.name for s in wave} <= {s.name for s in registry}
+    assert len(registry) == len({s.name.strip().lower() for s in registry})
+    assert len(registry) == len({s.url.rstrip("/").lower() for s in registry})
+
+
+@pytest.mark.parametrize("tags", [(), ("news",), ("news", "cve")])
+def test_fresh_generator_preserves_tag_tuples(tmp_path, tags):
+    import runpy
+
+    from omni import sources
+
+    emit = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))["emit"]
+    row = {"name": "test_feed", "category": "security", "kind": "rss",
+           "adapter": "rss", "url": "https://example.invalid/feed", "tags": tags}
+    path = tmp_path / "generated_sources.py"
+    assert emit([row], [], path, dedupe=False) == 1
+    generated = runpy.run_path(str(path))["fresh"](sources.S)
+    assert generated[0].tags == tags
+    assert isinstance(generated[0].tags, tuple)
+
+
+def test_timeline_local_day_and_rolling_24_hours(monkeypatch):
+    import datetime
+    from omni import panel_api, store
+
+    now = datetime.datetime(2026, 10, 20, 1, 0).timestamp()
+    class FrozenDate(datetime.date):
+        @classmethod
+        def today(cls):
+            return cls(2026, 10, 20)
+
+    monkeypatch.setattr(panel_api._dt, "date", FrozenDate)
+    monkeypatch.setattr(panel_api.time, "time", lambda: now)
+    with store.db() as c:
+        c.execute("UPDATE items SET seen=? WHERE source='pytest-panel'", (now - 1800,))
+        c.execute("UPDATE items SET seen=? WHERE source='pytest-other'", (now - 23 * 3600,))
+    data = panel_api.timeline(days=2, mode="seen")
+    assert data["last24"] == 2, "rolling 24 hours must include yesterday evening"
+    assert data["today"] == 1, "local 00:30 import belongs to today's bar"
+    assert [p["n"] for p in data["points"]] == [1, 1]
+
+
+def test_run_stats_count_partial_successes():
+    from omni import store
+
+    for status in ("ok", "ok 5/10", "пусто", "Exception: timeout"):
+        store.log_run("pytest", "test", "", 0, 0, status)
+    stats = store.stats()
+    assert stats["runs"] == 4
+    assert stats["runs_ok"] == 2
+
+
+def test_timeline_last24_respects_publication_mode(monkeypatch):
+    import datetime
+    from omni import panel_api, store
+
+    now = time.time()
+    monkeypatch.setattr(panel_api.time, "time", lambda: now)
+    old = datetime.datetime.fromtimestamp(now - 48 * 3600, datetime.timezone.utc).isoformat()
+    future = datetime.datetime.fromtimestamp(now + 48 * 3600, datetime.timezone.utc).isoformat()
+    with store.db() as c:
+        c.execute("UPDATE items SET seen=?", (now - 60,))
+        c.execute("UPDATE items SET published=? WHERE source='pytest-panel'", (old,))
+        c.execute("UPDATE items SET published=? WHERE source='pytest-other'", (future,))
+    assert panel_api.timeline(mode="seen")["last24"] == 2
+    assert panel_api.timeline(mode="pub")["last24"] == 0
+
+
+def test_http_exports_match_feed_filters_and_item_find(monkeypatch):
+    import csv
+    import datetime
+    import io
+    import json
+    import threading
+    import urllib.request
+    from http.server import ThreadingHTTPServer
+    from omni import serve, store
+    from omni.model import Item
+
+    today = datetime.date.today().isoformat()
+    old = (datetime.date.today() - datetime.timedelta(days=10)).isoformat()
+    rows = [
+        Item("pytest", "post", source="quality-http", title="promo code: QUAL2026 $20 30 days",
+             text="Official promotional offer for testing.", url="https://example.invalid/quality-new", published=today),
+        Item("pytest", "post", source="quality-http", title="Legacy license costs $99",
+             text="An old pricing page.", url="https://example.invalid/quality-old", published=old),
+        Item("pytest", "post", source="quality-http", title="Latest debugging guide",
+             text="A free guide to tracing request latency.", url="https://example.invalid/quality-free", published=today),
+    ]
+    store.add(rows)
+    for status in (" OK ", "ok 2/5", "пусто", "Exception: timeout", None):
+        store.log_run("pytest", "quality-http", "", 0, 0, status)
+    monkeypatch.setattr(serve, "UI_TOKEN", "")
+    server = ThreadingHTTPServer(("127.0.0.1", 0), serve.Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    base = "http://127.0.0.1:%s" % server.server_address[1]
+    query = "?src=quality-http&days=1&priced=1&limit=1000"
+    try:
+        def get(path):
+            with urllib.request.urlopen(base + path, timeout=10) as res:
+                assert res.status == 200
+                return res.read().decode("utf-8-sig")
+
+        feed = json.loads(get("/api/items" + query))
+        export = json.loads(get("/api/export.json" + query))
+        csv_rows = list(csv.DictReader(io.StringIO(get("/api/export.csv" + query)), delimiter=";"))
+        assert feed["total"] == export["total"] == 1
+        assert [r["url"] for r in feed["items"]] == [rows[0].url]
+        assert [r["url"] for r in export["items"]] == [rows[0].url]
+        assert [r["url"] for r in csv_rows] == [rows[0].url]
+        assert "QUAL2026" in feed["items"][0]["find"]["codes"]
+        runs = json.loads(get("/api/runs?limit=2"))
+        assert runs["total"] == 5 and runs["shown"] == 2
+        assert runs["by_status"] == {"ok": 2, "empty": 1, "err": 1, "none": 1}
+        states = {r["state"] for r in json.loads(get("/api/runs?limit=10"))["items"]}
+        assert states == {"ok", "empty", "err", "none"}
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+def test_web_has_free_panel_block():
+    """Вкладка абузов обязана показывать free-панели new-api."""
+    html = open(os.path.join(ROOT, "web", "index.html"), encoding="utf-8").read()
+    assert "loadPanels" in html
+    assert "/api/panels" in html
+    assert "pn-grid" in html
+    assert ".pn-grid{" in html

tools/fresh_candidates.py

@@ -4,7 +4,6 @@
 переезжают в omni/sources_fresh.py (генерируется `--emit`).
 Формат dict совпадает с Source(**d); адаптеры — только из feeds.ADAPTERS.
 """
-import datetime
 
 
 def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None, **kw):
@@ -14,10 +13,6 @@ def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False,
     return row
 
 
-_NOW = datetime.datetime.utcnow()
-_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
-_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
-
 CANDIDATES = [
     # ---------------- 1. security news / research ----------------
     d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),

tools/probe_fresh.py

@@ -7,8 +7,19 @@
 Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись.
 Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина
 в отчёте была фактической, а не «0 items».
+
+Публичные хелперы (для тестов/внешних вызовов):
+  candidate_source(c) -> (Source, marker)  — dict кандидата -> Source, url
+      динамического источника пересчитан ДО сети; маркер лежит в extra["_dynamic"].
+  _registry_sources() -> [Source]          — all_sources(include_fresh=False)
+  _registry_keys() -> (set url, set name)  — нормализованные ключи реестра без
+      текущей sources_fresh (иначе перегенерация считает свою волну дублем).
+  _registry_urls() -> set url              — совместимость.
+  emit(rows, dead, path, dedupe=True)      — только известный dynamic-маркер;
+      неизвестный маркер = ValueError, а не замороженный протухший url.
 """
 import argparse
+import datetime as _dt
 import json
 import sys
 import time
@@ -60,34 +71,130 @@ def probe_one(src, limit):
 
 
 def _norm(u):
-    u = u.strip().lower().rstrip("/")
+    u = str(u or "").strip().lower().rstrip("/")
     for pre in ("https://", "http://"):
         if u.startswith(pre):
             u = u[len(pre):]
     return u[4:] if u.startswith("www.") else u
 
 
-def _registry_urls():
+def _norm_name(n):
+    """Имя источника — первичный ключ source_flags/runs: сравниваем без регистра,
+    пробелов и разделителей, иначе nvd-recent и nvd_recent разъезжаются."""
+    return "".join(ch for ch in str(n or "").strip().lower() if ch.isalnum())
+
+
+def _registry_sources():
+    """Реестр без текущего sources_fresh: перегенерация не должна считать
+    собственную прошлую волну «уже существующей» и стирать свои же находки."""
     from omni.sources import all_sources
-    return {_norm(s.url) for s in all_sources()}
+    try:
+        return all_sources(include_fresh=False)
+    except TypeError:                          # реестр без флага — совместимость
+        return all_sources()
+
+
+def _registry_keys():
+    """(нормализованные url, нормализованные имена) реестра без sources_fresh."""
+    rows = _registry_sources()
+    return ({_norm(s.url) for s in rows if s.url},
+            {_norm_name(s.name) for s in rows if s.name})
+
+
+def _registry_urls():
+    """Только url-ключи реестра (совместимость с внешними вызовами)."""
+    return _registry_keys()[0]
+
+
+NVD_BASE = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+KNOWN_DYNAMIC = ("NVD_LAST_WEEK",)
+
+
+def _nvd_week_url():
+    """Фактический url NVD на момент прогона: окно lastMod <=120 дней, база
+    без окна отдаёт самые старые записи страницы и как «свежая» бесполезна."""
+    now = _dt.datetime.utcnow()
+    since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
+    return ("%s?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
+            % (NVD_BASE, since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))
+
+
+def candidate_source(c):
+    """dict кандидата -> (Source, dynamic-маркер), url пересчитан до сети.
+
+    Маркер сохраняется в extra["_dynamic"], чтобы отчёт/emit знали, что url
+    нельзя замораживать. Неизвестный маркер — ValueError: молча подставить
+    статический url значит опубликовать заведомо протухший источник.
+    """
+    row = dict(c)
+    marker = str(row.pop("dynamic", "") or "")
+    extra = dict(row.get("extra") or {})
+    if marker:
+        if marker not in KNOWN_DYNAMIC:
+            raise ValueError("unknown dynamic marker %r" % marker)
+        if marker == "NVD_LAST_WEEK":
+            row["url"] = _nvd_week_url()
+        extra["_dynamic"] = marker
+    row["extra"] = extra
+    return Source(**row), marker
 
 
 # контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
 CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
 
 
+def _extra_lit(extra):
+    """extra-литерал без служебных ключей: _dynamic — метаданные прогона, не источника."""
+    clean = {k: v for k, v in sorted((extra or {}).items()) if k != "_dynamic"}
+    return "" if not clean else ", extra=%r" % (clean,)
+
+
+def _frozen_window(rows):
+    """Строки, где в url уже стоит вычисленное окно (lastMod), но маркера нет:
+    такой отчёт старее фикса dynamic — при emit окно застынет на дате прогона."""
+    return [str(r.get("name")) for r in rows
+            if not r.get("dynamic") and "lastmodstartdate=" in str(r.get("url") or "").lower()]
+
+
 def emit(rows, dead, path, dedupe=True):
-    """omni/sources_fresh.py: только живые, без импортов из omni.sources."""
+    """omni/sources_fresh.py: только живые, без импортов из omni.sources.
-    def q(s):
+
-        return '"%s"' % str(s).replace('"', '\\"')
+    Литералы пишем через repr: json.dumps даёт true/false/null — это не Python.
+    Дедуп сверяет и url, и имя (нормализованные) и добавляет принятые строки в
+    виденное, поэтому партия уникальна и сама по себе. Динамический url
+    поддерживается только для известного маркера: неизвестный — явная ошибка,
+    а не замороженный url, который молча протухнет.
+    """
+    rows = list(rows)
+    unknown = sorted({str(r.get("dynamic")) for r in rows
+                      if r.get("dynamic") and r.get("dynamic") not in KNOWN_DYNAMIC})
+    if unknown:
+        raise ValueError("emit: неизвестный dynamic-маркер %s — замораживать url нельзя"
+                         % ", ".join(map(repr, unknown)))
+    frozen = _frozen_window(rows)
+    if frozen:
+        print("emit: ВНИМАНИЕ, окно в url без dynamic-маркера (%s): отчёт устарел, "
+              "перепрогони probe_fresh" % ", ".join(frozen))
 
     if dedupe:
-        have = _registry_urls()
+        have_url, have_name = _registry_keys()
-        kept, skipped = [], []
+        seen_url, seen_name = set(have_url), set(have_name)
-        for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
+        kept, skipped, dupes = [], [], []
-            n = _norm(r["url"])
+        for r in rows:
-            (skipped if n in have else kept).append(r)
+            n, nm = _norm(r.get("url")), _norm_name(r.get("name"))
+            in_registry = (bool(n) and n in have_url) or (bool(nm) and nm in have_name)
+            if r.get("name") in CONTENT_DUP:
+                in_registry = True
+            if in_registry or (bool(n) and n in seen_url) or (bool(nm) and nm in seen_name):
+                (skipped if in_registry else dupes).append(r)
+                continue
+            if n:
+                seen_url.add(n)
+            if nm:
+                seen_name.add(nm)
+            kept.append(r)
         print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
+        print("emit: дубли внутри партии -> %d: %s" % (len(dupes), ", ".join(r["name"] for r in dupes)))
         rows = kept
 
     lines = [
@@ -114,23 +221,23 @@ def emit(rows, dead, path, dedupe=True):
         "    return [",
     ]
     for r in rows:
-        extra = ""
+        marker = str(r.get("dynamic") or "")
-        if r.get("extra"):
+        tags = repr(tuple(r.get("tags") or ()))
-            extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True)
+        limit = r.get("limit")
-        limit = r.get("limit") or 15
+        lim = ", limit=%d" % limit if isinstance(limit, int) and limit != 20 else ""
-        lim = "" if limit == 15 else ", limit=%d" % limit
         det = ", detail=True" if r.get("detail") else ""
-        tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
+        if marker == "NVD_LAST_WEEK":
-        if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
+            lines.append("        S(%r, %r, %r, _nvd_week(), %r, %s%s%s%s)," % (
-            lines.append('        S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
+                r["name"], r.get("category", ""), r.get("kind", ""), r.get("adapter", "rss"),
-                q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
+                tags, lim, det, _extra_lit(r.get("extra"))))
             continue
-        lines.append('        S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
+        lines.append("        S(%r, %r, %r, %r, %r, %s%s%s%s)," % (
-            q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
+            r["name"], r.get("category", ""), r.get("kind", ""), r.get("url", ""), r.get("adapter", "rss"),
-            tags, lim, det, extra))
+            tags, lim, det, _extra_lit(r.get("extra"))))
     lines += ["    ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"]
     for r in dead:
-        lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90]))
+        lines.append("# %-26s %-44s %s" % (r.get("name", "?"), str(r.get("url") or "")[:44],
+                                          (r.get("err") or "0 items")[:90]))
     text = "\n".join(lines) + "\n"
     path.write_text(text, encoding="utf-8")
     return len(rows)
@@ -153,6 +260,12 @@ def main():
         rows = rep["results"]
         alive = [r for r in rows if r["ok"]]
         dead = [r for r in rows if not r["ok"]]
+        frozen = _frozen_window(alive)
+        if frozen:
+            print("emit-only: отчёт без dynamic-маркера для %s — окно в url застынет на дате прогона; "
+                  "перепрогони probe_fresh (сеть) или правь источники вручную"
+                  % ", ".join(frozen))
+            return 2
         n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
         print("omni/sources_fresh.py: %d источников" % n)
         return 0
@@ -161,11 +274,11 @@ def main():
     srcs = []
     for c in cands:
         try:
-            cc = dict(c)
+            src, _marker = candidate_source(c)
-            cc.pop("dynamic", None)
-            srcs.append(Source(**cc))
         except Exception as e:                  # noqa: BLE001
             print("skip %s: %s" % (c.get("name"), e))
+            continue
+        srcs.append(src)
 
     t0 = time.time()
     with ThreadPoolExecutor(max_workers=a.workers) as ex:

tools/probe_quality_sources.py

@@ -0,0 +1,439 @@
+"""Живой прогон пула quality-кандидатов (официальные release/security фиды).
+
+    python tools/probe_quality_sources.py                  # широкий пул -> data/quality_sources_pool.json
+    python tools/probe_quality_sources.py --only gh_rel_    # фильтр по имени
+    python tools/probe_quality_sources.py --confirm         # сверить SPECS модуля -> data/quality_sources_probe.json
+
+Что проверяется (acceptance):
+  1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS <item> и Atom <entry>);
+  2. >= MIN_ITEMS записей с непустым http(s)-URL;
+  3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка);
+  4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория
+     (в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в
+     all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть);
+  5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%).
+Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator
+в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0;
+ненулевой blank_url — сигнал регрессии адаптера, а не норма.
+
+Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется.
+"""
+import argparse
+import json
+import re
+import sys
+import time
+import urllib.parse
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from omni import net                      # noqa: E402
+from omni.sources import Source           # noqa: E402
+
+OUT = ROOT / "data" / "quality_sources_probe.json"
+OUT_POOL = ROOT / "data" / "quality_sources_pool.json"
+MIN_ITEMS = 2
+TITLE_OVERLAP_REJECT = 0.6
+ENTRY_OVERLAP_REJECT = 0.5
+TIMEOUT = 12
+
+GH = """huggingface/diffusers
+huggingface/peft
+huggingface/accelerate
+huggingface/text-generation-inference
+huggingface/tokenizers
+huggingface/candle
+pytorch/pytorch
+pytorch/torchao
+NVIDIA/TensorRT-LLM
+NVIDIA/nvidia-container-toolkit
+flashinfer-ai/flashinfer
+microsoft/onnxruntime
+microsoft/DeepSpeed
+ml-explore/mlx
+ggml-org/whisper.cpp
+openai/whisper
+AUTOMATIC1111/stable-diffusion-webui
+exo-explore/exo
+vllm-project/vllm-ascend
+caddyserver/caddy
+keycloak/keycloak
+grafana/grafana
+tailscale/tailscale
+envoyproxy/envoy
+hashicorp/vault
+hashicorp/terraform
+hashicorp/consul
+sigstore/cosign
+aquasecurity/trivy
+anchore/grype
+projectdiscovery/nuclei
+zaproxy/zaproxy
+moby/moby
+containerd/containerd
+opencontainers/runc
+kubernetes/kubernetes
+istio/istio
+argoproj/argo-cd
+cert-manager/cert-manager
+traefik/traefik
+prometheus/prometheus
+opensearch-project/OpenSearch
+elastic/elasticsearch
+minio/minio
+go-gitea/gitea
+nextcloud/server
+portainer/portainer
+wazuh/wazuh
+OISF/suricata
+zeek/zeek
+fail2ban/fail2ban
+openssl/openssl
+openssh/openssh-portable
+curl/curl
+spring-projects/spring-boot
+quarkusio/quarkus
+jenkinsci/jenkins
+immich-app/immich
+netbox-community/netbox
+open-telemetry/opentelemetry-collector
+modelcontextprotocol/python-sdk
+openai/openai-agents-python
+pydantic/pydantic-ai
+langchain-ai/langgraph"""
+
+# имя = gh_rel_<repo> без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers)
+def _gh_name(repo):
+    return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "")
+
+
+_AI_TAG = ("release", "github", "ai")
+_SEC_TAG = ("security", "release", "github")
+
+POOL = [
+    {"name": _gh_name(r), "category": "release", "kind": "atom",
+     "url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10}
+    for r in GH.split()
+]
+
+POOL += [
+    # ---------- официальные security-фиды вендоров/проектов ----------
+    {"name": "chrome_releases", "category": "security", "kind": "atom",
+     "url": "https://chromereleases.googleblog.com/feeds/posts/default",
+     "tags": ("security", "vendor", "chrome"), "limit": 15},
+    {"name": "project_zero", "category": "security", "kind": "atom",
+     "url": "https://googleprojectzero.blogspot.com/feeds/posts/default",
+     "tags": ("security", "research", "google"), "limit": 10},
+    {"name": "ubuntu_usn", "category": "security", "kind": "rss",
+     "url": "https://ubuntu.com/security/notices/rss.xml",
+     "tags": ("security", "vendor", "ubuntu"), "limit": 20},
+    {"name": "amazon_linux_alas", "category": "security", "kind": "rss",
+     "url": "https://alas.aws.amazon.com/alas.rss",
+     "tags": ("security", "vendor", "aws"), "limit": 20},
+    {"name": "msrc_blog", "category": "security", "kind": "rss",
+     "url": "https://msrc.microsoft.com/blog/feed",
+     "tags": ("security", "vendor", "microsoft"), "limit": 15},
+    {"name": "ms_security_blog", "category": "security", "kind": "rss",
+     "url": "https://www.microsoft.com/en-us/security/blog/feed/",
+     "tags": ("security", "vendor", "microsoft"), "limit": 15},
+    {"name": "cisco_talos", "category": "security", "kind": "rss",
+     "url": "https://blog.talosintelligence.com/rss/",
+     "tags": ("security", "research", "cisco"), "limit": 15},
+    {"name": "portswigger_research", "category": "security", "kind": "rss",
+     "url": "https://portswigger.net/research/rss",
+     "tags": ("security", "research", "web"), "limit": 15},
+    {"name": "trail_of_bits", "category": "security", "kind": "rss",
+     "url": "https://blog.trailofbits.com/feed/",
+     "tags": ("security", "research"), "limit": 15},
+    {"name": "cisa_advisories", "category": "security", "kind": "rss",
+     "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
+     "tags": ("security", "vendor", "cisa"), "limit": 20},
+    {"name": "sans_isc", "category": "security", "kind": "rss",
+     "url": "https://isc.sans.edu/rssfeed.xml",
+     "tags": ("security", "research", "sans"), "limit": 15},
+    {"name": "nodejs_blog", "category": "release", "kind": "rss",
+     "url": "https://nodejs.org/en/feed/blog.xml",
+     "tags": ("release", "vendor", "nodejs"), "limit": 15},
+    {"name": "rust_blog", "category": "release", "kind": "atom",
+     "url": "https://blog.rust-lang.org/feed.xml",
+     "tags": ("release", "vendor", "rust"), "limit": 15},
+    {"name": "go_blog", "category": "release", "kind": "atom",
+     "url": "https://go.dev/blog/feed.atom",
+     "tags": ("release", "vendor", "go"), "limit": 15},
+    {"name": "php_news", "category": "release", "kind": "atom",
+     "url": "https://www.php.net/feed.atom",
+     "tags": ("release", "vendor", "php"), "limit": 15},
+    {"name": "python_insider", "category": "release", "kind": "atom",
+     "url": "https://pythoninsider.blogspot.com/feeds/posts/default",
+     "tags": ("release", "vendor", "python"), "limit": 15},
+    {"name": "gitlab_releases", "category": "release", "kind": "atom",
+     "url": "https://about.gitlab.com/atom.xml",
+     "tags": ("release", "vendor", "gitlab"), "limit": 15},
+    {"name": "cloudflare_security", "category": "security", "kind": "rss",
+     "url": "https://blog.cloudflare.com/tag/security/rss/",
+     "tags": ("security", "vendor", "cloudflare"), "limit": 15},
+    {"name": "aws_security_bulletins", "category": "security", "kind": "rss",
+     "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
+     "tags": ("security", "vendor", "aws"), "limit": 20},
+    {"name": "hashicorp_blog", "category": "release", "kind": "rss",
+     "url": "https://www.hashicorp.com/blog/feed.xml",
+     "tags": ("release", "vendor", "hashicorp"), "limit": 15},
+    {"name": "kubernetes_blog", "category": "release", "kind": "rss",
+     "url": "https://kubernetes.io/feed.xml",
+     "tags": ("release", "vendor", "kubernetes"), "limit": 15},
+    {"name": "docker_blog", "category": "release", "kind": "rss",
+     "url": "https://www.docker.com/feed/",
+     "tags": ("release", "vendor", "docker"), "limit": 15},
+    # ---------- технические/AI release-блоги вендоров ----------
+    {"name": "nvidia_dev_blog", "category": "release", "kind": "rss",
+     "url": "https://developer.nvidia.com/blog/feed/",
+     "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
+    {"name": "pytorch_blog", "category": "release", "kind": "rss",
+     "url": "https://pytorch.org/feed/",
+     "tags": ("release", "vendor", "pytorch", "ai"), "limit": 15},
+    {"name": "vllm_blog", "category": "release", "kind": "rss",
+     "url": "https://blog.vllm.ai/feed.xml",
+     "tags": ("release", "vendor", "vllm", "ai"), "limit": 15},
+    {"name": "ray_blog", "category": "release", "kind": "rss",
+     "url": "https://www.anyscale.com/blog/rss.xml",
+     "tags": ("release", "vendor", "ray", "ai"), "limit": 15},
+    {"name": "langchain_blog", "category": "release", "kind": "rss",
+     "url": "https://blog.langchain.dev/rss/",
+     "tags": ("release", "vendor", "langchain", "ai"), "limit": 15},
+    {"name": "modal_blog", "category": "release", "kind": "rss",
+     "url": "https://modal.com/blog/feed.xml",
+     "tags": ("release", "vendor", "modal", "ai"), "limit": 15},
+    {"name": "ollama_blog", "category": "release", "kind": "rss",
+     "url": "https://ollama.com/blog/rss.xml",
+     "tags": ("release", "vendor", "ollama", "ai"), "limit": 15},
+]
+
+BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden",
+                   "404 not found", "not found", "rate limit", "too many requests",
+                   "checking your browser", "ddos-guard", "domain not found",
+                   "privacy error", "service unavailable", "bad gateway", "error")
+
+
+def _norm_url(u):
+    u = (u or "").strip().lower().rstrip("/")
+    for pre in ("https://", "http://"):
+        if u.startswith(pre):
+            u = u[len(pre):]
+    return u[4:] if u.startswith("www.") else u
+
+
+def _valid_url(u):
+    try:
+        p = urllib.parse.urlsplit(u or "")
+    except Exception:
+        return False
+    return p.scheme in ("http", "https") and bool(p.netloc)
+
+
+def _bad_title(t):
+    low = (t or "").strip().lower()
+    if len(low) < 3:
+        return True
+    return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60
+
+
+def _parse(text):
+    """Локальный парсер RSS/Atom -> [(title, link, published)]."""
+    out = []
+    try:
+        root = ET.fromstring(text.encode("utf-8", "ignore"))
+    except Exception:
+        return out
+    ns = {"a": "http://www.w3.org/2005/Atom"}
+    items = root.findall(".//item")
+    if items:
+        for e in items:
+            link = (e.findtext("link") or "").strip()
+            if not link:
+                ge = e.find("{http://www.w3.org/2005/Atom}link")
+                if ge is not None:
+                    link = (ge.get("href") or "").strip()
+            out.append(((e.findtext("title") or "").strip(), link,
+                        (e.findtext("pubDate") or e.findtext("published") or "").strip()))
+        return out
+    for e in root.findall(".//a:entry", ns):
+        link = ""
+        for le in e.findall("a:link", ns):
+            if le.get("rel") in (None, "alternate") and le.get("href"):
+                link = le.get("href").strip()
+                if le.get("rel") == "alternate":
+                    break
+        out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link,
+                    (e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip()))
+    return out
+
+
+def _registry():
+    from omni import sources as reg
+    rows = reg.all_sources()
+    names, urls, repos = set(), set(), set()
+    for s in rows:
+        names.add((s.name or "").strip().lower())
+        urls.add(_norm_url(s.url))
+        m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I)
+        if m:
+            repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git")))
+    return rows, names, urls, repos
+
+
+def _repo_of(url):
+    m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I)
+    return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None
+
+
+def _probe(spec):
+    t0 = time.time()
+    r = net.get(spec["url"], timeout=TIMEOUT)
+    status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"],
+              "url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"),
+              "limit": spec.get("limit", 10), "http": getattr(r, "status", 0),
+              "mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""),
+              "ms": int((time.time() - t0) * 1000), "err": ""}
+    if getattr(r, "error", ""):
+        status["err"] = str(r.error)[:160]
+    if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
+        status["entries"] = 0
+        status["reject"] = "http_%s" % getattr(r, "status", 0)
+        return status
+    parsed = _parse(r.text)
+    good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)]
+    status["entries"] = len(parsed)
+    status["valid"] = len(good)
+    status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]]
+    if len(good) < MIN_ITEMS:
+        status["reject"] = "only_%d_valid" % len(good)
+    else:
+        status["reject"] = ""
+    return status
+
+
+def _fetch_check(spec):
+    """Что вернул штатный адаптер (документируем эффект бага Atom-link)."""
+    from omni import feeds
+    src = Source(spec["name"], spec["category"], spec["kind"], spec["url"],
+                 spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10))
+    t0 = time.time()
+    try:
+        items = feeds.fetch(src, spec.get("limit", 10)) or []
+        err = ""
+    except Exception as e:                      # noqa: BLE001
+        items, err = [], "raise:%s" % str(e)[:120]
+    blank = sum(1 for i in items if not _valid_url(i.url))
+    return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000),
+            "err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]}
+                                    for i in items[:3]]}
+
+
+def _dup_reasons(status, names, urls, repos, accepted):
+    why = []
+    name = (status["name"] or "").lower()
+    if name in names:
+        why.append("name_in_registry")
+    if _norm_url(status["url"]) in urls:
+        why.append("url_in_registry")
+    repo = _repo_of(status["url"])
+    if repo and repo in repos:
+        why.append("repo_mirror:%s/%s" % repo)
+    entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])}
+    titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])}
+    for prev in accepted:
+        p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])}
+        p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])}
+        if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT:
+            why.append("entries_overlap:%s" % prev["name"])
+            break
+        if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT:
+            why.append("titles_overlap:%s" % prev["name"])
+            break
+    return why
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--only", default="")
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--limit", type=int, default=10)
+    ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality")
+    args = ap.parse_args()
+
+    if args.confirm:
+        from omni.sources_quality import SPECS as pool
+    else:
+        pool = POOL
+    if args.only:
+        pool = [p for p in pool if args.only in p["name"]]
+
+    rows, names, urls, repos = _registry()
+    # после интеграции quality_sources в all_sources() эти же записи есть в реестре:
+    # себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники
+    names -= {(p["name"] or "").lower() for p in pool}
+    urls -= {_norm_url(p["url"]) for p in pool}
+    repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r}
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        probed = list(ex.map(_probe, pool))
+    elapsed = round(time.time() - t0, 1)
+
+    accepted, rejected = [], []
+    for st in probed:
+        if st["reject"]:
+            st["dup"] = []
+            rejected.append(st)
+            continue
+        why = _dup_reasons(st, names, urls, repos, accepted)
+        st["dup"] = why
+        (rejected if why else accepted).append(st)
+
+    with ThreadPoolExecutor(max_workers=4) as ex:
+        for st, chk in zip(accepted, ex.map(_fetch_check, accepted)):
+            st["feeds_fetch"] = chk
+
+    report = {
+        "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "mode": "confirm" if args.confirm else "pool",
+        "pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows),
+        "module": "omni/sources_quality.py" if args.confirm else None,
+        "elapsed_s": elapsed, "min_items": MIN_ITEMS,
+        "acceptance": {
+            "min_valid_entries": MIN_ITEMS,
+            "valid_url_required": "http(s) с непустым netloc",
+            "bad_title_marks": list(BAD_TITLE_MARKS),
+            "dup_keys": ["name", "normalized_url", "github_repo_mirror",
+                         "entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT,
+                         "titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT],
+        },
+        "accepted_n": len(accepted), "rejected_n": len(rejected),
+        "accepted_by_category": {},
+        "accepted": accepted, "rejected": rejected,
+    }
+    for st in accepted:
+        report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1
+
+    target = OUT if args.confirm else OUT_POOL
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
+
+    print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed))
+    for st in accepted:
+        s = st["samples"][0] if st["samples"] else {"title": "", "url": ""}
+        ff = st.get("feeds_fetch") or {}
+        print("OK  %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s"
+              % (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0),
+                 ff.get("blank_url", 0), s["title"][:60], s["url"][:80]))
+    for st in rejected:
+        print("NO  %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []),
+                                   (st.get("samples") or [{}])[0].get("title", "")[:50]))
+    print("report:", target)
+
+
+if __name__ == "__main__":
+    main()

tools/run_fresh.py

@@ -23,10 +23,16 @@ def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--limit", type=int, default=10)
     ap.add_argument("--workers", type=int, default=8)
-    ap.add_argument("--out", default="data/fresh_run.json")
+    ap.add_argument("--wave", choices=("fresh", "quality"), default="fresh")
+    ap.add_argument("--out", default="")
     a = ap.parse_args()
 
+    if a.wave == "quality":
+        from omni.sources_quality import quality_sources
+        names = {f.name for f in quality_sources(_Fake)}
+    else:
         names = {f.name for f in _fresh(_Fake)}
+    output = a.out or "data/%s_run.json" % a.wave
     picked = [s for s in sources.all_sources() if s.name in names]
     print("источников свежей волны:", len(picked), "| лимит", a.limit)
     t0 = time.time()
@@ -45,7 +51,8 @@ def main():
     out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked),
            "alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup,
            "by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad}
-    json.dump(out, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    with open(output, "w", encoding="utf-8") as f:
+        json.dump(out, f, ensure_ascii=False, indent=1)
     print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad))
     print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup)
     print("топ:", list(out["by_source"].items())[:12])

tools/shot_abuse.py

@@ -9,7 +9,7 @@ with Camoufox(headless=True, geoip=False, humanize=True) as b:
     p = b.new_page(viewport={"width": 1400, "height": 1000})
     p.goto(URL, wait_until="domcontentloaded", timeout=60000)
     p.wait_for_selector("#view", timeout=60000)
-    p.wait_for_timeout(7000)
+    p.wait_for_timeout(14000)
     txt = p.inner_text("#view")[:900]
     p.screenshot(path=OUT, full_page=False)
     print(txt)

web/index.html

@@ -210,6 +210,11 @@ label.chk{display:inline-flex;align-items:center;gap:6px;color:var(--muted);font
   .top{padding:8px 10px}
   .crumb{display:none}
 }
+.pn-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(190px,1fr));gap:8px;margin-top:8px}
+.pn{display:flex;flex-direction:column;gap:2px;padding:8px 10px;border:1px solid var(--line);border-radius:10px;text-decoration:none;color:inherit;font-size:12px}
+.pn:hover{border-color:var(--acc)}
+.pn b{font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+.pn .muted{opacity:.6}
 </style>
 </head>
 <body>
@@ -261,6 +266,11 @@ const dt=s=>{if(!s)return"";s=String(s);const m=s.match(/^(\d{4})-(\d{2})-(\d{2}
 const ago=s=>{if(!s)return"";const t=Date.parse(!isNaN(s)?+s:s);if(!t)return dt(s);
   const d=(Date.now()-t)/1000;if(d<3600)return Math.max(1,Math.round(d/60))+" мин";if(d<86400)return Math.round(d/3600)+" ч";return Math.round(d/86400)+" дн"};
 const debounce=(f,ms=220)=>{let t;return(...a)=>{clearTimeout(t);t=setTimeout(()=>f(...a),ms)}};
+/* статус прогона: "ok*" — успех; "пусто"/"empty" — пустой ответ (не ошибка); "" — нет статуса */
+const isOkSt=s=>String(s==null?"":s).trim().toLowerCase().startsWith("ok");
+const isEmpSt=s=>["пусто","empty"].includes(String(s==null?"":s).trim().toLowerCase());
+const stKey=s=>{const v=String(s==null?"":s).trim();return !v?"none":(isOkSt(v)?"ok":(isEmpSt(v)?"empty":"err"))};
+const stPill=s=>{const k=stKey(s);return k==="ok"?"ok":(k==="err"?"no":"unk")};
 
 let TOAST_T;
 function toast(msg,bad=false){const el=$("#toast");el.textContent=msg;el.classList.toggle("bad",!!bad);el.classList.add("on");clearTimeout(TOAST_T);TOAST_T=setTimeout(()=>el.classList.remove("on"),2600);}
@@ -303,9 +313,9 @@ const NAV=[
   {id:"digest",   label:"Дайджест",    icon:"M6 3h9l4 4v14H6zM15 3v4h4"},
   {id:"watch",    label:"Watchlist",   icon:"M2 12s3.5-6 10-6 10 6 10 6-3.5 6-10 6S2 12 2 12zM12 9a3 3 0 1 0 0 6 3 3 0 0 0 0-6z"}
 ];
-const S={view:"overview",q:"",cat:"",type:"all",sort:"score",days:0,priced:false,
+const S={view:"overview",q:"",cat:"",type:"all",sort:"score",days:0,priced:false,src:"",
          offset:0,items:[],total:0,auto:localStorage.getItem("omni.auto")==="1",
-         types:[],cats:[]};
+         runQ:"",runF:"all",types:[],cats:[]};
 
 function route(){
   const h=location.hash.replace(/^#\/?/,"");
@@ -314,6 +324,7 @@ function route(){
   S.view=NAV.some(n=>n.id===v)?v:"overview";
   S.q=p.get("q")||""; S.cat=p.get("cat")||""; S.type=p.get("type")||"all";
   S.sort=p.get("sort")||"score"; S.days=+(p.get("days")||0); S.priced=p.get("priced")==="1";
+  S.src=p.get("src")||"";
   S.offset=0;
   $("#q").value=S.q;
   $(".crumb").innerHTML="<b>"+esc(NAV.find(n=>n.id===S.view).label)+"</b>"+(S.q?` · «${esc(S.q)}»`:"");
@@ -328,6 +339,7 @@ function go(view,params={},replace=false){
   const sort=params.sort??S.sort; if(sort!=="score")p.set("sort",sort);
   const days=params.days??S.days; if(days)p.set("days",days);
   if((params.priced??S.priced))p.set("priced","1");
+  const src=params.src!==undefined?params.src:S.src; if(src)p.set("src",src);
   const url="#/"+view+(p.toString()?"?"+p:"");
   if(replace)history.replaceState(null,"",url); else location.hash=url;
   if(replace)route();
@@ -376,10 +388,12 @@ V.overview=async()=>{
     api("/api/watch").catch(()=>({rules:[]})), api("/api/abuse?limit=8").catch(()=>({items:[]}))
   ]);
   S.stats=st;
-  const day=tl.last24||tl.points[tl.points.length-1]?.n||0, prev=tl.points[tl.points.length-2]?.n||0;
+  // last24 — честное скользящее окно от бэкенда; «сегодня» неполный день, процент не считаем
-  const dPct=prev?Math.round((day-prev)/prev*100):null;
+  const day=tl.last24??0;
   const live=(st.sources?(st.alive/st.sources*100):0).toFixed(0);
-  const okRate=st.runs?(st.runs_ok/st.runs*100).toFixed(0):0;
+  const bs=runs.by_status||{};
+  const runAll=(bs.ok||0)+(bs.empty||0)+(bs.err||0)+(bs.none||0);
+  const okRate=runAll?Math.round((bs.ok||0)/runAll*100):(st.runs?Math.round(st.runs_ok/st.runs*100):0);
   const kpi=(label,val,sub,accent)=>`<div class="card hov"><div class="sub">${label}</div>
      <div class="kpi num" style="${accent?`color:${accent}`:""}">${val}</div><div class="sub">${sub}</div></div>`;
   const hits=(watch.rules||[]).reduce((a,r)=>a+(r.count||0),0);
@@ -390,7 +404,7 @@ V.overview=async()=>{
       <button class="btn ${S.auto?"primary":""}" data-act="auto">${S.auto?"Автообновление вкл":"Автообновление выкл"}</button>
     </div></div>
   <div class="grid g4">
-    ${kpi("Записей всего",num(st.items),`+${num(day)} за сутки импорта${dPct===null?"":` <span class="${dPct>=0?"up":"down"}">${dPct>=0?"▲":"▼"} ${Math.abs(dPct)}%</span>`}`)}
+    ${kpi("Записей всего",num(st.items),`за последние 24 часа: +${num(day)}`)}
     ${kpi("Источников живых",`${num(st.alive)}<small>/ ${num(st.sources)}</small>`,`${live}% реестра отдают записи`)}
     ${kpi("Ключей готово",`${num(st.keys?.ready||0)}<small>/ ${num(st.keys?.keys||0)}</small>`,`${num(st.keys?.services||0)} сервисов · ротация и остывание`)}
     ${kpi("Watchlist",num((watch.rules||[]).length),`${num(hits)} совпадений по правилам`)}
@@ -413,10 +427,10 @@ V.overview=async()=>{
 
   <div class="grid g2" style="margin-top:14px">
     <div class="card"><h3>Свежее <span class="r"><a href="#/feed">вся лента →</a></span></h3><div id="ov-latest">${skel(4)}</div></div>
-    <div class="card"><h3>Последние прогоны <span class="r"><a href="#/runs">все →</a></span></h3>
+    <div class="card"><h3>Последние прогоны <span class="r">ok ${num(okRate)}% · <a href="#/runs">все →</a></span></h3>
       <table><tbody>${(runs.items||[]).map(r=>`<tr><td class="mut mono nowrap">${esc(r.when||"")}</td>
         <td class="nowrap">${esc(r.source||"—")}</td><td class="right mono">${r.got??""}/${r.kept??""}</td>
-        <td class="right"><span class="pill ${r.status==="ok"?"ok":"no"}"></span><span class="dim">${esc(r.status||"")}</span></td></tr>`).join("")}</tbody></table>
+        <td class="right"><span class="pill ${stPill(r.status)}" title="${esc(r.status||"нет статуса")}"></span><span class="dim">${esc(r.status||"—")}</span></td></tr>`).join("")}</tbody></table>
     </div>
   </div>
 
@@ -441,8 +455,8 @@ V.feed=async()=>{
   const box=$("#view");
   box.innerHTML=`<div class="page-h"><div><h2>Лента</h2><p>Все записи реестра: фильтры по типу, периоду, сортировке; клик — детали и соседи.</p></div>
     <div class="right">
-      <a class="btn sm" href="/api/export.csv?${exQS()}" download title="Экспорт текущего фильтра, до 1000 записей">CSV</a>
+      <a class="btn sm" href="/api/export.csv?${exQS()}" download title="Экспорт текущего фильтра: период, тип, источник, цена; до 1000 записей">CSV</a>
-      <a class="btn sm" href="/api/export.json?${exQS()}" download title="Экспорт текущего фильтра, до 1000 записей">JSON</a>
+      <a class="btn sm" href="/api/export.json?${exQS()}" download title="Экспорт текущего фильтра: период, тип, источник, цена; до 1000 записей">JSON</a>
     </div></div>
     <div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
       <span class="dim" style="font-size:11.5px">Типы:</span>
@@ -456,12 +470,15 @@ V.feed=async()=>{
     <div class="card" style="padding:0"><div id="list">${skel(6)}</div><div id="more" class="pager"></div></div>`;
   await loadItems(true);
 };
-const exQS=()=>new URLSearchParams({q:S.q,category:S.cat,type:S.type,sort:S.sort,...(S.priced?{priced:"1"}:{})}).toString();
+const exQS=()=>new URLSearchParams({q:S.q,category:S.cat,type:S.type,sort:S.sort,
+  ...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{}),...(S.src?{src:S.src}:{}),
+  limit:"1000"}).toString();
 
 async function loadItems(reset){
   if(reset){S.offset=0;S.items=[]}
   const p=new URLSearchParams({limit:"40",offset:String(S.offset),q:S.q,sort:S.sort,type:S.type,
-    ...(S.cat?{category:S.cat}:{}),...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{})});
+    ...(S.cat?{category:S.cat}:{}),...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{}),
+    ...(S.src?{src:S.src}:{})});
   const d=await api("/api/items?"+p);
   S.total=d.total; S.items=S.items.concat(d.items); S.offset+=d.items.length;
   const list=$("#list"); if(!list)return;
@@ -475,8 +492,8 @@ function findChips(f){
   if(!f) return "";
   const p=[];
   (f.codes||[]).slice(0,2).forEach(c=>p.push(`<span class="chip promo" title="промокод/код">code: ${esc(c)}</span>`));
-  if(f.amount) p.push(`<span class="chip credits">${esc(f.amount)}</span>`);
+  if(f.amount) p.push(`<span class="chip price" title="объём/сумма">${esc(f.amount)}</span>`);
-  if(f.term) p.push(`<span class="chip invite">${esc(f.term)}</span>`);
+  if(f.term) p.push(`<span class="chip trial" title="срок">${esc(f.term)}</span>`);
   return p.join("");
 }
 function itemRow(it,compact){
@@ -536,6 +553,7 @@ async function openItem(id){
 function closeDrawer(){$("#drawer").classList.remove("on");$("#backdrop").classList.remove("on");}
 
 /* ---------- abuse ---------- */
+
 V.abuse=async()=>{
   const d=await api("/api/abuse?limit=120");
   const tags=Object.entries(d.tags||{}).slice(0,14);
@@ -547,9 +565,24 @@ V.abuse=async()=>{
      ${tags.map(([t,n])=>`<span class="chip click" data-tag="${esc(t)}">${esc(d.labels?.[t]||t)} <span class="n">${num(n)}</span></span>`).join("")}
    </div></div>
    <div class="card" style="padding:0"><div id="ab-list">${(d.items||[]).map(it=>itemRow(it)).join("")||`<div class="empty">пока пусто</div>`}</div></div>
+   <div id="pn-box"></div>
    <div id="sum-box"></div>`;
   S.abuse=(d.items||[]);
+  loadPanels();
 };
+async function loadPanels(){
+  const box=$("#pn-box"); if(!box) return;
+  try{
+    const d=await api("/api/panels");
+    const all=(d.panels||[]).filter(p=>(p.free_count||p.free||0)>0);
+    const rows=all.slice(0,40);
+    if(!rows.length){ box.innerHTML=""; return; }
+    box.innerHTML=`<div class="card" style="margin-top:14px"><h3>Free-панели new-api <span class="r">всего ${num(d.total)} · с бесплатными ${num(all.length)}${all.length>rows.length?` · показано ${num(rows.length)}`:""}</span></h3>
+      <div class="pn-grid">${rows.map(p=>`<a class="pn" href="https://${esc(p.host||p.url||"")}" target="_blank" rel="noopener">
+        <b>${esc(p.host||p.url||"?")}</b><span class="mono">${num(p.free_count||p.free||0)} free</span>
+        <span class="muted">${num(p.models||p.model_count||0)} моделей</span></a>`).join("")}</div></div>`;
+  }catch(e){ box.innerHTML=""; }
+}
 async function abuseByTag(tag){
   const d=await api("/api/abuse?limit=60"+(tag?"&tag="+encodeURIComponent(tag):""));
   const box=$("#ab-list"); if(box) box.innerHTML=(d.items||[]).map(it=>itemRow(it)).join("")||`<div class="empty">пусто</div>`;
@@ -644,31 +677,36 @@ async function srcDetail(name){
 /* ---------- runs ---------- */
 V.runs=async()=>{
   const d=await api("/api/runs?limit=200");
-  const ok=(d.items||[]).filter(r=>r.status==="ok").length;
+  const bs=d.by_status||{};
+  const items=d.items||[];
+  const ok=bs.ok??items.filter(r=>isOkSt(r.status)).length;      // "ok", "ok-304" — успех
+  const empty=bs.empty??items.filter(r=>stKey(r.status)==="empty").length;
+  const err=bs.err??items.filter(r=>stKey(r.status)==="err").length;
+  const none=bs.none??items.filter(r=>stKey(r.status)==="none").length;
+  const rf=(k,l,n)=>`<span class="chip click ${S.runF===k?"on":""}" data-rf="${k}">${l} <span class="n">${num(n)}</span></span>`;
   $("#view").innerHTML=`
    <div class="page-h"><div><h2>Прогоны</h2><p>Журнал сборов: что запросили, сколько получили и сколько нового легло в базу.</p></div>
      <div class="right">
-       <span class="chip click on" data-rf="all">все <span class="n">${num(d.total)}</span></span>
+       ${rf("all","все",d.total)}${rf("ok","ok",ok)}${empty?rf("empty","пусто",empty):""}${rf("err","ошибки",err)}${none?rf("none","без статуса",none):""}</div></div>
-       <span class="chip click" data-rf="ok">ok <span class="n">${ok}</span></span>
-       <span class="chip click" data-rf="err">ошибки <span class="n">${d.total-ok}</span></span></div></div>
    <div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
-     <input id="r-q" placeholder="фильтр: источник или запрос" style="min-width:280px">
+     <input id="r-q" placeholder="фильтр: источник или запрос" style="min-width:280px" value="${esc(S.runQ||"")}">
-     <span class="spacer"></span><span class="dim mono">всего ${num(d.total)}</span></div></div>
+     <span class="spacer"></span><span class="dim mono">всего ${num(d.total)} · показано ${num(items.length)}</span></div></div>
    <div class="card" style="padding:0"><div class="tbl-wrap"><table id="r-tbl"><thead><tr>
      <th>когда</th><th>источник</th><th>запрос</th><th class="right">got</th><th class="right">kept</th><th class="right">байт</th><th>статус</th></tr></thead>
-     <tbody>${(d.items||[]).map(r=>`<tr data-st="${esc(r.status||"")}"><td class="mono dim nowrap">${esc(r.when||"")}</td><td>${esc(r.source||"—")}</td>
+     <tbody>${items.map(r=>`<tr data-st="${stKey(r.status)}"><td class="mono dim nowrap">${esc(r.when||"")}</td><td>${esc(r.source||"—")}</td>
        <td class="dim" style="max-width:380px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(r.query||"")}</td>
        <td class="right mono">${r.got??""}</td><td class="right mono">${r.kept??""}</td><td class="right mono dim">${r.bytes?num(r.bytes):""}</td>
-       <td><span class="pill ${r.status==="ok"?"ok":"no"}"></span>${esc(r.status||"")}</td></tr>`).join("")}</tbody></table></div></div>`;
+       <td><span class="pill ${stPill(r.status)}"></span>${esc(r.status||"—")}</td></tr>`).join("")}</tbody></table></div></div>`;
-  S.runQ=""; S.runF="all";
+  // S.runQ/S.runF не сбрасываем: автообновление перерисовывает раздел каждые 60 с
   const applyRuns=()=>{const v=(S.runQ||"").toLowerCase();
     $$("#r-tbl tbody tr").forEach(tr=>{
       const okT=!v||tr.textContent.toLowerCase().includes(v);
-      const okS=S.runF==="all"||(S.runF==="ok"?tr.dataset.st==="ok":tr.dataset.st!=="ok");
+      const okS=S.runF==="all"||S.runF===tr.dataset.st;
       tr.style.display=(okT&&okS)?"":"none";});};
   $("#r-q").addEventListener("input",debounce(e=>{S.runQ=e.target.value;applyRuns()},120));
   $$("#view [data-rf]").forEach(c=>c.addEventListener("click",()=>{
     $$("#view [data-rf]").forEach(x=>x.classList.remove("on"));c.classList.add("on");S.runF=c.dataset.rf;applyRuns();}));
+  applyRuns();
 };
 
 /* ---------- telegram ---------- */