абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты

omni/abuse.py

@@ -141,6 +141,13 @@ def _qualifies(source, title, text, tags):
         sc -= 26
         if not hits:
             return 0, []
+    src0 = str(source or "")
+    # new-api панели: «N бесплатных моделей (цена 0)» — это готовая абуз-поверхность
+    if src0.startswith("pn_"):
+        m = re.search(r"(\d{1,4})\s*бесплатн\w*\s*модел", blob, re.I)
+        if m:
+            cnt = min(int(m.group(1)), 500)
+            return 40 + min(cnt // 10, 30), ["freeapi"]
     raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
     src = str(source or "")
     known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_"))

omni/dates.py

@@ -0,0 +1,33 @@
+"""Publication dates from RSS/Atom/APIs, normalized to aware UTC datetimes."""
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+import re
+
+
+def parse_published(value):
+    """Accept Unix seconds/milliseconds, ISO 8601 or RFC 2822; invalid -> None.
+
+    Dates without a timezone are interpreted as UTC, not the host's local zone.
+    """
+    if value is None or not str(value).strip():
+        return None
+    text = str(value).strip()
+    try:
+        if re.fullmatch(r"\d{4}", text):
+            return None  # A bare year is not a Unix timestamp.
+        if re.fullmatch(r"\d{8}", text):
+            return datetime.strptime(text, "%Y%m%d").replace(tzinfo=timezone.utc)
+        if re.fullmatch(r"-?\d+(?:\.\d+)?", text):
+            stamp = float(text)
+            if abs(stamp) >= 10**12:
+                stamp /= 1000
+            return datetime.fromtimestamp(stamp, timezone.utc)
+        try:
+            dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
+        except ValueError:
+            dt = parsedate_to_datetime(text)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt.astimezone(timezone.utc)
+    except (ValueError, TypeError, OverflowError, OSError):
+        return None

omni/feeds.py

@@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET
 from bs4 import BeautifulSoup
 
 from . import config, net
+from .dates import parse_published
 from .model import Item
 
 UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
@@ -38,8 +39,9 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
         m = re.search(r"CVE-\d{4}-\d{4,7}", body or "")
         if m:
             t = "%s · %s" % (m.group(0), t[:120])
+    dt = parse_published(published)
     return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
-                author=_txt(author)[:80], published=str(published or ""), source=src.name,
+                author=_txt(author)[:80], published=dt.isoformat(timespec="seconds") if dt else str(published or ""), source=src.name,
                 extra=ex)
 
 
@@ -100,17 +102,26 @@ def _rss(src, limit):
         root = ET.fromstring(r.text.encode("utf-8", "ignore"))
     except Exception:
         return _rss_regex(src, r.text, limit)
-    ns = {"a": "http://www.w3.org/2005/Atom"}
+    ns = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/elements/1.1/"}
     entries = root.findall(".//item") or root.findall(".//a:entry", ns)
     for e in entries[:limit]:
-        g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
+        def g(tag):
+            direct = e.findtext(tag, namespaces=ns) or ""
+            if direct or ":" in tag:
+                return direct
+            return e.findtext("a:" + tag, namespaces=ns) or ""
+
         link = g("link")
         if not link:
-            le = e.find("a:link", ns) or e.find("link")
-            link = le.get("href") if le is not None and le.get("href") else ""
+            links = list(e.findall("a:link", ns)) + list(e.findall("link"))
+            le = next((el for el in links if el.get("rel", "alternate") == "alternate" and el.get("href")), None)
+            link = le.get("href", "") if le is not None else ""
+        if link:
+            link = urllib.parse.urljoin(src.url, link)
         body = g("description") or g("summary") or g("content")
+        author = e.findtext("a:author/a:name", namespaces=ns) or g("author") or g("dc:creator")
         out.append(_item(src, "entry", link, g("title"), body,
-                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
+                         author, g("pubDate") or g("published") or g("updated")))
     if not out and "arxiv.org" in src.url:
         return _arxiv_api(src, limit)
     return out
@@ -910,18 +921,52 @@ ADAPTERS = {
 }
 
 
+_CHALLENGE_TITLE = re.compile(
+    r"(?:just a moment|access denied|403 forbidden|404(?: not found)?|not found|"
+    r"verify (?:that )?you are human|checking your browser|"
+    r"attention required!?(?:\s*\|\s*cloudflare)?|please enable javascript)[.!…\s]*",
+    re.I,
+)
+
+
+def _valid_items(items):
+    """Drop broken records and obvious challenge pages, not short real posts."""
+    out, seen = [], set()
+    for it in items:
+        if not isinstance(it, Item) or any(not isinstance(getattr(it, field, None), str)
+                                          for field in ("url", "title", "text")):
+            continue
+        try:
+            u = urllib.parse.urlsplit(it.url)
+            valid_url = u.scheme in ("http", "https") and bool(u.hostname)
+        except ValueError:
+            valid_url = False
+        if not valid_url or not (it.title.strip() or it.text.strip()):
+            continue
+        if _CHALLENGE_TITLE.fullmatch(it.title.strip()) and (
+                len(it.text.strip()) < 160 or re.search(
+                    r"cloudflare|captcha|verify you are human|checking your browser|just a moment|attention required",
+                    it.text, re.I)):
+            continue
+        if it.url in seen:
+            continue
+        seen.add(it.url)
+        out.append(it)
+    return out
+
+
 def fetch(src, limit=None):
     fn = ADAPTERS.get(src.adapter)
     if not fn:
         return []
     n = limit or src.limit
     try:
-        return fn(src, n) or []
+        return _valid_items(fn(src, n) or [])
     except Exception:
         return []
 
 
-SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
+SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co", "services.nvd.nist.gov")
 
 
 def fetch_many(sources, limit=None, workers=8):
@@ -944,5 +989,6 @@ def fetch_many(sources, limit=None, workers=8):
             got = fetch(s, limit)
         report.append((s.name, len(got)))
         items.extend(got)
-        time.sleep(3.5 if "reddit.com" in s.url else 1.0)
+        delay = 6.5 if "services.nvd.nist.gov" in s.url else (3.5 if "reddit.com" in s.url else 1.0)
+        time.sleep(delay)
     return items, report

omni/panel_api.py

@@ -13,8 +13,8 @@ import urllib.parse
 from . import abuse as abuse_mod
 from . import config, sources, source_state, store, taxonomy
 
-BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
-          "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
+BUCKET = ("CASE WHEN length(published)=10 AND published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' "
+          "THEN published ELSE COALESCE(date(published,'localtime'),date(seen,'unixepoch','localtime')) END")
 
 
 def _clean(v, n=0):
@@ -32,7 +32,7 @@ def _clean(v, n=0):
 
 # ---------- обзор ----------
 
-BUCKET_SEEN = "date(seen,'unixepoch')"
+BUCKET_SEEN = "date(seen,'unixepoch','localtime')"
 
 
 def timeline(days=30, cat=None, mode="pub"):
@@ -40,6 +40,8 @@ def timeline(days=30, cat=None, mode="pub"):
 
     mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
     mode="seen" — строго по времени импорта: «что комбайн принёс за день».
+
+    last24 — скользящие 24 часа по выбранному mode; непарсируемая дата → seen.
     """
     days = 30 if days is None else int(days or 0)
     days = max(1, min(365, days))
@@ -57,6 +59,11 @@ def timeline(days=30, cat=None, mode="pub"):
         for b, n, cve, sev in c.execute(sql, args):
             if b:
                 got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
+        clock = "seen" if mode == "seen" else "COALESCE(CAST(strftime('%s',published) AS REAL),seen)"
+        recent_where = where + (" AND " if where else " WHERE ") + clock + " >= ? AND " + clock + " <= ?"
+        now = time.time()
+        last24 = c.execute("SELECT COUNT(*) FROM items" + recent_where,
+                           args + [now - 86400, now]).fetchone()[0]
     today = _dt.date.today()
     out = []
     for i in range(days - 1, -1, -1):
@@ -65,7 +72,8 @@ def timeline(days=30, cat=None, mode="pub"):
         out.append({"day": d, "n": n, "cve": cve, "sev": sev})
     return {"ok": True, "days": days, "mode": mode, "points": out,
             "total": sum(p["n"] for p in out),
-            "last24": (out[-1] or {}).get("n", 0),
+            "last24": int(last24),
+            "today": out[-1]["n"],
             "max": max([p["n"] for p in out] or [0])}
 
 

omni/rank.py

@@ -2,6 +2,8 @@
 import re
 import time
 
+from .dates import parse_published
+
 WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0,
           "firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8}
 
@@ -24,20 +26,12 @@ KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "resu
 
 
 def _fresh(published):
-    """published: unix-строка или ISO. Возвращает 0..1."""
-    if not published:
-        return 0.3
-    try:
-        if published.isdigit():
-            ts = int(published)
-            if ts > 10**12:
-                ts //= 1000
-        else:
-            ts = time.mktime(time.strptime(published[:19], "%Y-%m-%dT%H:%M:%S"))
-        days = max(0.0, (time.time() - ts) / 86400)
-        return max(0.0, 1.0 - days / 30.0)
-    except Exception:
+    """Unix seconds/milliseconds, ISO or RFC 2822. Return freshness in 0..1."""
+    dt = parse_published(published)
+    if dt is None:
         return 0.3
+    days = max(0.0, (time.time() - dt.timestamp()) / 86400)
+    return max(0.0, 1.0 - days / 30.0)
 
 
 def score(it):

omni/serve.py

@@ -115,6 +115,33 @@ def _names_in_category(cat):
     return [s.name for s in sources.all_sources() if s.category == cat]
 
 
+def _run_state(status):
+    """Класс статуса прогона: ok / empty / none / err.
+
+    "ok", "ok 0/10", "ok-304" — успех (startswith); "пусто"/"empty" — источник ответил пусто,
+    это не ошибка; отсутствующий статус (None/"") — отдельный класс none; остальное — err.
+    """
+    v = " ".join(str(status or "").split()).lower()
+    if not v:
+        return "none"
+    if v.startswith("ok"):
+        return "ok"
+    if v in ("пусто", "empty"):
+        return "empty"
+    return "err"
+
+
+def _since_days(days):
+    """?days=N -> граница since (YYYY-MM-DD) для store.browse2; 0 — без фильтра.
+
+    Единая логика для /api/items и экспортов: «сутки/неделя/месяц» значат одно и то же.
+    """
+    if not days:
+        return None
+    import datetime as _dt
+    return (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
+
+
 def _run_feed(job_id, category, limit, tags):
     picked = sources.pick(category, None, tags)
     _job(job_id, state="running", total=len(picked), done=0)
@@ -211,6 +238,18 @@ class Handler(BaseHTTPRequestHandler):
                 return self._send(404, "no ui", "text/plain")
             return self._send(200, f.read_bytes(), "text/html; charset=utf-8")
 
+        if path == "/api/panels":
+            import json as _json
+            fp = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "panels.json")
+            try:
+                with open(fp, "r", encoding="utf-8") as f:
+                    raw = _json.load(f)
+            except Exception:
+                raw = []
+            rows = raw if isinstance(raw, list) else (raw.get("panels") or raw.get("items") or [])
+            rows = sorted(rows, key=lambda x: -(x.get("free_count") or x.get("free") or 0))
+            return self._json({"ok": True, "total": len(rows), "panels": rows[:200]})
+
         if path == "/api/stats":
             s = store.stats()
             per_source = store.count_by_source()
@@ -264,6 +303,7 @@ class Handler(BaseHTTPRequestHandler):
             import time as _t
             con = store._con() if hasattr(store, "_con") else None
             rows = []
+            by_status = {"ok": 0, "empty": 0, "err": 0, "none": 0}
             try:
                 import sqlite3 as _s3
                 from .config import DB as _DB
@@ -274,12 +314,22 @@ class Handler(BaseHTTPRequestHandler):
                     d = dict(r)
                     ts = d.get("ts") or 0
                     d["when"] = _t.strftime("%Y-%m-%d %H:%M", _t.localtime(ts)) if ts else ""
+                    d["state"] = _run_state(d.get("status"))
+                    d["ok"] = d["state"] == "ok"
                     rows.append(d)
                 total = db.execute("SELECT count(*) FROM runs").fetchone()[0]
+                # статусы считаем по всей таблице: total-ok из среза limit давал ложные «ошибки»
+                sql = ("SELECT COALESCE(SUM(CASE WHEN lower(trim(status)) LIKE 'ok%' THEN 1 ELSE 0 END),0),"
+                       " COALESCE(SUM(CASE WHEN lower(trim(status)) IN ('пусто','empty') THEN 1 ELSE 0 END),0),"
+                       " COALESCE(SUM(CASE WHEN status IS NULL OR trim(status)='' THEN 1 ELSE 0 END),0)"
+                       " FROM runs")
+                n_ok, n_empty, n_none = db.execute(sql).fetchone()
+                by_status = {"ok": int(n_ok or 0), "empty": int(n_empty or 0), "none": int(n_none or 0)}
+                by_status["err"] = max(0, int(total) - by_status["ok"] - by_status["empty"] - by_status["none"])
                 db.close()
             except Exception as e:
                 return self._json({"error": "runs: %s" % e})
-            return self._json({"items": rows, "total": total, "shown": len(rows)})
+            return self._json({"items": rows, "total": total, "shown": len(rows), "by_status": by_status})
 
         if path == "/api/abuse":
             from . import abuse as _ab
@@ -355,13 +405,13 @@ class Handler(BaseHTTPRequestHandler):
             typ = (q.get("type") or ["all"])[0]
             src = (q.get("src") or [""])[0].strip() or None
             days = _intv(q, "days", 0, 0, 3650)
-            since = None
-            if days:
-                import datetime as _dt
-                since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
+            since = _since_days(days)
             rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
                                         sort=sort, priced=priced, typ=typ, since=since, src=src)
+            from . import abuse as _ab
             for _r in rows:
+                # find нужен itemRow/findChips: browse2 сам признаки абуза не считает
+                _r["find"] = _ab.extract(" ".join((_r.get("title") or "", _r.get("text") or "")))
                 _r["title"] = _clean_txt(_r.get("title"), 180)
                 _r["text"] = _clean_txt(_r.get("text"), 260)
                 _r["published"] = _fmt_date(_r.get("published"))
@@ -374,8 +424,11 @@ class Handler(BaseHTTPRequestHandler):
             priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
             sort = (q.get("sort") or ["score"])[0]
             typ = (q.get("type") or ["all"])[0]
-            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=5000, offset=0,
-                                        sort=sort, priced=priced, typ=typ)
+            since = _since_days(_intv(q, "days", 0, 0, 3650))
+            limit = _intv(q, "limit", 1000, 1, 5000)
+            src = (q.get("src") or [""])[0].strip() or None
+            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=0,
+                                        sort=sort, priced=priced, typ=typ, since=since, src=src)
             import io as _io
             buf = _io.StringIO()
             w = _csv.writer(buf, delimiter=";")
@@ -445,12 +498,17 @@ class Handler(BaseHTTPRequestHandler):
 
         if path == "/api/export.json":
             cat = (q.get("category") or [None])[0]
+            priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
+            since = _since_days(_intv(q, "days", 0, 0, 3650))
+            src = (q.get("src") or [""])[0].strip() or None
             rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(),
                                         names=_names_in_category(cat),
-                                        limit=_intv(q, "limit", 500, 1, 5000),
+                                        limit=_intv(q, "limit", 1000, 1, 5000),
                                         sort=(q.get("sort") or ["score"])[0],
-                                        typ=(q.get("type") or ["all"])[0])
-            data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8")
+                                        typ=(q.get("type") or ["all"])[0],
+                                        priced=priced, since=since, src=src)
+            data = json.dumps({"total": total, "count": len(rows), "items": rows},
+                              ensure_ascii=False, indent=1).encode("utf-8")
             self.send_response(200)
             self.send_header("Content-Type", "application/json; charset=utf-8")
             self.send_header("Content-Disposition", 'attachment; filename="omni.json"')

omni/sources.py

@@ -19,6 +19,7 @@ from .sources_tg_abuse import TG_ABUSE
 from .sources_ext4 import EXTRA5
 from .sources_ext5 import EXTRA6
 from .sources_fresh import fresh as _fresh_sources
+from .sources_quality import quality_sources as _quality_sources
 from .sources_x import X_SOURCES
 
 
@@ -298,7 +299,7 @@ def _gen_sources():
     return out
 
 
-def all_sources():
+def all_sources(include_fresh=True):
     """Реестр + пользовательские источники из панели, без дублей по url и по имени.
 
     Дедуп по двум ключам обязателен:
@@ -306,8 +307,10 @@ def all_sources():
       * name — имя источника является первичным ключом в source_flags/runs, поэтому
         зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
         www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
-    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна →
+    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая и quality-волны →
     пользовательские → abuse-radar → panels.json.
+
+    include_fresh=False исключает текущий generated fresh-модуль при регенерации.
     """
     from . import source_state
     extra = []
@@ -318,7 +321,8 @@ def all_sources():
              ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
         extra.append(Source(**d))
     seen_url, seen_name, out = set(), set(), []
-    for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources():
+    fresh = _fresh_sources(S) if include_fresh else []
+    for s in SOURCES + _gen_sources() + fresh + _quality_sources(S) + extra + radar_sources() + panel_sources():
         key = (s.url or "").rstrip("/").lower()
         name = (s.name or "").strip().lower()
         if (key and key in seen_url) or (name and name in seen_name):

omni/sources_ext.py

@@ -228,8 +228,9 @@ EXTRA = [
     # ---------------- security ----------------
     S("cisa_kev", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
       "cisa_kev", ("cve", "kev")),
-    S("nvd_recent", "security", "json",
-      "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&noRejected", "nvd", ("cve",)),
+    # nvd_recent намеренно НЕ здесь: живой nvd_recent с пересчитываемым окном
+    # lastMod отдаёт sources_fresh.py (tools/probe_fresh.py --emit). Статическая
+    # запись без окна здесь побеждала по first-wins и глушила свежую.
     S("github_advisories", "security", "rss", "https://github.com/advisories.atom", "rss", ("cve", "github")),
     S("snyk_blog", "security", "rss", "https://snyk.io/blog/feed/", "rss", ("appsec",)),
     S("trailofbits_blog", "security", "rss", "https://blog.trailofbits.com/feed/", "rss", ("research",)),

omni/sources_fresh.py

@@ -21,58 +21,58 @@ def fresh(S):
     return [
         S("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
         S("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
-        S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach")),
+        S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach",)),
         S("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
         S("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")),
         S("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
-        S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup")),
+        S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
         S("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", ("tg", "ai", "ru")),
         S("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
         S("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing", ("reddit", "blue-team")),
         S("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing", ("reddit", "exploit")),
         S("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25", "reddit_listing", ("reddit", "purple-team")),
         S("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum")),
-        S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec")),
+        S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec",)),
         S("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news")),
         S("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")),
-        S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti")),
-        S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news")),
-        S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research")),
+        S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti",)),
+        S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news",)),
+        S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
         S("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", "rss", ("cve", "browser")),
         S("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
         S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
-        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
-        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
+        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news",)),
+        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news",)),
         S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
         S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
         S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
         S("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")),
-        S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news")),
+        S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news",)),
         S("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")),
-        S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news")),
-        S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research")),
-        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
+        S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news",)),
+        S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research",)),
+        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research",)),
         S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
         S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
         S("nvd_exploited_kw", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", "nvd", ("cve", "exploited")),
         S("nvd_recent", "security", "json", _nvd_week(), "nvd", ("cve", "nvd")),
-        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
+        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
         S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
-        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
+        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
         S("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news")),
-        S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research")),
-        S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research")),
+        S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
+        S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research",)),
         S("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
-        S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti")),
-        S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain")),
-        S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit")),
+        S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
+        S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain",)),
+        S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit",)),
         S("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")),
         S("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")),
         S("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")),
-        S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news")),
+        S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news",)),
         S("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
-        S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware")),
-        S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve")),
+        S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware",)),
+        S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
         S("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
     ]
 

omni/sources_quality.py

@@ -0,0 +1,174 @@
+"""Quality-волна источников: официальные release-фиды проектов и security-адвайзори вендоров.
+
+Зачем отдельный модуль: основной реестр (sources.py) и волны ext/fresh уже покрывают
+топовые репозитории, но провал в (1) runtime/инференсе второго эшелона и (2) security-
+продуктах, где релиз = факт CVE/патча. Здесь только официальные Atom/RSS-фиды
+(github.com/<repo>/releases.atom, linux-дистрибутивные и вендорные advisory-фиды),
+каждая запись проверена живым прогоном.
+
+Модуль статический и НЕ импортирует .sources: sources.py уже импортирует волны,
+обратный импорт дал бы цикл. Дедуп по url/name делает all_sources().
+
+Категории — только существующие в реестре, без новой схемы:
+  models     — релизы AI-runtime/инференса (diffusers, TensorRT-LLM, onnxruntime, ...);
+  community  — SDK/агентные фреймворки (MCP python-sdk, openai-agents, langgraph);
+  security   — продукты, где релиз = патч/CVE, контейнеры и advisory-фиды дистрибутивов;
+  company    — официальные технические блоги вендоров (NVIDIA, HashiCorp).
+Признак «это релиз» остаётся тегом `release` у всех 37 записей, kind=atom/rss.
+
+Проверка (tools/probe_quality_sources.py, отчёт data/quality_sources_probe.json):
+  * каждое имя и URL сверены с полным реестром all_sources() (≈3.5k источников) — ни одного
+    дубля по имени, нормализованному URL и зеркалу репозитория (github.com/<o>/<r>);
+  * первые записи не пересекаются по URL/заголовкам с другими кандидатами (>=50%/60%);
+  * >= 2 записей с валидным http(s)-URL, без challenge/error-заголовков;
+  * штатный адаптер feeds.fetch (rss, Atom поддерживается) отдаёт 10-20 записей.
+
+Осознанно НЕ включено, хотя фиды живые (шум, не качество): pytorch/pytorch и
+pydantic/pydantic-ai (в releases.atom CI-теги viable/strict, clai2-bleeding),
+microsoft/DeepSpeed (nightly-last-green), keycloak отдаёт nightly-теги (оставлен —
+версии рядом), portainer/immich/netbox/nextcloud/gitea/elastic/opensearch/minio
+(низкий сигнал на релиз, не security), php.net (фид отдаёт 1 запись), а также
+зеркала и медиа-перепечатки (Cisco Talos, MSRC blog, PortSwigger, Cloudflare,
+Ubuntu USN, Rust/Go/Node/Python blogs) — они уже в реестре или дублируют его.
+"""
+
+SPECS = [
+    # ---------- AI runtime / inference (официальные релизы) ----------
+    {"name": "gh_rel_diffusers", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/diffusers/releases.atom",
+     "tags": ("release", "github", "ai", "image"), "limit": 10},
+    {"name": "gh_rel_peft", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/peft/releases.atom",
+     "tags": ("release", "github", "ai", "finetune"), "limit": 10},
+    {"name": "gh_rel_accelerate", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/accelerate/releases.atom",
+     "tags": ("release", "github", "ai", "training"), "limit": 10},
+    {"name": "gh_rel_textgenerationinference", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/text-generation-inference/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_tokenizers", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/tokenizers/releases.atom",
+     "tags": ("release", "github", "ai", "nlp"), "limit": 10},
+    {"name": "gh_rel_candle", "category": "models", "kind": "atom",
+     "url": "https://github.com/huggingface/candle/releases.atom",
+     "tags": ("release", "github", "ai", "rust"), "limit": 10},
+    {"name": "gh_rel_tensorrtllm", "category": "models", "kind": "atom",
+     "url": "https://github.com/NVIDIA/TensorRT-LLM/releases.atom",
+     "tags": ("release", "github", "ai", "nvidia", "inference"), "limit": 10},
+    {"name": "gh_rel_nvidiacontainertoolkit", "category": "models", "kind": "atom",
+     "url": "https://github.com/NVIDIA/nvidia-container-toolkit/releases.atom",
+     "tags": ("release", "github", "gpu", "container"), "limit": 10},
+    {"name": "gh_rel_flashinfer", "category": "models", "kind": "atom",
+     "url": "https://github.com/flashinfer-ai/flashinfer/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_onnxruntime", "category": "models", "kind": "atom",
+     "url": "https://github.com/microsoft/onnxruntime/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+    {"name": "gh_rel_mlx", "category": "models", "kind": "atom",
+     "url": "https://github.com/ml-explore/mlx/releases.atom",
+     "tags": ("release", "github", "ai", "apple"), "limit": 10},
+    {"name": "gh_rel_whispercpp", "category": "models", "kind": "atom",
+     "url": "https://github.com/ggml-org/whisper.cpp/releases.atom",
+     "tags": ("release", "github", "ai", "audio"), "limit": 10},
+    {"name": "gh_rel_exo", "category": "models", "kind": "atom",
+     "url": "https://github.com/exo-explore/exo/releases.atom",
+     "tags": ("release", "github", "ai", "local"), "limit": 10},
+    {"name": "gh_rel_vllmascend", "category": "models", "kind": "atom",
+     "url": "https://github.com/vllm-project/vllm-ascend/releases.atom",
+     "tags": ("release", "github", "ai", "inference"), "limit": 10},
+
+    # ---------- AI-агенты / SDK ----------
+    {"name": "gh_rel_mcp_pythonsdk", "category": "community", "kind": "atom",
+     "url": "https://github.com/modelcontextprotocol/python-sdk/releases.atom",
+     "tags": ("release", "github", "ai", "mcp"), "limit": 10},
+    {"name": "gh_rel_openaiagentspython", "category": "community", "kind": "atom",
+     "url": "https://github.com/openai/openai-agents-python/releases.atom",
+     "tags": ("release", "github", "ai", "agent"), "limit": 10},
+    {"name": "gh_rel_langgraph", "category": "community", "kind": "atom",
+     "url": "https://github.com/langchain-ai/langgraph/releases.atom",
+     "tags": ("release", "github", "ai", "agent"), "limit": 10},
+
+    # ---------- security-продукты: релиз = патч/CVE ----------
+    {"name": "gh_rel_caddy", "category": "security", "kind": "atom",
+     "url": "https://github.com/caddyserver/caddy/releases.atom",
+     "tags": ("security", "release", "github", "web"), "limit": 10},
+    {"name": "gh_rel_keycloak", "category": "security", "kind": "atom",
+     "url": "https://github.com/keycloak/keycloak/releases.atom",
+     "tags": ("security", "release", "github", "iam"), "limit": 10},
+    {"name": "gh_rel_grafana", "category": "security", "kind": "atom",
+     "url": "https://github.com/grafana/grafana/releases.atom",
+     "tags": ("security", "release", "github", "monitoring"), "limit": 10},
+    {"name": "gh_rel_tailscale", "category": "security", "kind": "atom",
+     "url": "https://github.com/tailscale/tailscale/releases.atom",
+     "tags": ("security", "release", "github", "vpn"), "limit": 10},
+    {"name": "gh_rel_envoy", "category": "security", "kind": "atom",
+     "url": "https://github.com/envoyproxy/envoy/releases.atom",
+     "tags": ("security", "release", "github", "proxy"), "limit": 10},
+    {"name": "gh_rel_vault", "category": "security", "kind": "atom",
+     "url": "https://github.com/hashicorp/vault/releases.atom",
+     "tags": ("security", "release", "github", "secrets"), "limit": 10},
+    {"name": "gh_rel_cosign", "category": "security", "kind": "atom",
+     "url": "https://github.com/sigstore/cosign/releases.atom",
+     "tags": ("security", "release", "github", "supply-chain"), "limit": 10},
+    {"name": "gh_rel_trivy", "category": "security", "kind": "atom",
+     "url": "https://github.com/aquasecurity/trivy/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_grype", "category": "security", "kind": "atom",
+     "url": "https://github.com/anchore/grype/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_nuclei", "category": "security", "kind": "atom",
+     "url": "https://github.com/projectdiscovery/nuclei/releases.atom",
+     "tags": ("security", "release", "github", "scanner"), "limit": 10},
+    {"name": "gh_rel_zaproxy", "category": "security", "kind": "atom",
+     "url": "https://github.com/zaproxy/zaproxy/releases.atom",
+     "tags": ("security", "release", "github", "web"), "limit": 10},
+    {"name": "gh_rel_openssl", "category": "security", "kind": "atom",
+     "url": "https://github.com/openssl/openssl/releases.atom",
+     "tags": ("security", "release", "github", "crypto"), "limit": 10},
+    {"name": "gh_rel_curl", "category": "security", "kind": "atom",
+     "url": "https://github.com/curl/curl/releases.atom",
+     "tags": ("security", "release", "github", "http"), "limit": 10},
+
+    # ---------- контейнеры/оркестрация: CVE-релевантные релизы ----------
+    {"name": "gh_rel_runc", "category": "security", "kind": "atom",
+     "url": "https://github.com/opencontainers/runc/releases.atom",
+     "tags": ("security", "release", "github", "container"), "limit": 10},
+    {"name": "gh_rel_containerd", "category": "security", "kind": "atom",
+     "url": "https://github.com/containerd/containerd/releases.atom",
+     "tags": ("security", "release", "github", "container"), "limit": 10},
+    {"name": "gh_rel_kubernetes", "category": "security", "kind": "atom",
+     "url": "https://github.com/kubernetes/kubernetes/releases.atom",
+     "tags": ("security", "release", "github", "kubernetes"), "limit": 10},
+
+    # ---------- официальные advisory/technical фиды вендоров ----------
+    {"name": "amazon_linux_alas", "category": "security", "kind": "rss",
+     "url": "https://alas.aws.amazon.com/alas.rss",
+     "tags": ("security", "release", "advisory", "linux", "aws"), "limit": 20},
+    {"name": "aws_security_bulletins", "category": "security", "kind": "rss",
+     "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
+     "tags": ("security", "release", "advisory", "aws"), "limit": 20},
+    {"name": "nvidia_dev_blog", "category": "company", "kind": "rss",
+     "url": "https://developer.nvidia.com/blog/feed/",
+     "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
+    {"name": "hashicorp_blog", "category": "company", "kind": "rss",
+     "url": "https://www.hashicorp.com/blog/feed.xml",
+     "tags": ("release", "vendor", "hashicorp"), "limit": 15},
+]
+
+NAMES = tuple(d["name"] for d in SPECS)
+
+
+def quality_sources(S, specs=None):
+    """Собрать Source-объекты quality-волны через переданный конструктор S.
+
+    S передаётся снаружи (обычно sources.Source), чтобы модуль не импортировал .sources —
+    это дал бы циклический импорт. Вызов позиционный, как в sources_fresh.fresh: так
+    работает и настоящий Source, и лёгкая заглушка `class _Fake: __init__(self, *a, **k)`,
+    которой tools/run_fresh.py --wave quality собирает имена.
+    Теги всегда кортежи; adapter по умолчанию rss — он же разбирает Atom.
+    """
+    out = []
+    for d in (SPECS if specs is None else specs):
+        out.append(S(d["name"], d["category"], d["kind"], d["url"],
+                     d.get("adapter", "rss"), tuple(d.get("tags") or ()), d.get("limit", 10)))
+    return out

omni/store.py

@@ -257,7 +257,7 @@ def stats():
         by_kind = c.execute("SELECT kind, COUNT(*) FROM items GROUP BY kind ORDER BY 2 DESC").fetchall()
         priced = c.execute("SELECT COUNT(*) FROM items WHERE price IS NOT NULL").fetchone()[0]
         runs = c.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
-        ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE status='ok'").fetchone()[0]
+        ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE lower(trim(status)) LIKE 'ok%'").fetchone()[0]
     return {"items": total, "priced": priced, "runs": runs, "runs_ok": ok_runs,
             "by_provider": by_prov, "by_kind": by_kind}