абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты

tools/fresh_candidates.py

@@ -4,7 +4,6 @@
 переезжают в omni/sources_fresh.py (генерируется `--emit`).
 Формат dict совпадает с Source(**d); адаптеры — только из feeds.ADAPTERS.
 """
-import datetime
 
 
 def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None, **kw):
@@ -14,10 +13,6 @@ def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False,
     return row
 
 
-_NOW = datetime.datetime.utcnow()
-_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
-_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
-
 CANDIDATES = [
     # ---------------- 1. security news / research ----------------
     d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),

tools/probe_fresh.py

@@ -7,8 +7,19 @@
 Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись.
 Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина
 в отчёте была фактической, а не «0 items».
+
+Публичные хелперы (для тестов/внешних вызовов):
+  candidate_source(c) -> (Source, marker)  — dict кандидата -> Source, url
+      динамического источника пересчитан ДО сети; маркер лежит в extra["_dynamic"].
+  _registry_sources() -> [Source]          — all_sources(include_fresh=False)
+  _registry_keys() -> (set url, set name)  — нормализованные ключи реестра без
+      текущей sources_fresh (иначе перегенерация считает свою волну дублем).
+  _registry_urls() -> set url              — совместимость.
+  emit(rows, dead, path, dedupe=True)      — только известный dynamic-маркер;
+      неизвестный маркер = ValueError, а не замороженный протухший url.
 """
 import argparse
+import datetime as _dt
 import json
 import sys
 import time
@@ -60,34 +71,130 @@ def probe_one(src, limit):
 
 
 def _norm(u):
-    u = u.strip().lower().rstrip("/")
+    u = str(u or "").strip().lower().rstrip("/")
     for pre in ("https://", "http://"):
         if u.startswith(pre):
             u = u[len(pre):]
     return u[4:] if u.startswith("www.") else u
 
 
-def _registry_urls():
+def _norm_name(n):
+    """Имя источника — первичный ключ source_flags/runs: сравниваем без регистра,
+    пробелов и разделителей, иначе nvd-recent и nvd_recent разъезжаются."""
+    return "".join(ch for ch in str(n or "").strip().lower() if ch.isalnum())
+
+
+def _registry_sources():
+    """Реестр без текущего sources_fresh: перегенерация не должна считать
+    собственную прошлую волну «уже существующей» и стирать свои же находки."""
     from omni.sources import all_sources
-    return {_norm(s.url) for s in all_sources()}
+    try:
+        return all_sources(include_fresh=False)
+    except TypeError:                          # реестр без флага — совместимость
+        return all_sources()
+
+
+def _registry_keys():
+    """(нормализованные url, нормализованные имена) реестра без sources_fresh."""
+    rows = _registry_sources()
+    return ({_norm(s.url) for s in rows if s.url},
+            {_norm_name(s.name) for s in rows if s.name})
+
+
+def _registry_urls():
+    """Только url-ключи реестра (совместимость с внешними вызовами)."""
+    return _registry_keys()[0]
+
+
+NVD_BASE = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+KNOWN_DYNAMIC = ("NVD_LAST_WEEK",)
+
+
+def _nvd_week_url():
+    """Фактический url NVD на момент прогона: окно lastMod <=120 дней, база
+    без окна отдаёт самые старые записи страницы и как «свежая» бесполезна."""
+    now = _dt.datetime.utcnow()
+    since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
+    return ("%s?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
+            % (NVD_BASE, since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))
+
+
+def candidate_source(c):
+    """dict кандидата -> (Source, dynamic-маркер), url пересчитан до сети.
+
+    Маркер сохраняется в extra["_dynamic"], чтобы отчёт/emit знали, что url
+    нельзя замораживать. Неизвестный маркер — ValueError: молча подставить
+    статический url значит опубликовать заведомо протухший источник.
+    """
+    row = dict(c)
+    marker = str(row.pop("dynamic", "") or "")
+    extra = dict(row.get("extra") or {})
+    if marker:
+        if marker not in KNOWN_DYNAMIC:
+            raise ValueError("unknown dynamic marker %r" % marker)
+        if marker == "NVD_LAST_WEEK":
+            row["url"] = _nvd_week_url()
+        extra["_dynamic"] = marker
+    row["extra"] = extra
+    return Source(**row), marker
 
 
 # контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
 CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
 
 
+def _extra_lit(extra):
+    """extra-литерал без служебных ключей: _dynamic — метаданные прогона, не источника."""
+    clean = {k: v for k, v in sorted((extra or {}).items()) if k != "_dynamic"}
+    return "" if not clean else ", extra=%r" % (clean,)
+
+
+def _frozen_window(rows):
+    """Строки, где в url уже стоит вычисленное окно (lastMod), но маркера нет:
+    такой отчёт старее фикса dynamic — при emit окно застынет на дате прогона."""
+    return [str(r.get("name")) for r in rows
+            if not r.get("dynamic") and "lastmodstartdate=" in str(r.get("url") or "").lower()]
+
+
 def emit(rows, dead, path, dedupe=True):
-    """omni/sources_fresh.py: только живые, без импортов из omni.sources."""
-    def q(s):
-        return '"%s"' % str(s).replace('"', '\\"')
+    """omni/sources_fresh.py: только живые, без импортов из omni.sources.
+
+    Литералы пишем через repr: json.dumps даёт true/false/null — это не Python.
+    Дедуп сверяет и url, и имя (нормализованные) и добавляет принятые строки в
+    виденное, поэтому партия уникальна и сама по себе. Динамический url
+    поддерживается только для известного маркера: неизвестный — явная ошибка,
+    а не замороженный url, который молча протухнет.
+    """
+    rows = list(rows)
+    unknown = sorted({str(r.get("dynamic")) for r in rows
+                      if r.get("dynamic") and r.get("dynamic") not in KNOWN_DYNAMIC})
+    if unknown:
+        raise ValueError("emit: неизвестный dynamic-маркер %s — замораживать url нельзя"
+                         % ", ".join(map(repr, unknown)))
+    frozen = _frozen_window(rows)
+    if frozen:
+        print("emit: ВНИМАНИЕ, окно в url без dynamic-маркера (%s): отчёт устарел, "
+              "перепрогони probe_fresh" % ", ".join(frozen))
 
     if dedupe:
-        have = _registry_urls()
-        kept, skipped = [], []
-        for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
-            n = _norm(r["url"])
-            (skipped if n in have else kept).append(r)
+        have_url, have_name = _registry_keys()
+        seen_url, seen_name = set(have_url), set(have_name)
+        kept, skipped, dupes = [], [], []
+        for r in rows:
+            n, nm = _norm(r.get("url")), _norm_name(r.get("name"))
+            in_registry = (bool(n) and n in have_url) or (bool(nm) and nm in have_name)
+            if r.get("name") in CONTENT_DUP:
+                in_registry = True
+            if in_registry or (bool(n) and n in seen_url) or (bool(nm) and nm in seen_name):
+                (skipped if in_registry else dupes).append(r)
+                continue
+            if n:
+                seen_url.add(n)
+            if nm:
+                seen_name.add(nm)
+            kept.append(r)
         print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
+        print("emit: дубли внутри партии -> %d: %s" % (len(dupes), ", ".join(r["name"] for r in dupes)))
         rows = kept
 
     lines = [
@@ -114,23 +221,23 @@ def emit(rows, dead, path, dedupe=True):
         "    return [",
     ]
     for r in rows:
-        extra = ""
-        if r.get("extra"):
-            extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True)
-        limit = r.get("limit") or 15
-        lim = "" if limit == 15 else ", limit=%d" % limit
+        marker = str(r.get("dynamic") or "")
+        tags = repr(tuple(r.get("tags") or ()))
+        limit = r.get("limit")
+        lim = ", limit=%d" % limit if isinstance(limit, int) and limit != 20 else ""
         det = ", detail=True" if r.get("detail") else ""
-        tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
-        if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
-            lines.append('        S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
-                q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
+        if marker == "NVD_LAST_WEEK":
+            lines.append("        S(%r, %r, %r, _nvd_week(), %r, %s%s%s%s)," % (
+                r["name"], r.get("category", ""), r.get("kind", ""), r.get("adapter", "rss"),
+                tags, lim, det, _extra_lit(r.get("extra"))))
             continue
-        lines.append('        S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
-            q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
-            tags, lim, det, extra))
+        lines.append("        S(%r, %r, %r, %r, %r, %s%s%s%s)," % (
+            r["name"], r.get("category", ""), r.get("kind", ""), r.get("url", ""), r.get("adapter", "rss"),
+            tags, lim, det, _extra_lit(r.get("extra"))))
     lines += ["    ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"]
     for r in dead:
-        lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90]))
+        lines.append("# %-26s %-44s %s" % (r.get("name", "?"), str(r.get("url") or "")[:44],
+                                          (r.get("err") or "0 items")[:90]))
     text = "\n".join(lines) + "\n"
     path.write_text(text, encoding="utf-8")
     return len(rows)
@@ -153,6 +260,12 @@ def main():
         rows = rep["results"]
         alive = [r for r in rows if r["ok"]]
         dead = [r for r in rows if not r["ok"]]
+        frozen = _frozen_window(alive)
+        if frozen:
+            print("emit-only: отчёт без dynamic-маркера для %s — окно в url застынет на дате прогона; "
+                  "перепрогони probe_fresh (сеть) или правь источники вручную"
+                  % ", ".join(frozen))
+            return 2
         n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
         print("omni/sources_fresh.py: %d источников" % n)
         return 0
@@ -161,11 +274,11 @@ def main():
     srcs = []
     for c in cands:
         try:
-            cc = dict(c)
-            cc.pop("dynamic", None)
-            srcs.append(Source(**cc))
+            src, _marker = candidate_source(c)
         except Exception as e:                  # noqa: BLE001
             print("skip %s: %s" % (c.get("name"), e))
+            continue
+        srcs.append(src)
 
     t0 = time.time()
     with ThreadPoolExecutor(max_workers=a.workers) as ex:

tools/probe_quality_sources.py

@@ -0,0 +1,439 @@
+"""Живой прогон пула quality-кандидатов (официальные release/security фиды).
+
+    python tools/probe_quality_sources.py                  # широкий пул -> data/quality_sources_pool.json
+    python tools/probe_quality_sources.py --only gh_rel_    # фильтр по имени
+    python tools/probe_quality_sources.py --confirm         # сверить SPECS модуля -> data/quality_sources_probe.json
+
+Что проверяется (acceptance):
+  1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS <item> и Atom <entry>);
+  2. >= MIN_ITEMS записей с непустым http(s)-URL;
+  3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка);
+  4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория
+     (в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в
+     all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть);
+  5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%).
+Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator
+в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0;
+ненулевой blank_url — сигнал регрессии адаптера, а не норма.
+
+Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется.
+"""
+import argparse
+import json
+import re
+import sys
+import time
+import urllib.parse
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from omni import net                      # noqa: E402
+from omni.sources import Source           # noqa: E402
+
+OUT = ROOT / "data" / "quality_sources_probe.json"
+OUT_POOL = ROOT / "data" / "quality_sources_pool.json"
+MIN_ITEMS = 2
+TITLE_OVERLAP_REJECT = 0.6
+ENTRY_OVERLAP_REJECT = 0.5
+TIMEOUT = 12
+
+GH = """huggingface/diffusers
+huggingface/peft
+huggingface/accelerate
+huggingface/text-generation-inference
+huggingface/tokenizers
+huggingface/candle
+pytorch/pytorch
+pytorch/torchao
+NVIDIA/TensorRT-LLM
+NVIDIA/nvidia-container-toolkit
+flashinfer-ai/flashinfer
+microsoft/onnxruntime
+microsoft/DeepSpeed
+ml-explore/mlx
+ggml-org/whisper.cpp
+openai/whisper
+AUTOMATIC1111/stable-diffusion-webui
+exo-explore/exo
+vllm-project/vllm-ascend
+caddyserver/caddy
+keycloak/keycloak
+grafana/grafana
+tailscale/tailscale
+envoyproxy/envoy
+hashicorp/vault
+hashicorp/terraform
+hashicorp/consul
+sigstore/cosign
+aquasecurity/trivy
+anchore/grype
+projectdiscovery/nuclei
+zaproxy/zaproxy
+moby/moby
+containerd/containerd
+opencontainers/runc
+kubernetes/kubernetes
+istio/istio
+argoproj/argo-cd
+cert-manager/cert-manager
+traefik/traefik
+prometheus/prometheus
+opensearch-project/OpenSearch
+elastic/elasticsearch
+minio/minio
+go-gitea/gitea
+nextcloud/server
+portainer/portainer
+wazuh/wazuh
+OISF/suricata
+zeek/zeek
+fail2ban/fail2ban
+openssl/openssl
+openssh/openssh-portable
+curl/curl
+spring-projects/spring-boot
+quarkusio/quarkus
+jenkinsci/jenkins
+immich-app/immich
+netbox-community/netbox
+open-telemetry/opentelemetry-collector
+modelcontextprotocol/python-sdk
+openai/openai-agents-python
+pydantic/pydantic-ai
+langchain-ai/langgraph"""
+
+# имя = gh_rel_<repo> без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers)
+def _gh_name(repo):
+    return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "")
+
+
+_AI_TAG = ("release", "github", "ai")
+_SEC_TAG = ("security", "release", "github")
+
+POOL = [
+    {"name": _gh_name(r), "category": "release", "kind": "atom",
+     "url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10}
+    for r in GH.split()
+]
+
+POOL += [
+    # ---------- официальные security-фиды вендоров/проектов ----------
+    {"name": "chrome_releases", "category": "security", "kind": "atom",
+     "url": "https://chromereleases.googleblog.com/feeds/posts/default",
+     "tags": ("security", "vendor", "chrome"), "limit": 15},
+    {"name": "project_zero", "category": "security", "kind": "atom",
+     "url": "https://googleprojectzero.blogspot.com/feeds/posts/default",
+     "tags": ("security", "research", "google"), "limit": 10},
+    {"name": "ubuntu_usn", "category": "security", "kind": "rss",
+     "url": "https://ubuntu.com/security/notices/rss.xml",
+     "tags": ("security", "vendor", "ubuntu"), "limit": 20},
+    {"name": "amazon_linux_alas", "category": "security", "kind": "rss",
+     "url": "https://alas.aws.amazon.com/alas.rss",
+     "tags": ("security", "vendor", "aws"), "limit": 20},
+    {"name": "msrc_blog", "category": "security", "kind": "rss",
+     "url": "https://msrc.microsoft.com/blog/feed",
+     "tags": ("security", "vendor", "microsoft"), "limit": 15},
+    {"name": "ms_security_blog", "category": "security", "kind": "rss",
+     "url": "https://www.microsoft.com/en-us/security/blog/feed/",
+     "tags": ("security", "vendor", "microsoft"), "limit": 15},
+    {"name": "cisco_talos", "category": "security", "kind": "rss",
+     "url": "https://blog.talosintelligence.com/rss/",
+     "tags": ("security", "research", "cisco"), "limit": 15},
+    {"name": "portswigger_research", "category": "security", "kind": "rss",
+     "url": "https://portswigger.net/research/rss",
+     "tags": ("security", "research", "web"), "limit": 15},
+    {"name": "trail_of_bits", "category": "security", "kind": "rss",
+     "url": "https://blog.trailofbits.com/feed/",
+     "tags": ("security", "research"), "limit": 15},
+    {"name": "cisa_advisories", "category": "security", "kind": "rss",
+     "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
+     "tags": ("security", "vendor", "cisa"), "limit": 20},
+    {"name": "sans_isc", "category": "security", "kind": "rss",
+     "url": "https://isc.sans.edu/rssfeed.xml",
+     "tags": ("security", "research", "sans"), "limit": 15},
+    {"name": "nodejs_blog", "category": "release", "kind": "rss",
+     "url": "https://nodejs.org/en/feed/blog.xml",
+     "tags": ("release", "vendor", "nodejs"), "limit": 15},
+    {"name": "rust_blog", "category": "release", "kind": "atom",
+     "url": "https://blog.rust-lang.org/feed.xml",
+     "tags": ("release", "vendor", "rust"), "limit": 15},
+    {"name": "go_blog", "category": "release", "kind": "atom",
+     "url": "https://go.dev/blog/feed.atom",
+     "tags": ("release", "vendor", "go"), "limit": 15},
+    {"name": "php_news", "category": "release", "kind": "atom",
+     "url": "https://www.php.net/feed.atom",
+     "tags": ("release", "vendor", "php"), "limit": 15},
+    {"name": "python_insider", "category": "release", "kind": "atom",
+     "url": "https://pythoninsider.blogspot.com/feeds/posts/default",
+     "tags": ("release", "vendor", "python"), "limit": 15},
+    {"name": "gitlab_releases", "category": "release", "kind": "atom",
+     "url": "https://about.gitlab.com/atom.xml",
+     "tags": ("release", "vendor", "gitlab"), "limit": 15},
+    {"name": "cloudflare_security", "category": "security", "kind": "rss",
+     "url": "https://blog.cloudflare.com/tag/security/rss/",
+     "tags": ("security", "vendor", "cloudflare"), "limit": 15},
+    {"name": "aws_security_bulletins", "category": "security", "kind": "rss",
+     "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
+     "tags": ("security", "vendor", "aws"), "limit": 20},
+    {"name": "hashicorp_blog", "category": "release", "kind": "rss",
+     "url": "https://www.hashicorp.com/blog/feed.xml",
+     "tags": ("release", "vendor", "hashicorp"), "limit": 15},
+    {"name": "kubernetes_blog", "category": "release", "kind": "rss",
+     "url": "https://kubernetes.io/feed.xml",
+     "tags": ("release", "vendor", "kubernetes"), "limit": 15},
+    {"name": "docker_blog", "category": "release", "kind": "rss",
+     "url": "https://www.docker.com/feed/",
+     "tags": ("release", "vendor", "docker"), "limit": 15},
+    # ---------- технические/AI release-блоги вендоров ----------
+    {"name": "nvidia_dev_blog", "category": "release", "kind": "rss",
+     "url": "https://developer.nvidia.com/blog/feed/",
+     "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
+    {"name": "pytorch_blog", "category": "release", "kind": "rss",
+     "url": "https://pytorch.org/feed/",
+     "tags": ("release", "vendor", "pytorch", "ai"), "limit": 15},
+    {"name": "vllm_blog", "category": "release", "kind": "rss",
+     "url": "https://blog.vllm.ai/feed.xml",
+     "tags": ("release", "vendor", "vllm", "ai"), "limit": 15},
+    {"name": "ray_blog", "category": "release", "kind": "rss",
+     "url": "https://www.anyscale.com/blog/rss.xml",
+     "tags": ("release", "vendor", "ray", "ai"), "limit": 15},
+    {"name": "langchain_blog", "category": "release", "kind": "rss",
+     "url": "https://blog.langchain.dev/rss/",
+     "tags": ("release", "vendor", "langchain", "ai"), "limit": 15},
+    {"name": "modal_blog", "category": "release", "kind": "rss",
+     "url": "https://modal.com/blog/feed.xml",
+     "tags": ("release", "vendor", "modal", "ai"), "limit": 15},
+    {"name": "ollama_blog", "category": "release", "kind": "rss",
+     "url": "https://ollama.com/blog/rss.xml",
+     "tags": ("release", "vendor", "ollama", "ai"), "limit": 15},
+]
+
+BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden",
+                   "404 not found", "not found", "rate limit", "too many requests",
+                   "checking your browser", "ddos-guard", "domain not found",
+                   "privacy error", "service unavailable", "bad gateway", "error")
+
+
+def _norm_url(u):
+    u = (u or "").strip().lower().rstrip("/")
+    for pre in ("https://", "http://"):
+        if u.startswith(pre):
+            u = u[len(pre):]
+    return u[4:] if u.startswith("www.") else u
+
+
+def _valid_url(u):
+    try:
+        p = urllib.parse.urlsplit(u or "")
+    except Exception:
+        return False
+    return p.scheme in ("http", "https") and bool(p.netloc)
+
+
+def _bad_title(t):
+    low = (t or "").strip().lower()
+    if len(low) < 3:
+        return True
+    return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60
+
+
+def _parse(text):
+    """Локальный парсер RSS/Atom -> [(title, link, published)]."""
+    out = []
+    try:
+        root = ET.fromstring(text.encode("utf-8", "ignore"))
+    except Exception:
+        return out
+    ns = {"a": "http://www.w3.org/2005/Atom"}
+    items = root.findall(".//item")
+    if items:
+        for e in items:
+            link = (e.findtext("link") or "").strip()
+            if not link:
+                ge = e.find("{http://www.w3.org/2005/Atom}link")
+                if ge is not None:
+                    link = (ge.get("href") or "").strip()
+            out.append(((e.findtext("title") or "").strip(), link,
+                        (e.findtext("pubDate") or e.findtext("published") or "").strip()))
+        return out
+    for e in root.findall(".//a:entry", ns):
+        link = ""
+        for le in e.findall("a:link", ns):
+            if le.get("rel") in (None, "alternate") and le.get("href"):
+                link = le.get("href").strip()
+                if le.get("rel") == "alternate":
+                    break
+        out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link,
+                    (e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip()))
+    return out
+
+
+def _registry():
+    from omni import sources as reg
+    rows = reg.all_sources()
+    names, urls, repos = set(), set(), set()
+    for s in rows:
+        names.add((s.name or "").strip().lower())
+        urls.add(_norm_url(s.url))
+        m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I)
+        if m:
+            repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git")))
+    return rows, names, urls, repos
+
+
+def _repo_of(url):
+    m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I)
+    return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None
+
+
+def _probe(spec):
+    t0 = time.time()
+    r = net.get(spec["url"], timeout=TIMEOUT)
+    status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"],
+              "url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"),
+              "limit": spec.get("limit", 10), "http": getattr(r, "status", 0),
+              "mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""),
+              "ms": int((time.time() - t0) * 1000), "err": ""}
+    if getattr(r, "error", ""):
+        status["err"] = str(r.error)[:160]
+    if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
+        status["entries"] = 0
+        status["reject"] = "http_%s" % getattr(r, "status", 0)
+        return status
+    parsed = _parse(r.text)
+    good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)]
+    status["entries"] = len(parsed)
+    status["valid"] = len(good)
+    status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]]
+    if len(good) < MIN_ITEMS:
+        status["reject"] = "only_%d_valid" % len(good)
+    else:
+        status["reject"] = ""
+    return status
+
+
+def _fetch_check(spec):
+    """Что вернул штатный адаптер (документируем эффект бага Atom-link)."""
+    from omni import feeds
+    src = Source(spec["name"], spec["category"], spec["kind"], spec["url"],
+                 spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10))
+    t0 = time.time()
+    try:
+        items = feeds.fetch(src, spec.get("limit", 10)) or []
+        err = ""
+    except Exception as e:                      # noqa: BLE001
+        items, err = [], "raise:%s" % str(e)[:120]
+    blank = sum(1 for i in items if not _valid_url(i.url))
+    return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000),
+            "err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]}
+                                    for i in items[:3]]}
+
+
+def _dup_reasons(status, names, urls, repos, accepted):
+    why = []
+    name = (status["name"] or "").lower()
+    if name in names:
+        why.append("name_in_registry")
+    if _norm_url(status["url"]) in urls:
+        why.append("url_in_registry")
+    repo = _repo_of(status["url"])
+    if repo and repo in repos:
+        why.append("repo_mirror:%s/%s" % repo)
+    entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])}
+    titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])}
+    for prev in accepted:
+        p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])}
+        p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])}
+        if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT:
+            why.append("entries_overlap:%s" % prev["name"])
+            break
+        if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT:
+            why.append("titles_overlap:%s" % prev["name"])
+            break
+    return why
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--only", default="")
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--limit", type=int, default=10)
+    ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality")
+    args = ap.parse_args()
+
+    if args.confirm:
+        from omni.sources_quality import SPECS as pool
+    else:
+        pool = POOL
+    if args.only:
+        pool = [p for p in pool if args.only in p["name"]]
+
+    rows, names, urls, repos = _registry()
+    # после интеграции quality_sources в all_sources() эти же записи есть в реестре:
+    # себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники
+    names -= {(p["name"] or "").lower() for p in pool}
+    urls -= {_norm_url(p["url"]) for p in pool}
+    repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r}
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        probed = list(ex.map(_probe, pool))
+    elapsed = round(time.time() - t0, 1)
+
+    accepted, rejected = [], []
+    for st in probed:
+        if st["reject"]:
+            st["dup"] = []
+            rejected.append(st)
+            continue
+        why = _dup_reasons(st, names, urls, repos, accepted)
+        st["dup"] = why
+        (rejected if why else accepted).append(st)
+
+    with ThreadPoolExecutor(max_workers=4) as ex:
+        for st, chk in zip(accepted, ex.map(_fetch_check, accepted)):
+            st["feeds_fetch"] = chk
+
+    report = {
+        "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "mode": "confirm" if args.confirm else "pool",
+        "pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows),
+        "module": "omni/sources_quality.py" if args.confirm else None,
+        "elapsed_s": elapsed, "min_items": MIN_ITEMS,
+        "acceptance": {
+            "min_valid_entries": MIN_ITEMS,
+            "valid_url_required": "http(s) с непустым netloc",
+            "bad_title_marks": list(BAD_TITLE_MARKS),
+            "dup_keys": ["name", "normalized_url", "github_repo_mirror",
+                         "entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT,
+                         "titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT],
+        },
+        "accepted_n": len(accepted), "rejected_n": len(rejected),
+        "accepted_by_category": {},
+        "accepted": accepted, "rejected": rejected,
+    }
+    for st in accepted:
+        report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1
+
+    target = OUT if args.confirm else OUT_POOL
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
+
+    print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed))
+    for st in accepted:
+        s = st["samples"][0] if st["samples"] else {"title": "", "url": ""}
+        ff = st.get("feeds_fetch") or {}
+        print("OK  %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s"
+              % (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0),
+                 ff.get("blank_url", 0), s["title"][:60], s["url"][:80]))
+    for st in rejected:
+        print("NO  %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []),
+                                   (st.get("samples") or [{}])[0].get("title", "")[:50]))
+    print("report:", target)
+
+
+if __name__ == "__main__":
+    main()

tools/run_fresh.py

@@ -23,10 +23,16 @@ def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--limit", type=int, default=10)
     ap.add_argument("--workers", type=int, default=8)
-    ap.add_argument("--out", default="data/fresh_run.json")
+    ap.add_argument("--wave", choices=("fresh", "quality"), default="fresh")
+    ap.add_argument("--out", default="")
     a = ap.parse_args()
 
-    names = {f.name for f in _fresh(_Fake)}
+    if a.wave == "quality":
+        from omni.sources_quality import quality_sources
+        names = {f.name for f in quality_sources(_Fake)}
+    else:
+        names = {f.name for f in _fresh(_Fake)}
+    output = a.out or "data/%s_run.json" % a.wave
     picked = [s for s in sources.all_sources() if s.name in names]
     print("источников свежей волны:", len(picked), "| лимит", a.limit)
     t0 = time.time()
@@ -45,7 +51,8 @@ def main():
     out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked),
            "alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup,
            "by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad}
-    json.dump(out, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    with open(output, "w", encoding="utf-8") as f:
+        json.dump(out, f, ensure_ascii=False, indent=1)
     print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad))
     print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup)
     print("топ:", list(out["by_source"].items())[:12])

tools/shot_abuse.py

@@ -9,7 +9,7 @@ with Camoufox(headless=True, geoip=False, humanize=True) as b:
     p = b.new_page(viewport={"width": 1400, "height": 1000})
     p.goto(URL, wait_until="domcontentloaded", timeout=60000)
     p.wait_for_selector("#view", timeout=60000)
-    p.wait_for_timeout(7000)
+    p.wait_for_timeout(14000)
     txt = p.inner_text("#view")[:900]
     p.screenshot(path=OUT, full_page=False)
     print(txt)