"""Живой прогон пула quality-кандидатов (официальные release/security фиды).

    python tools/probe_quality_sources.py                  # широкий пул -> data/quality_sources_pool.json
    python tools/probe_quality_sources.py --only gh_rel_    # фильтр по имени
    python tools/probe_quality_sources.py --confirm         # сверить SPECS модуля -> data/quality_sources_probe.json

Что проверяется (acceptance):
  1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS <item> и Atom <entry>);
  2. >= MIN_ITEMS записей с непустым http(s)-URL;
  3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка);
  4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория
     (в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в
     all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть);
  5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%).
Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator
в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0;
ненулевой blank_url — сигнал регрессии адаптера, а не норма.

Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется.
"""
import argparse
import json
import re
import sys
import time
import urllib.parse
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from omni import net                      # noqa: E402
from omni.sources import Source           # noqa: E402

OUT = ROOT / "data" / "quality_sources_probe.json"
OUT_POOL = ROOT / "data" / "quality_sources_pool.json"
MIN_ITEMS = 2
TITLE_OVERLAP_REJECT = 0.6
ENTRY_OVERLAP_REJECT = 0.5
TIMEOUT = 12

GH = """huggingface/diffusers
huggingface/peft
huggingface/accelerate
huggingface/text-generation-inference
huggingface/tokenizers
huggingface/candle
pytorch/pytorch
pytorch/torchao
NVIDIA/TensorRT-LLM
NVIDIA/nvidia-container-toolkit
flashinfer-ai/flashinfer
microsoft/onnxruntime
microsoft/DeepSpeed
ml-explore/mlx
ggml-org/whisper.cpp
openai/whisper
AUTOMATIC1111/stable-diffusion-webui
exo-explore/exo
vllm-project/vllm-ascend
caddyserver/caddy
keycloak/keycloak
grafana/grafana
tailscale/tailscale
envoyproxy/envoy
hashicorp/vault
hashicorp/terraform
hashicorp/consul
sigstore/cosign
aquasecurity/trivy
anchore/grype
projectdiscovery/nuclei
zaproxy/zaproxy
moby/moby
containerd/containerd
opencontainers/runc
kubernetes/kubernetes
istio/istio
argoproj/argo-cd
cert-manager/cert-manager
traefik/traefik
prometheus/prometheus
opensearch-project/OpenSearch
elastic/elasticsearch
minio/minio
go-gitea/gitea
nextcloud/server
portainer/portainer
wazuh/wazuh
OISF/suricata
zeek/zeek
fail2ban/fail2ban
openssl/openssl
openssh/openssh-portable
curl/curl
spring-projects/spring-boot
quarkusio/quarkus
jenkinsci/jenkins
immich-app/immich
netbox-community/netbox
open-telemetry/opentelemetry-collector
modelcontextprotocol/python-sdk
openai/openai-agents-python
pydantic/pydantic-ai
langchain-ai/langgraph"""

# имя = gh_rel_<repo> без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers)
def _gh_name(repo):
    return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "")


_AI_TAG = ("release", "github", "ai")
_SEC_TAG = ("security", "release", "github")

POOL = [
    {"name": _gh_name(r), "category": "release", "kind": "atom",
     "url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10}
    for r in GH.split()
]

POOL += [
    # ---------- официальные security-фиды вендоров/проектов ----------
    {"name": "chrome_releases", "category": "security", "kind": "atom",
     "url": "https://chromereleases.googleblog.com/feeds/posts/default",
     "tags": ("security", "vendor", "chrome"), "limit": 15},
    {"name": "project_zero", "category": "security", "kind": "atom",
     "url": "https://googleprojectzero.blogspot.com/feeds/posts/default",
     "tags": ("security", "research", "google"), "limit": 10},
    {"name": "ubuntu_usn", "category": "security", "kind": "rss",
     "url": "https://ubuntu.com/security/notices/rss.xml",
     "tags": ("security", "vendor", "ubuntu"), "limit": 20},
    {"name": "amazon_linux_alas", "category": "security", "kind": "rss",
     "url": "https://alas.aws.amazon.com/alas.rss",
     "tags": ("security", "vendor", "aws"), "limit": 20},
    {"name": "msrc_blog", "category": "security", "kind": "rss",
     "url": "https://msrc.microsoft.com/blog/feed",
     "tags": ("security", "vendor", "microsoft"), "limit": 15},
    {"name": "ms_security_blog", "category": "security", "kind": "rss",
     "url": "https://www.microsoft.com/en-us/security/blog/feed/",
     "tags": ("security", "vendor", "microsoft"), "limit": 15},
    {"name": "cisco_talos", "category": "security", "kind": "rss",
     "url": "https://blog.talosintelligence.com/rss/",
     "tags": ("security", "research", "cisco"), "limit": 15},
    {"name": "portswigger_research", "category": "security", "kind": "rss",
     "url": "https://portswigger.net/research/rss",
     "tags": ("security", "research", "web"), "limit": 15},
    {"name": "trail_of_bits", "category": "security", "kind": "rss",
     "url": "https://blog.trailofbits.com/feed/",
     "tags": ("security", "research"), "limit": 15},
    {"name": "cisa_advisories", "category": "security", "kind": "rss",
     "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
     "tags": ("security", "vendor", "cisa"), "limit": 20},
    {"name": "sans_isc", "category": "security", "kind": "rss",
     "url": "https://isc.sans.edu/rssfeed.xml",
     "tags": ("security", "research", "sans"), "limit": 15},
    {"name": "nodejs_blog", "category": "release", "kind": "rss",
     "url": "https://nodejs.org/en/feed/blog.xml",
     "tags": ("release", "vendor", "nodejs"), "limit": 15},
    {"name": "rust_blog", "category": "release", "kind": "atom",
     "url": "https://blog.rust-lang.org/feed.xml",
     "tags": ("release", "vendor", "rust"), "limit": 15},
    {"name": "go_blog", "category": "release", "kind": "atom",
     "url": "https://go.dev/blog/feed.atom",
     "tags": ("release", "vendor", "go"), "limit": 15},
    {"name": "php_news", "category": "release", "kind": "atom",
     "url": "https://www.php.net/feed.atom",
     "tags": ("release", "vendor", "php"), "limit": 15},
    {"name": "python_insider", "category": "release", "kind": "atom",
     "url": "https://pythoninsider.blogspot.com/feeds/posts/default",
     "tags": ("release", "vendor", "python"), "limit": 15},
    {"name": "gitlab_releases", "category": "release", "kind": "atom",
     "url": "https://about.gitlab.com/atom.xml",
     "tags": ("release", "vendor", "gitlab"), "limit": 15},
    {"name": "cloudflare_security", "category": "security", "kind": "rss",
     "url": "https://blog.cloudflare.com/tag/security/rss/",
     "tags": ("security", "vendor", "cloudflare"), "limit": 15},
    {"name": "aws_security_bulletins", "category": "security", "kind": "rss",
     "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
     "tags": ("security", "vendor", "aws"), "limit": 20},
    {"name": "hashicorp_blog", "category": "release", "kind": "rss",
     "url": "https://www.hashicorp.com/blog/feed.xml",
     "tags": ("release", "vendor", "hashicorp"), "limit": 15},
    {"name": "kubernetes_blog", "category": "release", "kind": "rss",
     "url": "https://kubernetes.io/feed.xml",
     "tags": ("release", "vendor", "kubernetes"), "limit": 15},
    {"name": "docker_blog", "category": "release", "kind": "rss",
     "url": "https://www.docker.com/feed/",
     "tags": ("release", "vendor", "docker"), "limit": 15},
    # ---------- технические/AI release-блоги вендоров ----------
    {"name": "nvidia_dev_blog", "category": "release", "kind": "rss",
     "url": "https://developer.nvidia.com/blog/feed/",
     "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
    {"name": "pytorch_blog", "category": "release", "kind": "rss",
     "url": "https://pytorch.org/feed/",
     "tags": ("release", "vendor", "pytorch", "ai"), "limit": 15},
    {"name": "vllm_blog", "category": "release", "kind": "rss",
     "url": "https://blog.vllm.ai/feed.xml",
     "tags": ("release", "vendor", "vllm", "ai"), "limit": 15},
    {"name": "ray_blog", "category": "release", "kind": "rss",
     "url": "https://www.anyscale.com/blog/rss.xml",
     "tags": ("release", "vendor", "ray", "ai"), "limit": 15},
    {"name": "langchain_blog", "category": "release", "kind": "rss",
     "url": "https://blog.langchain.dev/rss/",
     "tags": ("release", "vendor", "langchain", "ai"), "limit": 15},
    {"name": "modal_blog", "category": "release", "kind": "rss",
     "url": "https://modal.com/blog/feed.xml",
     "tags": ("release", "vendor", "modal", "ai"), "limit": 15},
    {"name": "ollama_blog", "category": "release", "kind": "rss",
     "url": "https://ollama.com/blog/rss.xml",
     "tags": ("release", "vendor", "ollama", "ai"), "limit": 15},
]

BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden",
                   "404 not found", "not found", "rate limit", "too many requests",
                   "checking your browser", "ddos-guard", "domain not found",
                   "privacy error", "service unavailable", "bad gateway", "error")


def _norm_url(u):
    u = (u or "").strip().lower().rstrip("/")
    for pre in ("https://", "http://"):
        if u.startswith(pre):
            u = u[len(pre):]
    return u[4:] if u.startswith("www.") else u


def _valid_url(u):
    try:
        p = urllib.parse.urlsplit(u or "")
    except Exception:
        return False
    return p.scheme in ("http", "https") and bool(p.netloc)


def _bad_title(t):
    low = (t or "").strip().lower()
    if len(low) < 3:
        return True
    return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60


def _parse(text):
    """Локальный парсер RSS/Atom -> [(title, link, published)]."""
    out = []
    try:
        root = ET.fromstring(text.encode("utf-8", "ignore"))
    except Exception:
        return out
    ns = {"a": "http://www.w3.org/2005/Atom"}
    items = root.findall(".//item")
    if items:
        for e in items:
            link = (e.findtext("link") or "").strip()
            if not link:
                ge = e.find("{http://www.w3.org/2005/Atom}link")
                if ge is not None:
                    link = (ge.get("href") or "").strip()
            out.append(((e.findtext("title") or "").strip(), link,
                        (e.findtext("pubDate") or e.findtext("published") or "").strip()))
        return out
    for e in root.findall(".//a:entry", ns):
        link = ""
        for le in e.findall("a:link", ns):
            if le.get("rel") in (None, "alternate") and le.get("href"):
                link = le.get("href").strip()
                if le.get("rel") == "alternate":
                    break
        out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link,
                    (e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip()))
    return out


def _registry():
    from omni import sources as reg
    rows = reg.all_sources()
    names, urls, repos = set(), set(), set()
    for s in rows:
        names.add((s.name or "").strip().lower())
        urls.add(_norm_url(s.url))
        m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I)
        if m:
            repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git")))
    return rows, names, urls, repos


def _repo_of(url):
    m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I)
    return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None


def _probe(spec):
    t0 = time.time()
    r = net.get(spec["url"], timeout=TIMEOUT)
    status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"],
              "url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"),
              "limit": spec.get("limit", 10), "http": getattr(r, "status", 0),
              "mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""),
              "ms": int((time.time() - t0) * 1000), "err": ""}
    if getattr(r, "error", ""):
        status["err"] = str(r.error)[:160]
    if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
        status["entries"] = 0
        status["reject"] = "http_%s" % getattr(r, "status", 0)
        return status
    parsed = _parse(r.text)
    good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)]
    status["entries"] = len(parsed)
    status["valid"] = len(good)
    status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]]
    if len(good) < MIN_ITEMS:
        status["reject"] = "only_%d_valid" % len(good)
    else:
        status["reject"] = ""
    return status


def _fetch_check(spec):
    """Что вернул штатный адаптер (документируем эффект бага Atom-link)."""
    from omni import feeds
    src = Source(spec["name"], spec["category"], spec["kind"], spec["url"],
                 spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10))
    t0 = time.time()
    try:
        items = feeds.fetch(src, spec.get("limit", 10)) or []
        err = ""
    except Exception as e:                      # noqa: BLE001
        items, err = [], "raise:%s" % str(e)[:120]
    blank = sum(1 for i in items if not _valid_url(i.url))
    return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000),
            "err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]}
                                    for i in items[:3]]}


def _dup_reasons(status, names, urls, repos, accepted):
    why = []
    name = (status["name"] or "").lower()
    if name in names:
        why.append("name_in_registry")
    if _norm_url(status["url"]) in urls:
        why.append("url_in_registry")
    repo = _repo_of(status["url"])
    if repo and repo in repos:
        why.append("repo_mirror:%s/%s" % repo)
    entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])}
    titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])}
    for prev in accepted:
        p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])}
        p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])}
        if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT:
            why.append("entries_overlap:%s" % prev["name"])
            break
        if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT:
            why.append("titles_overlap:%s" % prev["name"])
            break
    return why


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--only", default="")
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--limit", type=int, default=10)
    ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality")
    args = ap.parse_args()

    if args.confirm:
        from omni.sources_quality import SPECS as pool
    else:
        pool = POOL
    if args.only:
        pool = [p for p in pool if args.only in p["name"]]

    rows, names, urls, repos = _registry()
    # после интеграции quality_sources в all_sources() эти же записи есть в реестре:
    # себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники
    names -= {(p["name"] or "").lower() for p in pool}
    urls -= {_norm_url(p["url"]) for p in pool}
    repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r}
    t0 = time.time()
    with ThreadPoolExecutor(max_workers=args.workers) as ex:
        probed = list(ex.map(_probe, pool))
    elapsed = round(time.time() - t0, 1)

    accepted, rejected = [], []
    for st in probed:
        if st["reject"]:
            st["dup"] = []
            rejected.append(st)
            continue
        why = _dup_reasons(st, names, urls, repos, accepted)
        st["dup"] = why
        (rejected if why else accepted).append(st)

    with ThreadPoolExecutor(max_workers=4) as ex:
        for st, chk in zip(accepted, ex.map(_fetch_check, accepted)):
            st["feeds_fetch"] = chk

    report = {
        "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
        "mode": "confirm" if args.confirm else "pool",
        "pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows),
        "module": "omni/sources_quality.py" if args.confirm else None,
        "elapsed_s": elapsed, "min_items": MIN_ITEMS,
        "acceptance": {
            "min_valid_entries": MIN_ITEMS,
            "valid_url_required": "http(s) с непустым netloc",
            "bad_title_marks": list(BAD_TITLE_MARKS),
            "dup_keys": ["name", "normalized_url", "github_repo_mirror",
                         "entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT,
                         "titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT],
        },
        "accepted_n": len(accepted), "rejected_n": len(rejected),
        "accepted_by_category": {},
        "accepted": accepted, "rejected": rejected,
    }
    for st in accepted:
        report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1

    target = OUT if args.confirm else OUT_POOL
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")

    print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed))
    for st in accepted:
        s = st["samples"][0] if st["samples"] else {"title": "", "url": ""}
        ff = st.get("feeds_fetch") or {}
        print("OK  %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s"
              % (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0),
                 ff.get("blank_url", 0), s["title"][:60], s["url"][:80]))
    for st in rejected:
        print("NO  %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []),
                                   (st.get("samples") or [{}])[0].get("title", "")[:50]))
    print("report:", target)


if __name__ == "__main__":
    main()