439 lines
19 KiB
Python
439 lines
19 KiB
Python
"""Живой прогон пула quality-кандидатов (официальные release/security фиды).
|
||
|
||
python tools/probe_quality_sources.py # широкий пул -> data/quality_sources_pool.json
|
||
python tools/probe_quality_sources.py --only gh_rel_ # фильтр по имени
|
||
python tools/probe_quality_sources.py --confirm # сверить SPECS модуля -> data/quality_sources_probe.json
|
||
|
||
Что проверяется (acceptance):
|
||
1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS <item> и Atom <entry>);
|
||
2. >= MIN_ITEMS записей с непустым http(s)-URL;
|
||
3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка);
|
||
4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория
|
||
(в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в
|
||
all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть);
|
||
5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%).
|
||
Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator
|
||
в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0;
|
||
ненулевой blank_url — сигнал регрессии адаптера, а не норма.
|
||
|
||
Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
import xml.etree.ElementTree as ET
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from omni import net # noqa: E402
|
||
from omni.sources import Source # noqa: E402
|
||
|
||
OUT = ROOT / "data" / "quality_sources_probe.json"
|
||
OUT_POOL = ROOT / "data" / "quality_sources_pool.json"
|
||
MIN_ITEMS = 2
|
||
TITLE_OVERLAP_REJECT = 0.6
|
||
ENTRY_OVERLAP_REJECT = 0.5
|
||
TIMEOUT = 12
|
||
|
||
GH = """huggingface/diffusers
|
||
huggingface/peft
|
||
huggingface/accelerate
|
||
huggingface/text-generation-inference
|
||
huggingface/tokenizers
|
||
huggingface/candle
|
||
pytorch/pytorch
|
||
pytorch/torchao
|
||
NVIDIA/TensorRT-LLM
|
||
NVIDIA/nvidia-container-toolkit
|
||
flashinfer-ai/flashinfer
|
||
microsoft/onnxruntime
|
||
microsoft/DeepSpeed
|
||
ml-explore/mlx
|
||
ggml-org/whisper.cpp
|
||
openai/whisper
|
||
AUTOMATIC1111/stable-diffusion-webui
|
||
exo-explore/exo
|
||
vllm-project/vllm-ascend
|
||
caddyserver/caddy
|
||
keycloak/keycloak
|
||
grafana/grafana
|
||
tailscale/tailscale
|
||
envoyproxy/envoy
|
||
hashicorp/vault
|
||
hashicorp/terraform
|
||
hashicorp/consul
|
||
sigstore/cosign
|
||
aquasecurity/trivy
|
||
anchore/grype
|
||
projectdiscovery/nuclei
|
||
zaproxy/zaproxy
|
||
moby/moby
|
||
containerd/containerd
|
||
opencontainers/runc
|
||
kubernetes/kubernetes
|
||
istio/istio
|
||
argoproj/argo-cd
|
||
cert-manager/cert-manager
|
||
traefik/traefik
|
||
prometheus/prometheus
|
||
opensearch-project/OpenSearch
|
||
elastic/elasticsearch
|
||
minio/minio
|
||
go-gitea/gitea
|
||
nextcloud/server
|
||
portainer/portainer
|
||
wazuh/wazuh
|
||
OISF/suricata
|
||
zeek/zeek
|
||
fail2ban/fail2ban
|
||
openssl/openssl
|
||
openssh/openssh-portable
|
||
curl/curl
|
||
spring-projects/spring-boot
|
||
quarkusio/quarkus
|
||
jenkinsci/jenkins
|
||
immich-app/immich
|
||
netbox-community/netbox
|
||
open-telemetry/opentelemetry-collector
|
||
modelcontextprotocol/python-sdk
|
||
openai/openai-agents-python
|
||
pydantic/pydantic-ai
|
||
langchain-ai/langgraph"""
|
||
|
||
# имя = gh_rel_<repo> без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers)
|
||
def _gh_name(repo):
|
||
return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "")
|
||
|
||
|
||
_AI_TAG = ("release", "github", "ai")
|
||
_SEC_TAG = ("security", "release", "github")
|
||
|
||
POOL = [
|
||
{"name": _gh_name(r), "category": "release", "kind": "atom",
|
||
"url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10}
|
||
for r in GH.split()
|
||
]
|
||
|
||
POOL += [
|
||
# ---------- официальные security-фиды вендоров/проектов ----------
|
||
{"name": "chrome_releases", "category": "security", "kind": "atom",
|
||
"url": "https://chromereleases.googleblog.com/feeds/posts/default",
|
||
"tags": ("security", "vendor", "chrome"), "limit": 15},
|
||
{"name": "project_zero", "category": "security", "kind": "atom",
|
||
"url": "https://googleprojectzero.blogspot.com/feeds/posts/default",
|
||
"tags": ("security", "research", "google"), "limit": 10},
|
||
{"name": "ubuntu_usn", "category": "security", "kind": "rss",
|
||
"url": "https://ubuntu.com/security/notices/rss.xml",
|
||
"tags": ("security", "vendor", "ubuntu"), "limit": 20},
|
||
{"name": "amazon_linux_alas", "category": "security", "kind": "rss",
|
||
"url": "https://alas.aws.amazon.com/alas.rss",
|
||
"tags": ("security", "vendor", "aws"), "limit": 20},
|
||
{"name": "msrc_blog", "category": "security", "kind": "rss",
|
||
"url": "https://msrc.microsoft.com/blog/feed",
|
||
"tags": ("security", "vendor", "microsoft"), "limit": 15},
|
||
{"name": "ms_security_blog", "category": "security", "kind": "rss",
|
||
"url": "https://www.microsoft.com/en-us/security/blog/feed/",
|
||
"tags": ("security", "vendor", "microsoft"), "limit": 15},
|
||
{"name": "cisco_talos", "category": "security", "kind": "rss",
|
||
"url": "https://blog.talosintelligence.com/rss/",
|
||
"tags": ("security", "research", "cisco"), "limit": 15},
|
||
{"name": "portswigger_research", "category": "security", "kind": "rss",
|
||
"url": "https://portswigger.net/research/rss",
|
||
"tags": ("security", "research", "web"), "limit": 15},
|
||
{"name": "trail_of_bits", "category": "security", "kind": "rss",
|
||
"url": "https://blog.trailofbits.com/feed/",
|
||
"tags": ("security", "research"), "limit": 15},
|
||
{"name": "cisa_advisories", "category": "security", "kind": "rss",
|
||
"url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
|
||
"tags": ("security", "vendor", "cisa"), "limit": 20},
|
||
{"name": "sans_isc", "category": "security", "kind": "rss",
|
||
"url": "https://isc.sans.edu/rssfeed.xml",
|
||
"tags": ("security", "research", "sans"), "limit": 15},
|
||
{"name": "nodejs_blog", "category": "release", "kind": "rss",
|
||
"url": "https://nodejs.org/en/feed/blog.xml",
|
||
"tags": ("release", "vendor", "nodejs"), "limit": 15},
|
||
{"name": "rust_blog", "category": "release", "kind": "atom",
|
||
"url": "https://blog.rust-lang.org/feed.xml",
|
||
"tags": ("release", "vendor", "rust"), "limit": 15},
|
||
{"name": "go_blog", "category": "release", "kind": "atom",
|
||
"url": "https://go.dev/blog/feed.atom",
|
||
"tags": ("release", "vendor", "go"), "limit": 15},
|
||
{"name": "php_news", "category": "release", "kind": "atom",
|
||
"url": "https://www.php.net/feed.atom",
|
||
"tags": ("release", "vendor", "php"), "limit": 15},
|
||
{"name": "python_insider", "category": "release", "kind": "atom",
|
||
"url": "https://pythoninsider.blogspot.com/feeds/posts/default",
|
||
"tags": ("release", "vendor", "python"), "limit": 15},
|
||
{"name": "gitlab_releases", "category": "release", "kind": "atom",
|
||
"url": "https://about.gitlab.com/atom.xml",
|
||
"tags": ("release", "vendor", "gitlab"), "limit": 15},
|
||
{"name": "cloudflare_security", "category": "security", "kind": "rss",
|
||
"url": "https://blog.cloudflare.com/tag/security/rss/",
|
||
"tags": ("security", "vendor", "cloudflare"), "limit": 15},
|
||
{"name": "aws_security_bulletins", "category": "security", "kind": "rss",
|
||
"url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
|
||
"tags": ("security", "vendor", "aws"), "limit": 20},
|
||
{"name": "hashicorp_blog", "category": "release", "kind": "rss",
|
||
"url": "https://www.hashicorp.com/blog/feed.xml",
|
||
"tags": ("release", "vendor", "hashicorp"), "limit": 15},
|
||
{"name": "kubernetes_blog", "category": "release", "kind": "rss",
|
||
"url": "https://kubernetes.io/feed.xml",
|
||
"tags": ("release", "vendor", "kubernetes"), "limit": 15},
|
||
{"name": "docker_blog", "category": "release", "kind": "rss",
|
||
"url": "https://www.docker.com/feed/",
|
||
"tags": ("release", "vendor", "docker"), "limit": 15},
|
||
# ---------- технические/AI release-блоги вендоров ----------
|
||
{"name": "nvidia_dev_blog", "category": "release", "kind": "rss",
|
||
"url": "https://developer.nvidia.com/blog/feed/",
|
||
"tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
|
||
{"name": "pytorch_blog", "category": "release", "kind": "rss",
|
||
"url": "https://pytorch.org/feed/",
|
||
"tags": ("release", "vendor", "pytorch", "ai"), "limit": 15},
|
||
{"name": "vllm_blog", "category": "release", "kind": "rss",
|
||
"url": "https://blog.vllm.ai/feed.xml",
|
||
"tags": ("release", "vendor", "vllm", "ai"), "limit": 15},
|
||
{"name": "ray_blog", "category": "release", "kind": "rss",
|
||
"url": "https://www.anyscale.com/blog/rss.xml",
|
||
"tags": ("release", "vendor", "ray", "ai"), "limit": 15},
|
||
{"name": "langchain_blog", "category": "release", "kind": "rss",
|
||
"url": "https://blog.langchain.dev/rss/",
|
||
"tags": ("release", "vendor", "langchain", "ai"), "limit": 15},
|
||
{"name": "modal_blog", "category": "release", "kind": "rss",
|
||
"url": "https://modal.com/blog/feed.xml",
|
||
"tags": ("release", "vendor", "modal", "ai"), "limit": 15},
|
||
{"name": "ollama_blog", "category": "release", "kind": "rss",
|
||
"url": "https://ollama.com/blog/rss.xml",
|
||
"tags": ("release", "vendor", "ollama", "ai"), "limit": 15},
|
||
]
|
||
|
||
BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden",
|
||
"404 not found", "not found", "rate limit", "too many requests",
|
||
"checking your browser", "ddos-guard", "domain not found",
|
||
"privacy error", "service unavailable", "bad gateway", "error")
|
||
|
||
|
||
def _norm_url(u):
|
||
u = (u or "").strip().lower().rstrip("/")
|
||
for pre in ("https://", "http://"):
|
||
if u.startswith(pre):
|
||
u = u[len(pre):]
|
||
return u[4:] if u.startswith("www.") else u
|
||
|
||
|
||
def _valid_url(u):
|
||
try:
|
||
p = urllib.parse.urlsplit(u or "")
|
||
except Exception:
|
||
return False
|
||
return p.scheme in ("http", "https") and bool(p.netloc)
|
||
|
||
|
||
def _bad_title(t):
|
||
low = (t or "").strip().lower()
|
||
if len(low) < 3:
|
||
return True
|
||
return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60
|
||
|
||
|
||
def _parse(text):
|
||
"""Локальный парсер RSS/Atom -> [(title, link, published)]."""
|
||
out = []
|
||
try:
|
||
root = ET.fromstring(text.encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return out
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
items = root.findall(".//item")
|
||
if items:
|
||
for e in items:
|
||
link = (e.findtext("link") or "").strip()
|
||
if not link:
|
||
ge = e.find("{http://www.w3.org/2005/Atom}link")
|
||
if ge is not None:
|
||
link = (ge.get("href") or "").strip()
|
||
out.append(((e.findtext("title") or "").strip(), link,
|
||
(e.findtext("pubDate") or e.findtext("published") or "").strip()))
|
||
return out
|
||
for e in root.findall(".//a:entry", ns):
|
||
link = ""
|
||
for le in e.findall("a:link", ns):
|
||
if le.get("rel") in (None, "alternate") and le.get("href"):
|
||
link = le.get("href").strip()
|
||
if le.get("rel") == "alternate":
|
||
break
|
||
out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link,
|
||
(e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip()))
|
||
return out
|
||
|
||
|
||
def _registry():
|
||
from omni import sources as reg
|
||
rows = reg.all_sources()
|
||
names, urls, repos = set(), set(), set()
|
||
for s in rows:
|
||
names.add((s.name or "").strip().lower())
|
||
urls.add(_norm_url(s.url))
|
||
m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I)
|
||
if m:
|
||
repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git")))
|
||
return rows, names, urls, repos
|
||
|
||
|
||
def _repo_of(url):
|
||
m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I)
|
||
return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None
|
||
|
||
|
||
def _probe(spec):
|
||
t0 = time.time()
|
||
r = net.get(spec["url"], timeout=TIMEOUT)
|
||
status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"],
|
||
"url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"),
|
||
"limit": spec.get("limit", 10), "http": getattr(r, "status", 0),
|
||
"mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""),
|
||
"ms": int((time.time() - t0) * 1000), "err": ""}
|
||
if getattr(r, "error", ""):
|
||
status["err"] = str(r.error)[:160]
|
||
if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
|
||
status["entries"] = 0
|
||
status["reject"] = "http_%s" % getattr(r, "status", 0)
|
||
return status
|
||
parsed = _parse(r.text)
|
||
good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)]
|
||
status["entries"] = len(parsed)
|
||
status["valid"] = len(good)
|
||
status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]]
|
||
if len(good) < MIN_ITEMS:
|
||
status["reject"] = "only_%d_valid" % len(good)
|
||
else:
|
||
status["reject"] = ""
|
||
return status
|
||
|
||
|
||
def _fetch_check(spec):
|
||
"""Что вернул штатный адаптер (документируем эффект бага Atom-link)."""
|
||
from omni import feeds
|
||
src = Source(spec["name"], spec["category"], spec["kind"], spec["url"],
|
||
spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10))
|
||
t0 = time.time()
|
||
try:
|
||
items = feeds.fetch(src, spec.get("limit", 10)) or []
|
||
err = ""
|
||
except Exception as e: # noqa: BLE001
|
||
items, err = [], "raise:%s" % str(e)[:120]
|
||
blank = sum(1 for i in items if not _valid_url(i.url))
|
||
return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000),
|
||
"err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]}
|
||
for i in items[:3]]}
|
||
|
||
|
||
def _dup_reasons(status, names, urls, repos, accepted):
|
||
why = []
|
||
name = (status["name"] or "").lower()
|
||
if name in names:
|
||
why.append("name_in_registry")
|
||
if _norm_url(status["url"]) in urls:
|
||
why.append("url_in_registry")
|
||
repo = _repo_of(status["url"])
|
||
if repo and repo in repos:
|
||
why.append("repo_mirror:%s/%s" % repo)
|
||
entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])}
|
||
titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])}
|
||
for prev in accepted:
|
||
p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])}
|
||
p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])}
|
||
if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT:
|
||
why.append("entries_overlap:%s" % prev["name"])
|
||
break
|
||
if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT:
|
||
why.append("titles_overlap:%s" % prev["name"])
|
||
break
|
||
return why
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--only", default="")
|
||
ap.add_argument("--workers", type=int, default=8)
|
||
ap.add_argument("--limit", type=int, default=10)
|
||
ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality")
|
||
args = ap.parse_args()
|
||
|
||
if args.confirm:
|
||
from omni.sources_quality import SPECS as pool
|
||
else:
|
||
pool = POOL
|
||
if args.only:
|
||
pool = [p for p in pool if args.only in p["name"]]
|
||
|
||
rows, names, urls, repos = _registry()
|
||
# после интеграции quality_sources в all_sources() эти же записи есть в реестре:
|
||
# себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники
|
||
names -= {(p["name"] or "").lower() for p in pool}
|
||
urls -= {_norm_url(p["url"]) for p in pool}
|
||
repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r}
|
||
t0 = time.time()
|
||
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||
probed = list(ex.map(_probe, pool))
|
||
elapsed = round(time.time() - t0, 1)
|
||
|
||
accepted, rejected = [], []
|
||
for st in probed:
|
||
if st["reject"]:
|
||
st["dup"] = []
|
||
rejected.append(st)
|
||
continue
|
||
why = _dup_reasons(st, names, urls, repos, accepted)
|
||
st["dup"] = why
|
||
(rejected if why else accepted).append(st)
|
||
|
||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||
for st, chk in zip(accepted, ex.map(_fetch_check, accepted)):
|
||
st["feeds_fetch"] = chk
|
||
|
||
report = {
|
||
"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
"mode": "confirm" if args.confirm else "pool",
|
||
"pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows),
|
||
"module": "omni/sources_quality.py" if args.confirm else None,
|
||
"elapsed_s": elapsed, "min_items": MIN_ITEMS,
|
||
"acceptance": {
|
||
"min_valid_entries": MIN_ITEMS,
|
||
"valid_url_required": "http(s) с непустым netloc",
|
||
"bad_title_marks": list(BAD_TITLE_MARKS),
|
||
"dup_keys": ["name", "normalized_url", "github_repo_mirror",
|
||
"entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT,
|
||
"titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT],
|
||
},
|
||
"accepted_n": len(accepted), "rejected_n": len(rejected),
|
||
"accepted_by_category": {},
|
||
"accepted": accepted, "rejected": rejected,
|
||
}
|
||
for st in accepted:
|
||
report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1
|
||
|
||
target = OUT if args.confirm else OUT_POOL
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed))
|
||
for st in accepted:
|
||
s = st["samples"][0] if st["samples"] else {"title": "", "url": ""}
|
||
ff = st.get("feeds_fetch") or {}
|
||
print("OK %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s"
|
||
% (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0),
|
||
ff.get("blank_url", 0), s["title"][:60], s["url"][:80]))
|
||
for st in rejected:
|
||
print("NO %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []),
|
||
(st.get("samples") or [{}])[0].get("title", "")[:50]))
|
||
print("report:", target)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |