абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты
This commit is contained in:
Vlad Gothbreach
439
tools/probe_quality_sources.py
Normal file
439
tools/probe_quality_sources.py
Normal file
@@ -0,0 +1,439 @@
|
||||
"""Живой прогон пула quality-кандидатов (официальные release/security фиды).
|
||||
|
||||
python tools/probe_quality_sources.py # широкий пул -> data/quality_sources_pool.json
|
||||
python tools/probe_quality_sources.py --only gh_rel_ # фильтр по имени
|
||||
python tools/probe_quality_sources.py --confirm # сверить SPECS модуля -> data/quality_sources_probe.json
|
||||
|
||||
Что проверяется (acceptance):
|
||||
1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS <item> и Atom <entry>);
|
||||
2. >= MIN_ITEMS записей с непустым http(s)-URL;
|
||||
3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка);
|
||||
4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория
|
||||
(в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в
|
||||
all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть);
|
||||
5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%).
|
||||
Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator
|
||||
в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0;
|
||||
ненулевой blank_url — сигнал регрессии адаптера, а не норма.
|
||||
|
||||
Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from omni import net # noqa: E402
|
||||
from omni.sources import Source # noqa: E402
|
||||
|
||||
OUT = ROOT / "data" / "quality_sources_probe.json"
|
||||
OUT_POOL = ROOT / "data" / "quality_sources_pool.json"
|
||||
MIN_ITEMS = 2
|
||||
TITLE_OVERLAP_REJECT = 0.6
|
||||
ENTRY_OVERLAP_REJECT = 0.5
|
||||
TIMEOUT = 12
|
||||
|
||||
GH = """huggingface/diffusers
|
||||
huggingface/peft
|
||||
huggingface/accelerate
|
||||
huggingface/text-generation-inference
|
||||
huggingface/tokenizers
|
||||
huggingface/candle
|
||||
pytorch/pytorch
|
||||
pytorch/torchao
|
||||
NVIDIA/TensorRT-LLM
|
||||
NVIDIA/nvidia-container-toolkit
|
||||
flashinfer-ai/flashinfer
|
||||
microsoft/onnxruntime
|
||||
microsoft/DeepSpeed
|
||||
ml-explore/mlx
|
||||
ggml-org/whisper.cpp
|
||||
openai/whisper
|
||||
AUTOMATIC1111/stable-diffusion-webui
|
||||
exo-explore/exo
|
||||
vllm-project/vllm-ascend
|
||||
caddyserver/caddy
|
||||
keycloak/keycloak
|
||||
grafana/grafana
|
||||
tailscale/tailscale
|
||||
envoyproxy/envoy
|
||||
hashicorp/vault
|
||||
hashicorp/terraform
|
||||
hashicorp/consul
|
||||
sigstore/cosign
|
||||
aquasecurity/trivy
|
||||
anchore/grype
|
||||
projectdiscovery/nuclei
|
||||
zaproxy/zaproxy
|
||||
moby/moby
|
||||
containerd/containerd
|
||||
opencontainers/runc
|
||||
kubernetes/kubernetes
|
||||
istio/istio
|
||||
argoproj/argo-cd
|
||||
cert-manager/cert-manager
|
||||
traefik/traefik
|
||||
prometheus/prometheus
|
||||
opensearch-project/OpenSearch
|
||||
elastic/elasticsearch
|
||||
minio/minio
|
||||
go-gitea/gitea
|
||||
nextcloud/server
|
||||
portainer/portainer
|
||||
wazuh/wazuh
|
||||
OISF/suricata
|
||||
zeek/zeek
|
||||
fail2ban/fail2ban
|
||||
openssl/openssl
|
||||
openssh/openssh-portable
|
||||
curl/curl
|
||||
spring-projects/spring-boot
|
||||
quarkusio/quarkus
|
||||
jenkinsci/jenkins
|
||||
immich-app/immich
|
||||
netbox-community/netbox
|
||||
open-telemetry/opentelemetry-collector
|
||||
modelcontextprotocol/python-sdk
|
||||
openai/openai-agents-python
|
||||
pydantic/pydantic-ai
|
||||
langchain-ai/langgraph"""
|
||||
|
||||
# имя = gh_rel_<repo> без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers)
|
||||
def _gh_name(repo):
|
||||
return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "")
|
||||
|
||||
|
||||
_AI_TAG = ("release", "github", "ai")
|
||||
_SEC_TAG = ("security", "release", "github")
|
||||
|
||||
POOL = [
|
||||
{"name": _gh_name(r), "category": "release", "kind": "atom",
|
||||
"url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10}
|
||||
for r in GH.split()
|
||||
]
|
||||
|
||||
POOL += [
|
||||
# ---------- официальные security-фиды вендоров/проектов ----------
|
||||
{"name": "chrome_releases", "category": "security", "kind": "atom",
|
||||
"url": "https://chromereleases.googleblog.com/feeds/posts/default",
|
||||
"tags": ("security", "vendor", "chrome"), "limit": 15},
|
||||
{"name": "project_zero", "category": "security", "kind": "atom",
|
||||
"url": "https://googleprojectzero.blogspot.com/feeds/posts/default",
|
||||
"tags": ("security", "research", "google"), "limit": 10},
|
||||
{"name": "ubuntu_usn", "category": "security", "kind": "rss",
|
||||
"url": "https://ubuntu.com/security/notices/rss.xml",
|
||||
"tags": ("security", "vendor", "ubuntu"), "limit": 20},
|
||||
{"name": "amazon_linux_alas", "category": "security", "kind": "rss",
|
||||
"url": "https://alas.aws.amazon.com/alas.rss",
|
||||
"tags": ("security", "vendor", "aws"), "limit": 20},
|
||||
{"name": "msrc_blog", "category": "security", "kind": "rss",
|
||||
"url": "https://msrc.microsoft.com/blog/feed",
|
||||
"tags": ("security", "vendor", "microsoft"), "limit": 15},
|
||||
{"name": "ms_security_blog", "category": "security", "kind": "rss",
|
||||
"url": "https://www.microsoft.com/en-us/security/blog/feed/",
|
||||
"tags": ("security", "vendor", "microsoft"), "limit": 15},
|
||||
{"name": "cisco_talos", "category": "security", "kind": "rss",
|
||||
"url": "https://blog.talosintelligence.com/rss/",
|
||||
"tags": ("security", "research", "cisco"), "limit": 15},
|
||||
{"name": "portswigger_research", "category": "security", "kind": "rss",
|
||||
"url": "https://portswigger.net/research/rss",
|
||||
"tags": ("security", "research", "web"), "limit": 15},
|
||||
{"name": "trail_of_bits", "category": "security", "kind": "rss",
|
||||
"url": "https://blog.trailofbits.com/feed/",
|
||||
"tags": ("security", "research"), "limit": 15},
|
||||
{"name": "cisa_advisories", "category": "security", "kind": "rss",
|
||||
"url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
|
||||
"tags": ("security", "vendor", "cisa"), "limit": 20},
|
||||
{"name": "sans_isc", "category": "security", "kind": "rss",
|
||||
"url": "https://isc.sans.edu/rssfeed.xml",
|
||||
"tags": ("security", "research", "sans"), "limit": 15},
|
||||
{"name": "nodejs_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://nodejs.org/en/feed/blog.xml",
|
||||
"tags": ("release", "vendor", "nodejs"), "limit": 15},
|
||||
{"name": "rust_blog", "category": "release", "kind": "atom",
|
||||
"url": "https://blog.rust-lang.org/feed.xml",
|
||||
"tags": ("release", "vendor", "rust"), "limit": 15},
|
||||
{"name": "go_blog", "category": "release", "kind": "atom",
|
||||
"url": "https://go.dev/blog/feed.atom",
|
||||
"tags": ("release", "vendor", "go"), "limit": 15},
|
||||
{"name": "php_news", "category": "release", "kind": "atom",
|
||||
"url": "https://www.php.net/feed.atom",
|
||||
"tags": ("release", "vendor", "php"), "limit": 15},
|
||||
{"name": "python_insider", "category": "release", "kind": "atom",
|
||||
"url": "https://pythoninsider.blogspot.com/feeds/posts/default",
|
||||
"tags": ("release", "vendor", "python"), "limit": 15},
|
||||
{"name": "gitlab_releases", "category": "release", "kind": "atom",
|
||||
"url": "https://about.gitlab.com/atom.xml",
|
||||
"tags": ("release", "vendor", "gitlab"), "limit": 15},
|
||||
{"name": "cloudflare_security", "category": "security", "kind": "rss",
|
||||
"url": "https://blog.cloudflare.com/tag/security/rss/",
|
||||
"tags": ("security", "vendor", "cloudflare"), "limit": 15},
|
||||
{"name": "aws_security_bulletins", "category": "security", "kind": "rss",
|
||||
"url": "https://aws.amazon.com/security/security-bulletins/rss/feed/",
|
||||
"tags": ("security", "vendor", "aws"), "limit": 20},
|
||||
{"name": "hashicorp_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://www.hashicorp.com/blog/feed.xml",
|
||||
"tags": ("release", "vendor", "hashicorp"), "limit": 15},
|
||||
{"name": "kubernetes_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://kubernetes.io/feed.xml",
|
||||
"tags": ("release", "vendor", "kubernetes"), "limit": 15},
|
||||
{"name": "docker_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://www.docker.com/feed/",
|
||||
"tags": ("release", "vendor", "docker"), "limit": 15},
|
||||
# ---------- технические/AI release-блоги вендоров ----------
|
||||
{"name": "nvidia_dev_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://developer.nvidia.com/blog/feed/",
|
||||
"tags": ("release", "vendor", "nvidia", "ai"), "limit": 15},
|
||||
{"name": "pytorch_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://pytorch.org/feed/",
|
||||
"tags": ("release", "vendor", "pytorch", "ai"), "limit": 15},
|
||||
{"name": "vllm_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://blog.vllm.ai/feed.xml",
|
||||
"tags": ("release", "vendor", "vllm", "ai"), "limit": 15},
|
||||
{"name": "ray_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://www.anyscale.com/blog/rss.xml",
|
||||
"tags": ("release", "vendor", "ray", "ai"), "limit": 15},
|
||||
{"name": "langchain_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://blog.langchain.dev/rss/",
|
||||
"tags": ("release", "vendor", "langchain", "ai"), "limit": 15},
|
||||
{"name": "modal_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://modal.com/blog/feed.xml",
|
||||
"tags": ("release", "vendor", "modal", "ai"), "limit": 15},
|
||||
{"name": "ollama_blog", "category": "release", "kind": "rss",
|
||||
"url": "https://ollama.com/blog/rss.xml",
|
||||
"tags": ("release", "vendor", "ollama", "ai"), "limit": 15},
|
||||
]
|
||||
|
||||
BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden",
|
||||
"404 not found", "not found", "rate limit", "too many requests",
|
||||
"checking your browser", "ddos-guard", "domain not found",
|
||||
"privacy error", "service unavailable", "bad gateway", "error")
|
||||
|
||||
|
||||
def _norm_url(u):
|
||||
u = (u or "").strip().lower().rstrip("/")
|
||||
for pre in ("https://", "http://"):
|
||||
if u.startswith(pre):
|
||||
u = u[len(pre):]
|
||||
return u[4:] if u.startswith("www.") else u
|
||||
|
||||
|
||||
def _valid_url(u):
|
||||
try:
|
||||
p = urllib.parse.urlsplit(u or "")
|
||||
except Exception:
|
||||
return False
|
||||
return p.scheme in ("http", "https") and bool(p.netloc)
|
||||
|
||||
|
||||
def _bad_title(t):
|
||||
low = (t or "").strip().lower()
|
||||
if len(low) < 3:
|
||||
return True
|
||||
return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60
|
||||
|
||||
|
||||
def _parse(text):
|
||||
"""Локальный парсер RSS/Atom -> [(title, link, published)]."""
|
||||
out = []
|
||||
try:
|
||||
root = ET.fromstring(text.encode("utf-8", "ignore"))
|
||||
except Exception:
|
||||
return out
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
items = root.findall(".//item")
|
||||
if items:
|
||||
for e in items:
|
||||
link = (e.findtext("link") or "").strip()
|
||||
if not link:
|
||||
ge = e.find("{http://www.w3.org/2005/Atom}link")
|
||||
if ge is not None:
|
||||
link = (ge.get("href") or "").strip()
|
||||
out.append(((e.findtext("title") or "").strip(), link,
|
||||
(e.findtext("pubDate") or e.findtext("published") or "").strip()))
|
||||
return out
|
||||
for e in root.findall(".//a:entry", ns):
|
||||
link = ""
|
||||
for le in e.findall("a:link", ns):
|
||||
if le.get("rel") in (None, "alternate") and le.get("href"):
|
||||
link = le.get("href").strip()
|
||||
if le.get("rel") == "alternate":
|
||||
break
|
||||
out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link,
|
||||
(e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip()))
|
||||
return out
|
||||
|
||||
|
||||
def _registry():
|
||||
from omni import sources as reg
|
||||
rows = reg.all_sources()
|
||||
names, urls, repos = set(), set(), set()
|
||||
for s in rows:
|
||||
names.add((s.name or "").strip().lower())
|
||||
urls.add(_norm_url(s.url))
|
||||
m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I)
|
||||
if m:
|
||||
repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git")))
|
||||
return rows, names, urls, repos
|
||||
|
||||
|
||||
def _repo_of(url):
|
||||
m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I)
|
||||
return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None
|
||||
|
||||
|
||||
def _probe(spec):
|
||||
t0 = time.time()
|
||||
r = net.get(spec["url"], timeout=TIMEOUT)
|
||||
status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"],
|
||||
"url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"),
|
||||
"limit": spec.get("limit", 10), "http": getattr(r, "status", 0),
|
||||
"mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""),
|
||||
"ms": int((time.time() - t0) * 1000), "err": ""}
|
||||
if getattr(r, "error", ""):
|
||||
status["err"] = str(r.error)[:160]
|
||||
if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
|
||||
status["entries"] = 0
|
||||
status["reject"] = "http_%s" % getattr(r, "status", 0)
|
||||
return status
|
||||
parsed = _parse(r.text)
|
||||
good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)]
|
||||
status["entries"] = len(parsed)
|
||||
status["valid"] = len(good)
|
||||
status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]]
|
||||
if len(good) < MIN_ITEMS:
|
||||
status["reject"] = "only_%d_valid" % len(good)
|
||||
else:
|
||||
status["reject"] = ""
|
||||
return status
|
||||
|
||||
|
||||
def _fetch_check(spec):
|
||||
"""Что вернул штатный адаптер (документируем эффект бага Atom-link)."""
|
||||
from omni import feeds
|
||||
src = Source(spec["name"], spec["category"], spec["kind"], spec["url"],
|
||||
spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10))
|
||||
t0 = time.time()
|
||||
try:
|
||||
items = feeds.fetch(src, spec.get("limit", 10)) or []
|
||||
err = ""
|
||||
except Exception as e: # noqa: BLE001
|
||||
items, err = [], "raise:%s" % str(e)[:120]
|
||||
blank = sum(1 for i in items if not _valid_url(i.url))
|
||||
return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000),
|
||||
"err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]}
|
||||
for i in items[:3]]}
|
||||
|
||||
|
||||
def _dup_reasons(status, names, urls, repos, accepted):
|
||||
why = []
|
||||
name = (status["name"] or "").lower()
|
||||
if name in names:
|
||||
why.append("name_in_registry")
|
||||
if _norm_url(status["url"]) in urls:
|
||||
why.append("url_in_registry")
|
||||
repo = _repo_of(status["url"])
|
||||
if repo and repo in repos:
|
||||
why.append("repo_mirror:%s/%s" % repo)
|
||||
entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])}
|
||||
titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])}
|
||||
for prev in accepted:
|
||||
p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])}
|
||||
p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])}
|
||||
if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT:
|
||||
why.append("entries_overlap:%s" % prev["name"])
|
||||
break
|
||||
if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT:
|
||||
why.append("titles_overlap:%s" % prev["name"])
|
||||
break
|
||||
return why
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--only", default="")
|
||||
ap.add_argument("--workers", type=int, default=8)
|
||||
ap.add_argument("--limit", type=int, default=10)
|
||||
ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.confirm:
|
||||
from omni.sources_quality import SPECS as pool
|
||||
else:
|
||||
pool = POOL
|
||||
if args.only:
|
||||
pool = [p for p in pool if args.only in p["name"]]
|
||||
|
||||
rows, names, urls, repos = _registry()
|
||||
# после интеграции quality_sources в all_sources() эти же записи есть в реестре:
|
||||
# себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники
|
||||
names -= {(p["name"] or "").lower() for p in pool}
|
||||
urls -= {_norm_url(p["url"]) for p in pool}
|
||||
repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r}
|
||||
t0 = time.time()
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
probed = list(ex.map(_probe, pool))
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
|
||||
accepted, rejected = [], []
|
||||
for st in probed:
|
||||
if st["reject"]:
|
||||
st["dup"] = []
|
||||
rejected.append(st)
|
||||
continue
|
||||
why = _dup_reasons(st, names, urls, repos, accepted)
|
||||
st["dup"] = why
|
||||
(rejected if why else accepted).append(st)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
for st, chk in zip(accepted, ex.map(_fetch_check, accepted)):
|
||||
st["feeds_fetch"] = chk
|
||||
|
||||
report = {
|
||||
"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"mode": "confirm" if args.confirm else "pool",
|
||||
"pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows),
|
||||
"module": "omni/sources_quality.py" if args.confirm else None,
|
||||
"elapsed_s": elapsed, "min_items": MIN_ITEMS,
|
||||
"acceptance": {
|
||||
"min_valid_entries": MIN_ITEMS,
|
||||
"valid_url_required": "http(s) с непустым netloc",
|
||||
"bad_title_marks": list(BAD_TITLE_MARKS),
|
||||
"dup_keys": ["name", "normalized_url", "github_repo_mirror",
|
||||
"entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT,
|
||||
"titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT],
|
||||
},
|
||||
"accepted_n": len(accepted), "rejected_n": len(rejected),
|
||||
"accepted_by_category": {},
|
||||
"accepted": accepted, "rejected": rejected,
|
||||
}
|
||||
for st in accepted:
|
||||
report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1
|
||||
|
||||
target = OUT if args.confirm else OUT_POOL
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed))
|
||||
for st in accepted:
|
||||
s = st["samples"][0] if st["samples"] else {"title": "", "url": ""}
|
||||
ff = st.get("feeds_fetch") or {}
|
||||
print("OK %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s"
|
||||
% (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0),
|
||||
ff.get("blank_url", 0), s["title"][:60], s["url"][:80]))
|
||||
for st in rejected:
|
||||
print("NO %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []),
|
||||
(st.get("samples") or [{}])[0].get("title", "")[:50]))
|
||||
print("report:", target)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user