199 lines
8.9 KiB
Python
199 lines
8.9 KiB
Python
"""Живой прогон кандидатов из tools/fresh_candidates.py через штатные адаптеры omni.
|
||
|
||
python tools/probe_fresh.py # прогон + data/fresh_probe.json
|
||
python tools/probe_fresh.py --emit # + перегенерация omni/sources_fresh.py
|
||
python tools/probe_fresh.py --only tg_ --workers 4 --limit 5
|
||
|
||
Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись.
|
||
Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина
|
||
в отчёте была фактической, а не «0 items».
|
||
"""
|
||
import argparse
|
||
import json
|
||
import sys
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(ROOT))
|
||
sys.path.insert(0, str(ROOT / "tools"))
|
||
|
||
from omni import feeds, net # noqa: E402
|
||
from omni.sources import Source # noqa: E402
|
||
from fresh_candidates import CANDIDATES # noqa: E402
|
||
|
||
|
||
def _diag(src):
|
||
"""Почему пусто: код ответа, блокировка, размер тела."""
|
||
try:
|
||
r = net.get(src.url, timeout=20)
|
||
bits = ["http %s" % r.status]
|
||
if getattr(r, "blocked", False):
|
||
bits.append("blocked")
|
||
if getattr(r, "error", ""):
|
||
bits.append("err:%s" % str(r.error)[:120])
|
||
bits.append("%db" % len(r.text or ""))
|
||
return ", ".join(bits)
|
||
except Exception as e: # noqa: BLE001
|
||
return "exception:%s" % str(e)[:140]
|
||
|
||
|
||
def probe_one(src, limit):
|
||
t0 = time.time()
|
||
if src.adapter not in feeds.ADAPTERS:
|
||
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
||
"url": src.url, "tags": list(src.tags), "ok": False, "n": 0,
|
||
"err": "no_adapter:%s" % src.adapter, "ms": 0, "samples": []}
|
||
try:
|
||
items = feeds.fetch(src, limit) or []
|
||
err = ""
|
||
except Exception as e: # noqa: BLE001
|
||
items, err = [], "raise:%s" % str(e)[:140]
|
||
ms = int((time.time() - t0) * 1000)
|
||
if not items and not err:
|
||
err = _diag(src)
|
||
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
||
"url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items), "dynamic": src.extra.get("_dynamic", ""),
|
||
"err": err, "ms": ms,
|
||
"samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
|
||
|
||
|
||
def _norm(u):
|
||
u = u.strip().lower().rstrip("/")
|
||
for pre in ("https://", "http://"):
|
||
if u.startswith(pre):
|
||
u = u[len(pre):]
|
||
return u[4:] if u.startswith("www.") else u
|
||
|
||
|
||
def _registry_urls():
|
||
from omni.sources import all_sources
|
||
return {_norm(s.url) for s in all_sources()}
|
||
|
||
|
||
# контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
|
||
CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
|
||
|
||
|
||
def emit(rows, dead, path, dedupe=True):
|
||
"""omni/sources_fresh.py: только живые, без импортов из omni.sources."""
|
||
def q(s):
|
||
return '"%s"' % str(s).replace('"', '\\"')
|
||
|
||
if dedupe:
|
||
have = _registry_urls()
|
||
kept, skipped = [], []
|
||
for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
|
||
n = _norm(r["url"])
|
||
(skipped if n in have else kept).append(r)
|
||
print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
|
||
rows = kept
|
||
|
||
lines = [
|
||
'"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.',
|
||
"",
|
||
"Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).",
|
||
"Отклонённые кандидаты и причины — в хвостовом комментарии.",
|
||
"Импортов из omni.sources нет намеренно (циклический импорт).",
|
||
'"""',
|
||
"",
|
||
"import datetime as _dt",
|
||
"",
|
||
"",
|
||
"def _nvd_week():",
|
||
' """NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""',
|
||
" now = _dt.datetime.utcnow()",
|
||
' since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")',
|
||
' return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"',
|
||
' "&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))',
|
||
"",
|
||
"",
|
||
"def fresh(S):",
|
||
' """S — конструктор Source; возвращает только проверенные сетью источники."""',
|
||
" return [",
|
||
]
|
||
for r in rows:
|
||
extra = ""
|
||
if r.get("extra"):
|
||
extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True)
|
||
limit = r.get("limit") or 15
|
||
lim = "" if limit == 15 else ", limit=%d" % limit
|
||
det = ", detail=True" if r.get("detail") else ""
|
||
tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
|
||
if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
|
||
lines.append(' S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
|
||
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
|
||
continue
|
||
lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
|
||
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
|
||
tags, lim, det, extra))
|
||
lines += [" ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"]
|
||
for r in dead:
|
||
lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90]))
|
||
text = "\n".join(lines) + "\n"
|
||
path.write_text(text, encoding="utf-8")
|
||
return len(rows)
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--limit", type=int, default=5)
|
||
ap.add_argument("--workers", type=int, default=6)
|
||
ap.add_argument("--only", default="")
|
||
ap.add_argument("--out", default="data/fresh_probe.json")
|
||
ap.add_argument("--emit", action="store_true")
|
||
ap.add_argument("--fresh", action="store_true", help="прогонять только то, что уже в sources_fresh.py")
|
||
ap.add_argument("--emit-only", action="store_true", help="перегенерировать sources_fresh.py из готового JSON")
|
||
ap.add_argument("--no-dedupe", action="store_true", help="не отбрасывать источники, уже присутствующие в реестре")
|
||
a = ap.parse_args()
|
||
|
||
if a.emit_only:
|
||
rep = json.loads((ROOT / a.out).read_text(encoding="utf-8"))
|
||
rows = rep["results"]
|
||
alive = [r for r in rows if r["ok"]]
|
||
dead = [r for r in rows if not r["ok"]]
|
||
n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
|
||
print("omni/sources_fresh.py: %d источников" % n)
|
||
return 0
|
||
|
||
cands = [c for c in CANDIDATES if not a.only or a.only in c["name"]]
|
||
srcs = []
|
||
for c in cands:
|
||
try:
|
||
cc = dict(c)
|
||
cc.pop("dynamic", None)
|
||
srcs.append(Source(**cc))
|
||
except Exception as e: # noqa: BLE001
|
||
print("skip %s: %s" % (c.get("name"), e))
|
||
|
||
t0 = time.time()
|
||
with ThreadPoolExecutor(max_workers=a.workers) as ex:
|
||
rows = list(ex.map(lambda s: probe_one(s, a.limit), srcs))
|
||
|
||
alive = [r for r in rows if r["ok"]]
|
||
dead = [r for r in rows if not r["ok"]]
|
||
cats = {}
|
||
for r in alive:
|
||
cats[r["category"]] = cats.get(r["category"], 0) + 1
|
||
out = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "limit": a.limit, "workers": a.workers,
|
||
"total": len(rows), "alive": len(alive), "dead": len(dead), "seconds": round(time.time() - t0, 1),
|
||
"by_category": dict(sorted(cats.items(), key=lambda kv: -kv[1])),
|
||
"results": sorted(rows, key=lambda r: (not r["ok"], r["category"], r["name"]))}
|
||
p = ROOT / a.out
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
print("всего %d | живых %d | мёртвых %d | %.1fs" % (len(rows), len(alive), len(dead), time.time() - t0))
|
||
print("по категориям:", out["by_category"])
|
||
for r in dead:
|
||
print(" dead %-24s %s" % (r["name"], r["err"][:110]))
|
||
|
||
if a.emit:
|
||
n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
|
||
print("omni/sources_fresh.py: %d источников" % n)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main()) |