абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days

tools/probe_fresh.py

@@ -54,7 +54,7 @@ def probe_one(src, limit):
     if not items and not err:
         err = _diag(src)
     return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
-            "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items),
+            "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items), "dynamic": src.extra.get("_dynamic", ""),
             "err": err, "ms": ms,
             "samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
 
@@ -69,11 +69,11 @@ def _norm(u):
 
 def _registry_urls():
     from omni.sources import all_sources
-    out = set()
-    for s in all_sources():
-        out.add(_norm(s.url))
-        out.add(_norm(s.url).split("?")[0])
-    return out
+    return {_norm(s.url) for s in all_sources()}
+
+
+# контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
+CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
 
 
 def emit(rows, dead, path, dedupe=True):
@@ -84,9 +84,9 @@ def emit(rows, dead, path, dedupe=True):
     if dedupe:
         have = _registry_urls()
         kept, skipped = [], []
-        for r in rows:
+        for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
             n = _norm(r["url"])
-            (skipped if (n in have or n.split("?")[0] in have) else kept).append(r)
+            (skipped if n in have else kept).append(r)
         print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
         rows = kept
 
@@ -98,6 +98,16 @@ def emit(rows, dead, path, dedupe=True):
         "Импортов из omni.sources нет намеренно (циклический импорт).",
         '"""',
         "",
+        "import datetime as _dt",
+        "",
+        "",
+        "def _nvd_week():",
+        '    """NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""',
+        "    now = _dt.datetime.utcnow()",
+        '    since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")',
+        '    return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"',
+        '            "&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))',
+        "",
         "",
         "def fresh(S):",
         '    """S — конструктор Source; возвращает только проверенные сетью источники."""',
@@ -111,6 +121,10 @@ def emit(rows, dead, path, dedupe=True):
         lim = "" if limit == 15 else ", limit=%d" % limit
         det = ", detail=True" if r.get("detail") else ""
         tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
+        if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
+            lines.append('        S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
+                q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
+            continue
         lines.append('        S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
             q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
             tags, lim, det, extra))
@@ -147,7 +161,9 @@ def main():
     srcs = []
     for c in cands:
         try:
-            srcs.append(Source(**c))
+            cc = dict(c)
+            cc.pop("dynamic", None)
+            srcs.append(Source(**cc))
         except Exception as e:                  # noqa: BLE001
             print("skip %s: %s" % (c.get("name"), e))