абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days

tools/check_candidates.py

@@ -0,0 +1,79 @@
+"""Проверка кандидатов в источники абузов: сколько items реально отдаёт адаптер."""
+import json
+import sys
+
+sys.path.insert(0, ".")
+
+from omni import feeds
+from types import SimpleNamespace
+
+CANDS = [
+    # TG-превью
+    ("ab_tg_free_apis", "tg_web", "https://t.me/s/free_apis", {}),
+    ("ab_tg_besplatno_chatgpt", "tg_web", "https://t.me/s/besplatno_chatgpt", {}),
+    ("ab_tg_gpt_frees", "tg_web", "https://t.me/s/gpt_frees", {}),
+    ("ab_tg_promokody", "tg_web", "https://t.me/s/promokody", {}),
+    ("ab_tg_promokodq", "tg_web", "https://t.me/s/promokodq", {}),
+    ("ab_tg_free_softiki", "tg_web", "https://t.me/s/free_softiki", {}),
+    ("ab_tg_baza_dannykh", "tg_web", "https://t.me/s/baza_dannykh_ru", {}),
+    ("ab_tg_donzo", "tg_web", "https://t.me/s/virtualnyydondon_aktivatsiadonzo", {}),
+    ("ab_tg_cracksoftiki", "tg_web", "https://t.me/s/cracksoftiki", {}),
+    ("ab_tg_cracksoftwux", "tg_web", "https://t.me/s/crackSoftiowux", {}),
+    ("ab_tg_taynikstore", "tg_web", "https://t.me/s/taynikstoree", {}),
+    ("ab_tg_nodelocrus", "tg_web", "https://t.me/s/NODELOCRUS", {}),
+    ("ab_tg_aivision", "tg_web", "https://t.me/s/a_vision_ai", {}),
+    ("ab_tg_freeapi_chat", "tg_web", "https://t.me/s/freeapi_chat", {}),
+    ("ab_tg_vpn_free", "tg_web", "https://t.me/s/vpn_free_servers", {}),
+    ("ab_tg_sms_free", "tg_web", "https://t.me/s/freenumbers", {}),
+    # reddit
+    ("ab_reddit_freebies2", "reddit_listing", "https://www.reddit.com/r/freebies/.json?limit=25", {}),
+    ("ab_reddit_selfhosted", "reddit_listing", "https://www.reddit.com/r/selfhosted/.json?limit=25", {}),
+    ("ab_reddit_localllama", "reddit_listing", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", {}),
+    ("ab_reddit_chatgptcoding", "reddit_listing", "https://www.reddit.com/r/ChatGPTCoding/.json?limit=25", {}),
+    ("ab_reddit_beermoney", "reddit_listing", "https://www.reddit.com/r/beermoney/.json?limit=25", {}),
+    ("ab_reddit_sideloaded", "reddit_listing", "https://www.reddit.com/r/sideloaded/.json?limit=25", {}),
+    ("ab_reddit_piracy2", "reddit_listing", "https://www.reddit.com/r/Piracy/.json?limit=25", {}),
+    ("ab_reddit_freegame", "reddit_listing", "https://www.reddit.com/r/FreeGameFindings/.json?limit=25", {}),
+    # discourse
+    ("ab_dis_n8n", "discourse", "https://community.n8n.io/latest.json", {}),
+    ("ab_dis_jina", "discourse", "https://discourse.jina.ai/latest.json", {}),
+    ("ab_dis_cloudflare", "discourse", "https://community.cloudflare.com/latest.json", {}),
+    ("ab_dis_vercel", "discourse", "https://community.vercel.com/latest.json", {}),
+    # rss
+    ("ab_rss_slickdeals2", "rss", "https://slickdeals.net/newsearch.php?mode=frontpage&searcharea=deals&searchin=first&rss=1", {}),
+    ("ab_rss_dealnews", "rss", "https://www.dealnews.com/rss/", {}),
+    ("ab_rss_freeStuffTimes", "rss", "https://feeds.feedburner.com/free-stuff-times", {}),
+    ("ab_rss_humble", "rss", "https://www.humblebundle.com/rss", {}),
+    # sitemap
+    ("ab_sitemap_cracked", "sitemap_recent", "https://cracked.io/sitemap.xml", {"days": 30}),
+    ("ab_sitemap_nulled", "sitemap_recent", "https://nulled.to/sitemap.xml", {"days": 30}),
+    # github / hn
+    ("ab_gh_freeapi_keys", "github_search", "https://api.github.com/search/repositories", {"q": "free api keys pool", "sort": "updated"}),
+    ("ab_gh_leak_pool", "github_search", "https://api.github.com/search/repositories", {"q": "leaked api keys checker", "sort": "updated"}),
+    ("ab_hn_free_tokens", "hn_search", "https://hn.algolia.com/api/v1/search_by_date", {"q": "free tokens api"}),
+]
+
+out = []
+for name, adapter, url, extra in CANDS:
+    fn = feeds.ADAPTERS.get(adapter)
+    if not fn:
+        out.append((name, adapter, -1, "нет адаптера"))
+        continue
+    src = SimpleNamespace(name=name, category="abuse", kind="site", url=url,
+                          adapter=adapter, tags=("abuse",), limit=8, detail=False,
+                          extra=dict(extra))
+    try:
+        items = fn(src, 8)
+        n = len(items)
+        sample = (items[0].title or "")[:60] if items else ""
+    except Exception as e:
+        n, sample = -2, "ERR %s" % str(e)[:60]
+    out.append((name, adapter, n, sample))
+
+ok = [x for x in out if x[2] > 0]
+print("проверено:", len(out), "| живых:", len(ok))
+for name, adapter, n, sample in out:
+    print("%-28s %-16s %3s  %s" % (name, adapter, n, sample))
+json.dump([{"name": n, "adapter": a, "url": u, "extra": e, "items": c}
+           for (n, a, c, _), (_, _, u, e) in zip(out, CANDS)], open("data/_cand_checked.json", "w", encoding="utf-8"),
+          ensure_ascii=False, indent=1)

tools/fresh_candidates.py

@@ -7,9 +7,11 @@
 import datetime
 
 
-def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
-    return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
-            "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
+def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None, **kw):
+    row = {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
+           "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
+    row.update(kw)
+    return row
 
 
 _NOW = datetime.datetime.utcnow()
@@ -35,7 +37,8 @@ CANDIDATES = [
     d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
     d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
     d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
-    d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
+    # elastic_security_labs — в реестре уже есть https://www.elastic.co/security-labs/rss/feed.xml
+
     d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
     d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
     d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
@@ -56,9 +59,10 @@ CANDIDATES = [
     d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
 
     # ---------------- 2. CVE / vulnerabilities ----------------
-    d("nvd_recent", "security", "json",
-      "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
-      % (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")),
+    # nvd_recent: окно last-modified считается в момент вызова (см. emit -> dynamic),
+    # статическая дата протухает уже через неделю.
+    d("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0", "nvd",
+      ("cve", "nvd"), dynamic="NVD_LAST_WEEK"),
     d("nvd_exploited_kw", "security", "json",
       "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
       "nvd", ("cve", "exploited")),

tools/probe_fresh.py

@@ -54,7 +54,7 @@ def probe_one(src, limit):
     if not items and not err:
         err = _diag(src)
     return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
-            "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items),
+            "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items), "dynamic": src.extra.get("_dynamic", ""),
             "err": err, "ms": ms,
             "samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
 
@@ -69,11 +69,11 @@ def _norm(u):
 
 def _registry_urls():
     from omni.sources import all_sources
-    out = set()
-    for s in all_sources():
-        out.add(_norm(s.url))
-        out.add(_norm(s.url).split("?")[0])
-    return out
+    return {_norm(s.url) for s in all_sources()}
+
+
+# контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
+CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
 
 
 def emit(rows, dead, path, dedupe=True):
@@ -84,9 +84,9 @@ def emit(rows, dead, path, dedupe=True):
     if dedupe:
         have = _registry_urls()
         kept, skipped = [], []
-        for r in rows:
+        for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
             n = _norm(r["url"])
-            (skipped if (n in have or n.split("?")[0] in have) else kept).append(r)
+            (skipped if n in have else kept).append(r)
         print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
         rows = kept
 
@@ -98,6 +98,16 @@ def emit(rows, dead, path, dedupe=True):
         "Импортов из omni.sources нет намеренно (циклический импорт).",
         '"""',
         "",
+        "import datetime as _dt",
+        "",
+        "",
+        "def _nvd_week():",
+        '    """NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""',
+        "    now = _dt.datetime.utcnow()",
+        '    since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")',
+        '    return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"',
+        '            "&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))',
+        "",
         "",
         "def fresh(S):",
         '    """S — конструктор Source; возвращает только проверенные сетью источники."""',
@@ -111,6 +121,10 @@ def emit(rows, dead, path, dedupe=True):
         lim = "" if limit == 15 else ", limit=%d" % limit
         det = ", detail=True" if r.get("detail") else ""
         tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
+        if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
+            lines.append('        S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
+                q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
+            continue
         lines.append('        S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
             q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
             tags, lim, det, extra))
@@ -147,7 +161,9 @@ def main():
     srcs = []
     for c in cands:
         try:
-            srcs.append(Source(**c))
+            cc = dict(c)
+            cc.pop("dynamic", None)
+            srcs.append(Source(**cc))
         except Exception as e:                  # noqa: BLE001
             print("skip %s: %s" % (c.get("name"), e))
 

tools/run_fresh.py

@@ -0,0 +1,57 @@
+"""Прогон свежей волны источников: фактическая отдача и сколько легло в базу.
+
+python tools/run_fresh.py [--limit 10] [--workers 8]
+"""
+import argparse
+import json
+import pathlib
+import sys
+import time
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+
+from omni import feeds, rank, sources, source_state, store
+from omni.sources_fresh import fresh as _fresh
+
+
+class _Fake:
+    def __init__(self, *a, **k):
+        self.name = a[0]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=10)
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--out", default="data/fresh_run.json")
+    a = ap.parse_args()
+
+    names = {f.name for f in _fresh(_Fake)}
+    picked = [s for s in sources.all_sources() if s.name in names]
+    print("источников свежей волны:", len(picked), "| лимит", a.limit)
+    t0 = time.time()
+    items, report = feeds.fetch_many(picked, limit=a.limit, workers=a.workers)
+    items = rank.rank(items)
+    new, dup = store.add(items)
+    rep = dict(report)
+    ok = {k: v for k, v in rep.items() if v}
+    bad = sorted(k for k, v in rep.items() if not v)
+    by_src = {}
+    for it in items:
+        by_src[it.source] = by_src.get(it.source, 0) + 1
+    # состояние реестра: что реально отдало записи
+    for name, n in rep.items():
+        source_state.record_check(name, 200 if n else 0, n, "ok" if n else "пусто при проверке")
+    out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked),
+           "alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup,
+           "by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad}
+    json.dump(out, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad))
+    print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup)
+    print("топ:", list(out["by_source"].items())[:12])
+    if bad:
+        print("пустые:", ", ".join(bad))
+
+
+if __name__ == "__main__":
+    main()

tools/shot_abuse.py

@@ -0,0 +1,16 @@
+"""Скриншот вкладки «Абузы» панели omni-parse."""
+import sys
+from camoufox.sync_api import Camoufox
+
+OUT = sys.argv[1] if len(sys.argv) > 1 else "data/dash_abuse.png"
+URL = "http://127.0.0.1:8078/#/abuse"
+
+with Camoufox(headless=True, geoip=False, humanize=True) as b:
+    p = b.new_page(viewport={"width": 1400, "height": 1000})
+    p.goto(URL, wait_until="domcontentloaded", timeout=60000)
+    p.wait_for_selector("#view", timeout=60000)
+    p.wait_for_timeout(7000)
+    txt = p.inner_text("#view")[:900]
+    p.screenshot(path=OUT, full_page=False)
+    print(txt)
+    print("--- saved:", OUT)