abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000

.local/diag5.py

@@ -0,0 +1,89 @@
+import sys, json, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+from bs4 import BeautifulSoup
+
+NEW = [
+    ("helpnetsecurity", "https://www.helpnetsecurity.com/feed/"),
+    ("infosecurity_magazine", "https://www.infosecurity-magazine.com/rss/news/"),
+    ("cybersecuritydive", "https://www.cybersecuritydive.com/feeds/news/"),
+    ("theregister_security", "https://www.theregister.com/security/headlines.atom"),
+    ("arstechnica_security", "https://arstechnica.com/security/feed/"),
+    ("csoonline", "https://www.csoonline.com/feed/"),
+    ("hackerone_blog", "https://www.hackerone.com/blog.rss"),
+    ("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
+    ("semgrep_blog", "https://semgrep.dev/blog/rss.xml"),
+    ("acunetix_blog", "https://www.acunetix.com/blog/feed/"),
+    ("trustwave_spiderlabs", "https://www.trustwave.com/en-us/resources/blogs/spiderlabs-blog/rss.xml"),
+    ("cybereason_blog", "https://www.cybereason.com/blog/rss.xml"),
+    ("sentinelone_blog", "https://www.sentinelone.com/blog/feed/"),
+    ("microsoft_security_blog", "https://www.microsoft.com/en-us/security/blog/feed/"),
+    ("ncsc_uk", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml"),
+    ("sonatype_blog", "https://www.sonatype.com/blog/rss.xml"),
+    ("invicti_blog", "https://www.invicti.com/blog/feed/"),
+    ("govinfosecurity", "https://www.govinfosecurity.com/rss"),
+    ("bankinfosecurity", "https://www.bankinfosecurity.com/rss"),
+    ("healthcareinfosecurity", "https://www.healthcareinfosecurity.com/rss"),
+    ("cve_circl_alt", "https://cve.circl.lu/api/last/40"),
+    ("vulmon_alt", "https://vulmon.com/rss/latest"),
+    ("packetstorm_files", "https://packetstormsecurity.com/files/rss.xml"),
+    ("exploitdb_rss2", "https://www.exploit-db.com/rss.xml"),
+    ("zdi_blog", "https://www.zerodayinitiative.com/blog?format=rss"),
+    ("hackmag_alt", "https://hackmag.com/rss"),
+    ("securitylab_c", "https://www.securitylab.ru/rss/all.xml"),
+    ("securitylab_d", "https://www.securitylab.ru/news/rss.xml"),
+    ("anquanke_rss2", "https://www.anquanke.com/rss/news"),
+    ("anquanke_rss3", "https://www.anquanke.com/rss?lang=zh"),
+    ("freebuf_alt", "https://www.freebuf.com/"),
+    ("sploitus_rss", "https://sploitus.com/rss"),
+    ("cvefeed_alt", "https://cvefeed.io/rssfeed/latest.xml"),
+    ("reddit_blueteamsec", "https://www.reddit.com/r/blueteamsec/.json?limit=25"),
+    ("reddit_malware", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reverseengineering", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_exploitdev", "https://www.reddit.com/r/ExploitDev/.json?limit=25"),
+    ("reddit_threatintel", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+    ("reddit_purpleteamsec", "https://www.reddit.com/r/purpleteamsec/.json?limit=25"),
+    ("serverfault_sec", "https://security.stackexchange.com/feeds/tag?tagnames=vulnerability&sort=newest"),
+]
+
+
+def probe(url, adapter="rss", extra=None, detail=False):
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=adapter,
+                 tags=(), limit=5, detail=detail, extra=extra or {})
+    r = net.get(url, timeout=25)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "st": r.status, "ct": (r.headers or {}).get("content-type", "")[:30],
+            "n": len(items), "first": (items[0].title[:70] if items else "")}
+
+
+def run(t):
+    name, url = t
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    try:
+        d = probe(url, ad)
+    except Exception as e:
+        d = {"u": url, "err": str(e)[:80], "n": 0}
+    d["name"] = name
+    return d
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, NEW):
+        print(json.dumps(r, ensure_ascii=False))
+
+# debug openwall prefix filter
+r = net.get("https://www.openwall.com/lists/oss-security/", timeout=25)
+soup = BeautifulSoup(r.text or "", "lxml")
+paths = []
+for a in soup.find_all("a", href=True)[:40]:
+    paths.append(urllib.parse.urlsplit(urllib.parse.urljoin("https://www.openwall.com", str(a["href"]))).path)
+print(json.dumps({"openwall_sample": paths[:20]}, ensure_ascii=False))
+
+# detail=True variants
+for u, extra in (("https://www.t00ls.com/", {"include_prefixes": ["/articles-", "/Vuls-"]}),
+                 ("https://www.debian.org/security/", {"include_prefixes": ["/tracker/DSA-"]})):
+    d = probe(u, "html_listing", extra, detail=True)
+    d["url"] = u
+    print(json.dumps(d, ensure_ascii=False))

.local/diag6.py

@@ -0,0 +1,69 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
+      "breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
+      "hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
+      "cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
+      "osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]
+
+RSS = [
+    ("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
+    ("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
+    ("industrialcyber", "https://industrialcyber.co/feed/"),
+    ("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
+    ("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
+    ("malwaretech_blog", "https://www.malwaretech.com/feed"),
+    ("scworld", "https://www.scworld.com/rss.xml"),
+    ("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
+    ("securityweek_alt", "https://www.securityweek.com/feed/"),
+    ("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
+    ("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
+    ("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+]
+
+
+def tg(url):
+    r = net.get(url, timeout=25)
+    t = r.text or ""
+    if t.count("tgme_widget_message_wrap") < 3:
+        return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
+    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+def rss(url):
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]
+
+
+def run(j):
+    kind, u = j
+    try:
+        return tg(u) if kind == "tg" else rss(u)
+    except Exception as e:
+        return {"u": u, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, jobs):
+        print(json.dumps(r, ensure_ascii=False))
+
+# openwall: почему фильтр не срабатывает
+src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
+             adapter="html_listing", tags=(), limit=5,
+             extra={"include_prefixes": ["/lists/oss-security/"]})
+print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
+r = net.get(src.url, timeout=25)
+print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/"))

.local/diag7.py

@@ -0,0 +1,43 @@
+import sys, json
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+C = [
+    ("cisa_news", "https://www.cisa.gov/news-events/news/rss.xml", "rss", {}),
+    ("fortinet_blog", "https://www.fortinet.com/blog/rss.xml", "rss", {}),
+    ("mcafee_blogs", "https://www.mcafee.com/blogs/feed/", "rss", {}),
+    ("avast_blog", "https://www.avast.com/en-us/blog/feed", "rss", {}),
+    ("eset_press", "https://www.eset.com/int/about/newsroom/press-releases/feed/", "rss", {}),
+    ("gti_threat_intel", "https://cloud.google.com/blog/topics/threat-intelligence/rss/", "rss", {}),
+    ("checkpoint_blog", "https://www.checkpoint.com/blog/feed/", "rss", {}),
+    ("anquanke_news", "https://www.anquanke.com/news", "html_listing", {"include_prefixes": ["/post/id/"]}),
+    ("secrss_vuln", "https://www.secrss.com/articles", "html_listing", {"include_prefixes": ["/articles/"]}),
+    ("bleeping_dup_check", "https://www.bleepingcomputer.com/feed/", "rss", {}),
+    ("therecord_dup_check", "https://therecord.media/feed", "rss", {}),
+    ("deepwatch_blog", "https://www.deepwatch.com/feed/", "rss", {}),
+    ("huntress_alt", "https://www.huntress.com/blog/feed", "rss", {}),
+    ("dragos_alt", "https://www.dragos.com/feed/", "rss", {}),
+    ("scmedia_alt", "https://www.scmagazine.com/feed", "rss", {}),
+    ("trendmicro_alt", "https://www.trendmicro.com/en_us/research/feed", "rss", {}),
+    ("zscaler_alt", "https://www.zscaler.com/blogs/security-research/rss", "rss", {}),
+    ("gbhackers_alt", "https://gbhackers.com/feed", "rss", {}),
+    ("cybersecnews_alt", "https://cybersecuritynews.com/feed", "rss", {}),
+]
+
+
+def run(t):
+    name, url, ad, extra = t
+    src = Source(name=name, category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5, extra=extra)
+    try:
+        r = net.get(url, timeout=25)
+        items = feeds.fetch(src, 5)
+        return {"name": name, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
+    except Exception as e:
+        return {"name": name, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, C):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag8.py

@@ -0,0 +1,79 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+RSS = [
+    ("itsecurityguru", "https://www.itsecurityguru.co.uk/feed/"),
+    ("securityledger", "https://securityledger.com/feed/"),
+    ("zdnet_security", "https://www.zdnet.com/topic/security/rss.xml"),
+    ("grahamcluley", "https://www.grahamcluley.com/feed/"),
+    ("cyberdefensemagazine", "https://www.cyberdefensemagazine.com/feed/"),
+    ("hackersonlineclub", "https://www.hackersonlineclub.com/feed"),
+    ("netspi", "https://www.netspi.com/feed/"),
+    ("hackingarticles", "https://www.hackingarticles.in/feed/"),
+    ("ired_team", "https://www.ired.team/feed.xml"),
+    ("pentestpartners", "https://www.pentestpartners.com/feed/"),
+    ("trustedsec", "https://www.trustedsec.com/feed/"),
+    ("specterops", "https://posts.specterops.io/feed"),
+    ("secjuice", "https://www.secjuice.com/feed/"),
+    ("varonis_blog", "https://www.varonis.com/blog/feed"),
+    ("netenrich", "https://netenrich.com/feed/"),
+    ("opennet_ru", "https://opennet.ru/rss.xml"),
+    ("linuxorg_ru", "https://www.linux.org.ru/feed.jsp"),
+    ("zd3news_security", "https://3dnews.ru/security/rss"),
+    ("securelist_ru", "https://securelist.ru/feed/"),
+    ("hackware_ru", "https://hackware.ru/feed/"),
+    ("cyberint_blog", "https://cyberint.com/blog/feed/"),
+    ("gbhackers_rss", "https://gbhackers.com/rss/"),
+    ("cybersecnews_rss", "https://cybersecuritynews.com/rss/"),
+    ("chaitin_blog", "https://www.chaitin.cn/zh/blog/feed"),
+    ("riskivy_blog", "https://blog.riskivy.com/feed"),
+    ("qianxin_news", "https://www.qianxin.com/news/rss"),
+    ("secwiki_news", "https://www.sec-wiki.com/news/rss"),
+    ("bleeping_ransomware", "https://www.bleepingcomputer.com/tag/ransomware/feed/"),
+    ("hackread_cybercrime", "https://hackread.com/category/cyber-crime/feed/"),
+    ("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
+    ("riskybiz", "https://risky.biz/feeds/risky-business-news/"),
+]
+
+TG = ["ITSecForU", "CyberSecurityNews", "securitylab", "xakepnews", "exploitdb", "hacktips",
+      "pentesttools", "sysadmin_ru", "ai_papay", "gonzo_ai", "llm_ru", "malwareunpacked",
+      "databreachnews", "cve2026", "hackersnewsru", "kasperskyclub", "SecNewsRu", "websec",
+      "cyber_ru", "hackernews_ru", "redos", "anti_malware", "Antimalware", "sec_news"]
+
+
+def rss(url):
+    ad = "rss"
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
+    r = net.get(url, timeout=25)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+def tg(h):
+    url = "https://t.me/s/" + h
+    r = net.get(url, timeout=25)
+    t = r.text or ""
+    if t.count("tgme_widget_message_wrap") < 3:
+        return {"u": url, "n": 0, "why": "no preview"}
+    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+jobs = [("rss", u) for _, u in RSS] + [("tg", h) for h in TG]
+
+
+def run(j):
+    kind, x = j
+    try:
+        return tg(x) if kind == "tg" else rss(x)
+    except Exception as e:
+        return {"u": x, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, jobs):
+        print(json.dumps(r, ensure_ascii=False))

omni/abuse.py

@@ -141,24 +141,27 @@ def _qualifies(source, title, text, tags):
         sc -= 26
         if not hits:
             return 0, []
-    raw_key = bool(re.search(r"(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
-    if NOISE.search(source or "") and not raw_key:
+    raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
+    src = str(source or "")
+    known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_"))
+    if NOISE.search(src) and not (raw_key or known_prefix):
         return 0, []
-    if cat is None:
-        # старые записи хранят категорию вместо имени источника
-        if not raw_key and not str(source or "").startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo")):
+    if cat is None and not raw_key and not known_prefix:
         return 0, []
-    elif cat not in ABUSE_CATS and not raw_key:
+    if cat is not None and cat not in ABUSE_CATS and not (raw_key or known_prefix):
         return 0, []
-    if cat == "abuse" and sc >= ABUSE_SRC_SC:
+    # панели new-api, github/hn и abuse-источники считаем абузными по факту источника
+    if cat == "abuse" or known_prefix:
+        if sc >= ABUSE_SRC_SC or strong or len(hits) >= 2:
+            return sc or 1, hits
+    if strong and sc >= 16:
         return sc, hits
-    if strong and sc >= 24:
+    if len(hits) >= 2 and sc >= 30:
         return sc, hits
-    if len(hits) >= 2 and sc >= 40:
+    if len(hits) >= 3 and sc >= 20:
         return sc, hits
     return 0, []
 
-
 def _connect():
     con = sqlite3.connect(str(DB), timeout=60)
     con.execute("PRAGMA busy_timeout=60000")
@@ -248,5 +251,26 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
     for it in items:
         for t in it["tags"]:
             counts[t] = counts.get(t, 0) + 1
-    return {"items": items[:limit], "total": len(items), "tags": counts,
+    # total = реальный размер выдачи в индексе, не размер окна чтения
+    total = len(items)
+    try:
+        con = _connect()
+        sql = "SELECT COUNT(*) FROM abuse_idx a JOIN items i ON i.id = a.id"
+        where, args = [], []
+        if tag:
+            where.append("(',' || a.tags || ',') LIKE ?")
+            args.append("%," + tag + ",%")
+        if q:
+            where.append("(i.title LIKE ? OR i.text LIKE ? OR i.source LIKE ?)")
+            args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q]
+        if since_days:
+            where.append("i.published >= date('now', ?)")
+            args.append("-%d day" % int(since_days))
+        if where:
+            sql += " WHERE " + " AND ".join(where)
+        total = con.execute(sql, args).fetchone()[0] or total
+        con.close()
+    except Exception:
+        pass
+    return {"items": items[:limit], "total": total, "tags": counts,
             "labels": TAG_LABEL}

omni/panel_api.py

@@ -30,9 +30,17 @@ def _clean(v, n=0):
 
 # ---------- обзор ----------
 
-def timeline(days=30, cat=None):
-    """Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями."""
+BUCKET_SEEN = "date(seen,'unixepoch')"
+
+
+def timeline(days=30, cat=None, mode="pub"):
+    """Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями.
+
+    mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
+    mode="seen" — строго по времени импорта: «что комбайн принёс за день».
+    """
     days = max(1, min(365, int(days or 30)))
+    bucket = BUCKET_SEEN if mode == "seen" else BUCKET
     names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
     where, args = "", []
     if names is not None:
@@ -40,7 +48,7 @@ def timeline(days=30, cat=None):
         args = list(names)
     sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
            " SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
-           " FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
+           " FROM (SELECT " + bucket + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
     got = {}
     with store.db() as c:
         for b, n, cve, sev in c.execute(sql, args):
@@ -52,8 +60,9 @@ def timeline(days=30, cat=None):
         d = (today - _dt.timedelta(days=i)).isoformat()
         n, cve, sev = got.get(d, (0, 0, 0))
         out.append({"day": d, "n": n, "cve": cve, "sev": sev})
-    return {"ok": True, "days": days, "points": out,
+    return {"ok": True, "days": days, "mode": mode, "points": out,
             "total": sum(p["n"] for p in out),
+            "last24": (out[-1] or {}).get("n", 0),
             "max": max([p["n"] for p in out] or [0])}
 
 

omni/serve.py

@@ -287,9 +287,9 @@ class Handler(BaseHTTPRequestHandler):
             if tag in ("all", "все"):
                 tag = None
             try:
-                lim = _intv(q, "limit", 40, 1, 200)
+                lim = _intv(q, "limit", 120, 1, 3000)
             except ValueError:
-                lim = 40
+                lim = 120
             return self._json(_ab.scan(limit=lim, tag=tag))
 
         if path == "/api/summary":
@@ -403,7 +403,8 @@ class Handler(BaseHTTPRequestHandler):
         if path == "/api/timeline":
             days = _intv(q, "days", 30, 1, 365)
             cat = (q.get("category") or [None])[0]
-            return self._json(panel_api.timeline(days=days, cat=cat))
+            mode = (q.get("mode") or ["pub"])[0]
+            return self._json(panel_api.timeline(days=days, cat=cat, mode=mode if mode in ("pub", "seen") else "pub"))
 
         if path == "/api/suggest":
             return self._json(panel_api.suggest((q.get("q") or [""])[0], _intv(q, "limit", 8, 1, 20)))

omni/sources_fresh.py

@@ -0,0 +1,80 @@
+"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.
+
+Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).
+Отклонённые кандидаты и причины — в хвостовом комментарии.
+Импортов из omni.sources нет намеренно (циклический импорт).
+"""
+
+
+def fresh(S):
+    """S — конструктор Source; возвращает только проверенные сетью источники."""
+    return [
+        S("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
+        S("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
+        S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach")),
+        S("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
+        S("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")),
+        S("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
+        S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup")),
+        S("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", ("tg", "ai", "ru")),
+        S("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
+        S("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing", ("reddit", "blue-team")),
+        S("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing", ("reddit", "exploit")),
+        S("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25", "reddit_listing", ("reddit", "purple-team")),
+        S("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum")),
+        S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec")),
+        S("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news")),
+        S("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")),
+        S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti")),
+        S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news")),
+        S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research")),
+        S("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", "rss", ("cve", "browser")),
+        S("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
+        S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
+        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
+        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
+        S("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research")),
+        S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
+        S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
+        S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
+        S("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")),
+        S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news")),
+        S("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")),
+        S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news")),
+        S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research")),
+        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
+        S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
+        S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
+        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
+        S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
+        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
+        S("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news")),
+        S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research")),
+        S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research")),
+        S("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
+        S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti")),
+        S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain")),
+        S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit")),
+        S("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")),
+        S("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")),
+        S("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")),
+        S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news")),
+        S("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
+        S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware")),
+        S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve")),
+        S("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
+    ]
+
+
+# --- отклонено прогоном (не отдают записи без ключа/логина) ---
+# darknetlive                https://darknetlive.com/feed/                http 200, 127811b
+# ransomlook_recent          https://www.ransomlook.io/api/recent         http 0, err:ReadTimeout: The read operation timed out, 0b
+# ransomware_live_recent     https://api.ransomware.live/recentvictims    http 200, 76549b
+# codeby_forum               https://codeby.net/forums/                   http 0, err:ConnectTimeout: timed out, 0b
+# anti_malware_ru            https://www.anti-malware.ru/rss.xml          http 0, err:ConnectTimeout: timed out, 0b
+# bdu_fstec                  https://bdu.fstec.ru/rss/vulxml              http 0, err:ConnectTimeout: timed out, 0b
+# cnews_security             https://www.cnews.ru/inc/rss/news.xml        http 0, err:ConnectTimeout: timed out, 0b
+# freebuf_feed               https://www.freebuf.com/feed                 http 405, 2569b
+# kaspersky_ru_blog          https://www.kaspersky.ru/blog/feed/          http 0, err:ConnectTimeout: timed out, 0b
+# securitylab_rss            https://www.securitylab.ru/rss/all.xml       http 404, 17877b
+# seebug_paper               https://paper.seebug.org/rss/                http 0, err:ConnectTimeout: _ssl.c:989: The handshake operation timed out, 0b

tools/fresh_candidates.py

@@ -1,9 +1,8 @@
-"""Кандидаты для расширения реестра omni: security/CVE/leak/CN/RU/tg.
+"""Кандидаты для расширения реестра omni: security/CVE/leaks/CN/RU/tg.
 
 Источник истины для tools/probe_fresh.py. Живые строки после прогона
 переезжают в omni/sources_fresh.py (генерируется `--emit`).
-
-Формат dict совпадает с Source(**d). Адаптеры — только из feeds.ADAPTERS.
+Формат dict совпадает с Source(**d); адаптеры — только из feeds.ADAPTERS.
 """
 import datetime
 
@@ -13,50 +12,48 @@ def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False,
             "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
 
 
-# NVD recent: окно считается на импорте (API требует обе даты и <=120 дней)
 _NOW = datetime.datetime.utcnow()
 _NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
 _NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
 
 CANDIDATES = [
-    # ---------------- 1. security news / research (RSS) ----------------
+    # ---------------- 1. security news / research ----------------
     d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),
-    d("gbhackers", "security", "rss", "https://gbhackers.com/feed/", "rss", ("news",)),
-    d("cybersecuritynews", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", ("news",)),
-    d("scmedia", "security", "rss", "https://www.scmagazine.com/rss.xml", "rss", ("news",)),
     d("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ("news",)),
-    d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
-    d("seclists_bugtraq", "security", "rss", "https://seclists.org/rss/bugtraq.rss", "rss", ("mailing", "exploit")),
-    d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
-    d("huntress_blog", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", ("research", "dfir")),
-    d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("research", "cti")),
-    d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("research", "cti")),
+    d("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news",)),
+    d("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news",)),
+    d("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news",)),
+    d("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news",)),
+    d("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news",)),
+    d("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news",)),
+    d("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")),
+    d("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research",)),
+    d("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")),
+
+    # ------------- research / vendor labs -------------
+    d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("cti",)),
+    d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
     d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
-    d("sophos_news", "security", "rss", "https://news.sophos.com/en-us/feed/", "rss", ("research",)),
-    d("trendmicro_si", "security", "rss", "https://www.trendmicro.com/rss/security-intelligence.xml", "rss", ("research",)),
-    d("volexity_blog", "security", "rss", "https://www.volexity.com/blog/feed/", "rss", ("research", "dfir")),
     d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
     d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
     d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
-    d("intezer_blog", "security", "rss", "https://intezer.com/blog/feed/", "rss", ("malware",)),
     d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
     d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
-    d("zscaler_research", "security", "rss",
-      "https://www.zscaler.com/blogs/security-research/rss.xml", "rss", ("research",)),
     d("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
-    d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/resources/blog/feed/", "rss", ("cti",)),
-    d("dragos_blog", "security", "rss", "https://www.dragos.com/feed/", "rss", ("ics",)),
-    d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("research", "apt")),
-    d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")),
+    d("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research",)),
+    d("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research",)),
+    d("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware",)),
     d("jfrog_blog", "security", "rss", "https://jfrog.com/blog/feed/", "rss", ("supply-chain",)),
     d("gitguardian_blog", "security", "rss", "https://blog.gitguardian.com/rss/", "rss", ("secrets",)),
-    d("truffle_security", "security", "rss", "https://trufflesecurity.com/blog/rss.xml", "rss", ("secrets",)),
+    d("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain",)),
+    d("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec",)),
     d("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
-    d("hudsonrock_blog", "security", "rss", "https://www.hudsonrock.com/blog/rss.xml", "rss", ("stealer",)),
-    d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("blog", "writeup")),
+    d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti",)),
+    d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
+    d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")),
+    d("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
     d("0xdf", "security", "rss", "https://0xdf.gitlab.io/feed.xml", "rss", ("writeup", "htb")),
-    d("hackmag", "security", "rss", "https://hackmag.com/feed/", "rss", ("ru",)),
-    d("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("cloud",)),
+    d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
 
     # ---------------- 2. CVE / vulnerabilities ----------------
     d("nvd_recent", "security", "json",
@@ -67,54 +64,38 @@ CANDIDATES = [
       "nvd", ("cve", "exploited")),
     d("github_advisories_api", "security", "json",
       "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
-    d("cve_circl_last", "security", "json", "https://cve.circl.lu/api/last/40", "cve_circl", ("cve",)),
     d("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
-    d("gentoo_glsa", "security", "rss", "https://security.gentoo.org/glsa/feed.xml", "rss", ("cve", "linux")),
-    d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/feed.atom", "rss", ("cve", "linux")),
-    d("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing", ("cve", "linux"),
-      detail=False, extra={"include_prefixes": ["/security/2026/"], "exclude_exact": ["/security/"]}),
-    d("oss_security_openwall", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
-      ("mailing", "cve"), extra={"include_prefixes": ["/lists/oss-security/2026/"]}),
+    d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")),
     d("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing",
       ("cve", "browser"), extra={"include_prefixes": ["/en-US/security/advisories/mfsa"]}),
     d("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default",
       "rss", ("cve", "browser")),
-    d("cvefeed_latest", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ("cve",)),
     d("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
-    d("cxsecurity", "security", "rss", "https://cxsecurity.com/rss/", "rss", ("exploit",)),
-    d("vulmon_recent", "security", "rss", "https://vulmon.com/rss/recent", "rss", ("cve",)),
-    d("cisa_ics_advisories", "security", "rss",
-      "https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml", "rss", ("ics", "cve")),
-    d("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
-    d("epss_recent", "security", "json", "https://api.first.org/data/v1/epss?limit=25&order=!epss", "generic_json",
-      ("cve", "epss")),
+    d("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit",)),
+    d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss",
+      ("mailing", "exploit")),
+    d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
 
     # ---------------- 3. leaks / ransomware / underground ----------------
-    d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet", "news")),
     d("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
     d("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
     d("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
+    d("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach",)),
+    d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet",)),
     d("ransomware_live_recent", "abuse", "json", "https://api.ransomware.live/recentvictims", "generic_json",
       ("ransomware",)),
     d("ransomlook_recent", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", ("ransomware",)),
-    d("ransomware_live_groups", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", ("ransomware",)),
-    d("shadowserver_news", "abuse", "rss", "https://www.shadowserver.org/news/feed/", "rss", ("infra",)),
 
-    # ---------------- 4. китайские security ----------------
-    d("anquanke_rss", "security", "rss", "https://www.anquanke.com/rss", "rss", ("cn", "news")),
-    d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")),
+    # ---------------- 4. китайские ----------------
     d("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
     d("seebug_paper", "security", "rss", "https://paper.seebug.org/rss/", "rss", ("cn", "research")),
-    d("xz_aliyun", "security", "html", "https://xz.aliyun.com/", "html_listing", ("cn", "research"),
-      extra={"include_prefixes": ["/t/"]}),
-    d("aqniu", "security", "rss", "https://www.aqniu.com/feed", "rss", ("cn", "news")),
-    d("knownsec_blog", "security", "rss", "https://blog.knownsec.com/feed/", "rss", ("cn", "research")),
-    d("butian_forum", "forum", "html", "https://forum.butian.net/", "html_listing", ("cn", "forum"),
-      extra={"include_prefixes": ["/share/"]}),
-    d("kanxue_news_html", "forum", "html", "https://bbs.kanxue.com/", "html_listing", ("cn", "forum"),
-      extra={"include_prefixes": ["/thread-"]}),
-    d("t00ls_html", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"),
-      extra={"include_prefixes": ["/article-"]}),
+    d("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news"),
+      extra={"include_prefixes": ["/articles/"], "exclude_exact": ["/articles"]}),
+    d("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news"),
+      extra={"include_prefixes": ["/post/id/"]}),
+    d("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"), detail=True,
+      extra={"include_prefixes": ["/articles-", "/Vuls-"]}),
+    d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")),
 
     # ---------------- 5. русскоязычные ----------------
     d("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
@@ -128,22 +109,26 @@ CANDIDATES = [
     d("bdu_fstec", "security", "rss", "https://bdu.fstec.ru/rss/vulxml", "rss", ("ru", "cve")),
     d("codeby_forum", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", ("ru", "forum")),
 
-    # ---------------- 6. telegram (t.me/s, публичное превью) ----------------
-    d("tg_thehackersnews", "security", "html", "https://t.me/s/TheHackersNews", "tg_web", ("tg", "news")),
+    # ---------------- 6. telegram (t.me/s) ----------------
+    d("tg_thehackernews", "security", "html", "https://t.me/s/thehackernews", "tg_web", ("tg", "news")),
     d("tg_bleepingcomputer", "security", "html", "https://t.me/s/bleepingcomputer", "tg_web", ("tg", "news")),
     d("tg_vxunderground", "security", "html", "https://t.me/s/vxunderground", "tg_web", ("tg", "malware")),
-    d("tg_secator", "security", "html", "https://t.me/s/SecAtor", "tg_web", ("tg", "ru")),
-    d("tg_itsecforu", "security", "html", "https://t.me/s/itsecforu", "tg_web", ("tg", "ru")),
-    d("tg_hitech_crime", "security", "html", "https://t.me/s/hitech_crime", "tg_web", ("tg", "ru")),
-    d("tg_codeby", "security", "html", "https://t.me/s/codeby", "tg_web", ("tg", "ru")),
-    d("tg_darkwebinformer", "abuse", "html", "https://t.me/s/DarkWebInformer", "tg_web", ("tg", "leak")),
-    d("tg_breachdetector", "abuse", "html", "https://t.me/s/breachdetector", "tg_web", ("tg", "leak")),
-    d("tg_databreaches", "abuse", "html", "https://t.me/s/databreaches", "tg_web", ("tg", "leak")),
-    d("tg_ransomware_live", "abuse", "html", "https://t.me/s/ransomware_live", "tg_web", ("tg", "ransomware")),
-    d("tg_cybersecuritynews", "security", "html", "https://t.me/s/cybersecuritynews", "tg_web", ("tg", "news")),
+    d("tg_true_secator", "security", "html", "https://t.me/s/true_secator", "tg_web", ("tg", "ru")),
+    d("tg_infosec_ru", "security", "html", "https://t.me/s/infosec_ru", "tg_web", ("tg", "ru")),
+    d("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")),
+    d("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")),
+    d("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")),
+    d("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")),
     d("tg_ai_newz", "community", "html", "https://t.me/s/ai_newz", "tg_web", ("tg", "ai", "ru")),
     d("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web",
       ("tg", "ai", "ru")),
     d("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
-    d("tg_bo0om", "abuse", "html", "https://t.me/s/bo0om", "tg_web", ("tg", "leak", "ru")),
+
+    # ---------------- reddit (forum) ----------------
+    d("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing",
+      ("reddit", "blue-team")),
+    d("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing",
+      ("reddit", "exploit")),
+    d("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25",
+      "reddit_listing", ("reddit", "purple-team")),
 ]

tools/probe_api.py

@@ -0,0 +1,45 @@
+"""Замер API панели: код, время, ключи ответа. Без секретов, только метрики."""
+import json
+import time
+import urllib.parse
+import urllib.request
+
+BASE = "http://127.0.0.1:8078"
+PATHS = [
+    "/api/health",
+    "/api/stats",
+    "/api/timeline?days=30",
+    "/api/items?limit=5",
+    "/api/abuse?limit=3",
+    "/api/tg/status",
+    "/api/keys",
+    "/api/watch",
+    "/api/digest?days=1",
+    "/api/suggest?q=cve",
+]
+
+
+def main():
+    for p in PATHS:
+        t0 = time.time()
+        try:
+            raw = urllib.request.urlopen(BASE + p, timeout=90).read().decode("utf-8", "ignore")
+            d = json.loads(raw)
+            keys = list(d.keys()) if isinstance(d, dict) else type(d).__name__
+            extra = ""
+            if isinstance(d, dict):
+                if "items" in d and isinstance(d["items"], list):
+                    extra = " n=%d" % len(d["items"])
+                    if d["items"]:
+                        extra += " keys=%s" % ",".join(list(d["items"][0].keys())[:14])
+                if "me" in d:
+                    extra = " " + json.dumps(d, ensure_ascii=False)[:200]
+                if "md" in d:
+                    extra = " md=%d chars" % len(d["md"])
+            print("%-34s %6.2fs %s%s" % (p, time.time() - t0, keys, extra))
+        except Exception as e:
+            print("%-34s %6.2fs ERR %s" % (p, time.time() - t0, e))
+
+
+if __name__ == "__main__":
+    main()

tools/probe_fresh.py

@@ -59,11 +59,37 @@ def probe_one(src, limit):
             "samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
 
 
-def emit(rows, dead, path):
+def _norm(u):
+    u = u.strip().lower().rstrip("/")
+    for pre in ("https://", "http://"):
+        if u.startswith(pre):
+            u = u[len(pre):]
+    return u[4:] if u.startswith("www.") else u
+
+
+def _registry_urls():
+    from omni.sources import all_sources
+    out = set()
+    for s in all_sources():
+        out.add(_norm(s.url))
+        out.add(_norm(s.url).split("?")[0])
+    return out
+
+
+def emit(rows, dead, path, dedupe=True):
     """omni/sources_fresh.py: только живые, без импортов из omni.sources."""
     def q(s):
         return '"%s"' % str(s).replace('"', '\\"')
 
+    if dedupe:
+        have = _registry_urls()
+        kept, skipped = [], []
+        for r in rows:
+            n = _norm(r["url"])
+            (skipped if (n in have or n.split("?")[0] in have) else kept).append(r)
+        print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
+        rows = kept
+
     lines = [
         '"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.',
         "",
@@ -104,8 +130,19 @@ def main():
     ap.add_argument("--out", default="data/fresh_probe.json")
     ap.add_argument("--emit", action="store_true")
     ap.add_argument("--fresh", action="store_true", help="прогонять только то, что уже в sources_fresh.py")
+    ap.add_argument("--emit-only", action="store_true", help="перегенерировать sources_fresh.py из готового JSON")
+    ap.add_argument("--no-dedupe", action="store_true", help="не отбрасывать источники, уже присутствующие в реестре")
     a = ap.parse_args()
 
+    if a.emit_only:
+        rep = json.loads((ROOT / a.out).read_text(encoding="utf-8"))
+        rows = rep["results"]
+        alive = [r for r in rows if r["ok"]]
+        dead = [r for r in rows if not r["ok"]]
+        n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
+        print("omni/sources_fresh.py: %d источников" % n)
+        return 0
+
     cands = [c for c in CANDIDATES if not a.only or a.only in c["name"]]
     srcs = []
     for c in cands:
@@ -137,7 +174,7 @@ def main():
         print("  dead %-24s %s" % (r["name"], r["err"][:110]))
 
     if a.emit:
-        n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py")
+        n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe)
         print("omni/sources_fresh.py: %d источников" % n)
     return 0
 

tools/ui_debug.py

@@ -0,0 +1,32 @@
+"""Дебаг панели: клик по каждому разделу, hash, начало DOM, ошибки консоли."""
+import asyncio
+import sys
+
+from playwright.async_api import async_playwright
+
+
+async def main(url="http://127.0.0.1:8078/"):
+    msgs = []
+    async with async_playwright() as p:
+        b = await p.chromium.launch()
+        pg = await b.new_page(viewport={"width": 1400, "height": 900})
+        pg.on("console", lambda m: msgs.append([m.type, m.text[:200]]))
+        pg.on("pageerror", lambda e: msgs.append(["pageerror", str(e)[:300]]))
+        await pg.goto(url, wait_until="domcontentloaded")
+        await pg.wait_for_selector(".kpi", timeout=20000)
+        for nid in ("feed", "abuse", "sources", "runs", "tg", "keys", "digest", "watch", "overview"):
+            await pg.click(f"[data-nav={nid}]")
+            await pg.wait_for_timeout(1600)
+            h = await pg.evaluate("() => location.hash")
+            head = (await pg.locator("#view").inner_text())[:180].replace("\n", " | ")
+            items = await pg.locator("#view .item").count()
+            rows = await pg.locator("#view table tbody tr").count()
+            print(f"[{nid}] hash={h!r} items={items} rows={rows}\n   {head}")
+        print("\nCONSOLE:")
+        for m in msgs[-25:]:
+            print("  ", m)
+        await b.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main(*(sys.argv[1:] or [])))