abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000

.local/diag5.py

@@ -0,0 +1,89 @@
+import sys, json, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+from bs4 import BeautifulSoup
+
+NEW = [
+    ("helpnetsecurity", "https://www.helpnetsecurity.com/feed/"),
+    ("infosecurity_magazine", "https://www.infosecurity-magazine.com/rss/news/"),
+    ("cybersecuritydive", "https://www.cybersecuritydive.com/feeds/news/"),
+    ("theregister_security", "https://www.theregister.com/security/headlines.atom"),
+    ("arstechnica_security", "https://arstechnica.com/security/feed/"),
+    ("csoonline", "https://www.csoonline.com/feed/"),
+    ("hackerone_blog", "https://www.hackerone.com/blog.rss"),
+    ("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
+    ("semgrep_blog", "https://semgrep.dev/blog/rss.xml"),
+    ("acunetix_blog", "https://www.acunetix.com/blog/feed/"),
+    ("trustwave_spiderlabs", "https://www.trustwave.com/en-us/resources/blogs/spiderlabs-blog/rss.xml"),
+    ("cybereason_blog", "https://www.cybereason.com/blog/rss.xml"),
+    ("sentinelone_blog", "https://www.sentinelone.com/blog/feed/"),
+    ("microsoft_security_blog", "https://www.microsoft.com/en-us/security/blog/feed/"),
+    ("ncsc_uk", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml"),
+    ("sonatype_blog", "https://www.sonatype.com/blog/rss.xml"),
+    ("invicti_blog", "https://www.invicti.com/blog/feed/"),
+    ("govinfosecurity", "https://www.govinfosecurity.com/rss"),
+    ("bankinfosecurity", "https://www.bankinfosecurity.com/rss"),
+    ("healthcareinfosecurity", "https://www.healthcareinfosecurity.com/rss"),
+    ("cve_circl_alt", "https://cve.circl.lu/api/last/40"),
+    ("vulmon_alt", "https://vulmon.com/rss/latest"),
+    ("packetstorm_files", "https://packetstormsecurity.com/files/rss.xml"),
+    ("exploitdb_rss2", "https://www.exploit-db.com/rss.xml"),
+    ("zdi_blog", "https://www.zerodayinitiative.com/blog?format=rss"),
+    ("hackmag_alt", "https://hackmag.com/rss"),
+    ("securitylab_c", "https://www.securitylab.ru/rss/all.xml"),
+    ("securitylab_d", "https://www.securitylab.ru/news/rss.xml"),
+    ("anquanke_rss2", "https://www.anquanke.com/rss/news"),
+    ("anquanke_rss3", "https://www.anquanke.com/rss?lang=zh"),
+    ("freebuf_alt", "https://www.freebuf.com/"),
+    ("sploitus_rss", "https://sploitus.com/rss"),
+    ("cvefeed_alt", "https://cvefeed.io/rssfeed/latest.xml"),
+    ("reddit_blueteamsec", "https://www.reddit.com/r/blueteamsec/.json?limit=25"),
+    ("reddit_malware", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reverseengineering", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_exploitdev", "https://www.reddit.com/r/ExploitDev/.json?limit=25"),
+    ("reddit_threatintel", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+    ("reddit_purpleteamsec", "https://www.reddit.com/r/purpleteamsec/.json?limit=25"),
+    ("serverfault_sec", "https://security.stackexchange.com/feeds/tag?tagnames=vulnerability&sort=newest"),
+]
+
+
+def probe(url, adapter="rss", extra=None, detail=False):
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=adapter,
+                 tags=(), limit=5, detail=detail, extra=extra or {})
+    r = net.get(url, timeout=25)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "st": r.status, "ct": (r.headers or {}).get("content-type", "")[:30],
+            "n": len(items), "first": (items[0].title[:70] if items else "")}
+
+
+def run(t):
+    name, url = t
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    try:
+        d = probe(url, ad)
+    except Exception as e:
+        d = {"u": url, "err": str(e)[:80], "n": 0}
+    d["name"] = name
+    return d
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, NEW):
+        print(json.dumps(r, ensure_ascii=False))
+
+# debug openwall prefix filter
+r = net.get("https://www.openwall.com/lists/oss-security/", timeout=25)
+soup = BeautifulSoup(r.text or "", "lxml")
+paths = []
+for a in soup.find_all("a", href=True)[:40]:
+    paths.append(urllib.parse.urlsplit(urllib.parse.urljoin("https://www.openwall.com", str(a["href"]))).path)
+print(json.dumps({"openwall_sample": paths[:20]}, ensure_ascii=False))
+
+# detail=True variants
+for u, extra in (("https://www.t00ls.com/", {"include_prefixes": ["/articles-", "/Vuls-"]}),
+                 ("https://www.debian.org/security/", {"include_prefixes": ["/tracker/DSA-"]})):
+    d = probe(u, "html_listing", extra, detail=True)
+    d["url"] = u
+    print(json.dumps(d, ensure_ascii=False))

.local/diag6.py

@@ -0,0 +1,69 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
+      "breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
+      "hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
+      "cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
+      "osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]
+
+RSS = [
+    ("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
+    ("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
+    ("industrialcyber", "https://industrialcyber.co/feed/"),
+    ("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
+    ("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
+    ("malwaretech_blog", "https://www.malwaretech.com/feed"),
+    ("scworld", "https://www.scworld.com/rss.xml"),
+    ("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
+    ("securityweek_alt", "https://www.securityweek.com/feed/"),
+    ("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
+    ("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
+    ("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+]
+
+
+def tg(url):
+    r = net.get(url, timeout=25)
+    t = r.text or ""
+    if t.count("tgme_widget_message_wrap") < 3:
+        return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
+    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+def rss(url):
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]
+
+
+def run(j):
+    kind, u = j
+    try:
+        return tg(u) if kind == "tg" else rss(u)
+    except Exception as e:
+        return {"u": u, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, jobs):
+        print(json.dumps(r, ensure_ascii=False))
+
+# openwall: почему фильтр не срабатывает
+src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
+             adapter="html_listing", tags=(), limit=5,
+             extra={"include_prefixes": ["/lists/oss-security/"]})
+print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
+r = net.get(src.url, timeout=25)
+print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/"))

.local/diag7.py

@@ -0,0 +1,43 @@
+import sys, json
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+C = [
+    ("cisa_news", "https://www.cisa.gov/news-events/news/rss.xml", "rss", {}),
+    ("fortinet_blog", "https://www.fortinet.com/blog/rss.xml", "rss", {}),
+    ("mcafee_blogs", "https://www.mcafee.com/blogs/feed/", "rss", {}),
+    ("avast_blog", "https://www.avast.com/en-us/blog/feed", "rss", {}),
+    ("eset_press", "https://www.eset.com/int/about/newsroom/press-releases/feed/", "rss", {}),
+    ("gti_threat_intel", "https://cloud.google.com/blog/topics/threat-intelligence/rss/", "rss", {}),
+    ("checkpoint_blog", "https://www.checkpoint.com/blog/feed/", "rss", {}),
+    ("anquanke_news", "https://www.anquanke.com/news", "html_listing", {"include_prefixes": ["/post/id/"]}),
+    ("secrss_vuln", "https://www.secrss.com/articles", "html_listing", {"include_prefixes": ["/articles/"]}),
+    ("bleeping_dup_check", "https://www.bleepingcomputer.com/feed/", "rss", {}),
+    ("therecord_dup_check", "https://therecord.media/feed", "rss", {}),
+    ("deepwatch_blog", "https://www.deepwatch.com/feed/", "rss", {}),
+    ("huntress_alt", "https://www.huntress.com/blog/feed", "rss", {}),
+    ("dragos_alt", "https://www.dragos.com/feed/", "rss", {}),
+    ("scmedia_alt", "https://www.scmagazine.com/feed", "rss", {}),
+    ("trendmicro_alt", "https://www.trendmicro.com/en_us/research/feed", "rss", {}),
+    ("zscaler_alt", "https://www.zscaler.com/blogs/security-research/rss", "rss", {}),
+    ("gbhackers_alt", "https://gbhackers.com/feed", "rss", {}),
+    ("cybersecnews_alt", "https://cybersecuritynews.com/feed", "rss", {}),
+]
+
+
+def run(t):
+    name, url, ad, extra = t
+    src = Source(name=name, category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5, extra=extra)
+    try:
+        r = net.get(url, timeout=25)
+        items = feeds.fetch(src, 5)
+        return {"name": name, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
+    except Exception as e:
+        return {"name": name, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, C):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag8.py

@@ -0,0 +1,79 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+RSS = [
+    ("itsecurityguru", "https://www.itsecurityguru.co.uk/feed/"),
+    ("securityledger", "https://securityledger.com/feed/"),
+    ("zdnet_security", "https://www.zdnet.com/topic/security/rss.xml"),
+    ("grahamcluley", "https://www.grahamcluley.com/feed/"),
+    ("cyberdefensemagazine", "https://www.cyberdefensemagazine.com/feed/"),
+    ("hackersonlineclub", "https://www.hackersonlineclub.com/feed"),
+    ("netspi", "https://www.netspi.com/feed/"),
+    ("hackingarticles", "https://www.hackingarticles.in/feed/"),
+    ("ired_team", "https://www.ired.team/feed.xml"),
+    ("pentestpartners", "https://www.pentestpartners.com/feed/"),
+    ("trustedsec", "https://www.trustedsec.com/feed/"),
+    ("specterops", "https://posts.specterops.io/feed"),
+    ("secjuice", "https://www.secjuice.com/feed/"),
+    ("varonis_blog", "https://www.varonis.com/blog/feed"),
+    ("netenrich", "https://netenrich.com/feed/"),
+    ("opennet_ru", "https://opennet.ru/rss.xml"),
+    ("linuxorg_ru", "https://www.linux.org.ru/feed.jsp"),
+    ("zd3news_security", "https://3dnews.ru/security/rss"),
+    ("securelist_ru", "https://securelist.ru/feed/"),
+    ("hackware_ru", "https://hackware.ru/feed/"),
+    ("cyberint_blog", "https://cyberint.com/blog/feed/"),
+    ("gbhackers_rss", "https://gbhackers.com/rss/"),
+    ("cybersecnews_rss", "https://cybersecuritynews.com/rss/"),
+    ("chaitin_blog", "https://www.chaitin.cn/zh/blog/feed"),
+    ("riskivy_blog", "https://blog.riskivy.com/feed"),
+    ("qianxin_news", "https://www.qianxin.com/news/rss"),
+    ("secwiki_news", "https://www.sec-wiki.com/news/rss"),
+    ("bleeping_ransomware", "https://www.bleepingcomputer.com/tag/ransomware/feed/"),
+    ("hackread_cybercrime", "https://hackread.com/category/cyber-crime/feed/"),
+    ("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
+    ("riskybiz", "https://risky.biz/feeds/risky-business-news/"),
+]
+
+TG = ["ITSecForU", "CyberSecurityNews", "securitylab", "xakepnews", "exploitdb", "hacktips",
+      "pentesttools", "sysadmin_ru", "ai_papay", "gonzo_ai", "llm_ru", "malwareunpacked",
+      "databreachnews", "cve2026", "hackersnewsru", "kasperskyclub", "SecNewsRu", "websec",
+      "cyber_ru", "hackernews_ru", "redos", "anti_malware", "Antimalware", "sec_news"]
+
+
+def rss(url):
+    ad = "rss"
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
+    r = net.get(url, timeout=25)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+def tg(h):
+    url = "https://t.me/s/" + h
+    r = net.get(url, timeout=25)
+    t = r.text or ""
+    if t.count("tgme_widget_message_wrap") < 3:
+        return {"u": url, "n": 0, "why": "no preview"}
+    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+jobs = [("rss", u) for _, u in RSS] + [("tg", h) for h in TG]
+
+
+def run(j):
+    kind, x = j
+    try:
+        return tg(x) if kind == "tg" else rss(x)
+    except Exception as e:
+        return {"u": x, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, jobs):
+        print(json.dumps(r, ensure_ascii=False))