abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000

.local/diag5.py

@@ -0,0 +1,89 @@
+import sys, json, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+from bs4 import BeautifulSoup
+
+NEW = [
+    ("helpnetsecurity", "https://www.helpnetsecurity.com/feed/"),
+    ("infosecurity_magazine", "https://www.infosecurity-magazine.com/rss/news/"),
+    ("cybersecuritydive", "https://www.cybersecuritydive.com/feeds/news/"),
+    ("theregister_security", "https://www.theregister.com/security/headlines.atom"),
+    ("arstechnica_security", "https://arstechnica.com/security/feed/"),
+    ("csoonline", "https://www.csoonline.com/feed/"),
+    ("hackerone_blog", "https://www.hackerone.com/blog.rss"),
+    ("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
+    ("semgrep_blog", "https://semgrep.dev/blog/rss.xml"),
+    ("acunetix_blog", "https://www.acunetix.com/blog/feed/"),
+    ("trustwave_spiderlabs", "https://www.trustwave.com/en-us/resources/blogs/spiderlabs-blog/rss.xml"),
+    ("cybereason_blog", "https://www.cybereason.com/blog/rss.xml"),
+    ("sentinelone_blog", "https://www.sentinelone.com/blog/feed/"),
+    ("microsoft_security_blog", "https://www.microsoft.com/en-us/security/blog/feed/"),
+    ("ncsc_uk", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml"),
+    ("sonatype_blog", "https://www.sonatype.com/blog/rss.xml"),
+    ("invicti_blog", "https://www.invicti.com/blog/feed/"),
+    ("govinfosecurity", "https://www.govinfosecurity.com/rss"),
+    ("bankinfosecurity", "https://www.bankinfosecurity.com/rss"),
+    ("healthcareinfosecurity", "https://www.healthcareinfosecurity.com/rss"),
+    ("cve_circl_alt", "https://cve.circl.lu/api/last/40"),
+    ("vulmon_alt", "https://vulmon.com/rss/latest"),
+    ("packetstorm_files", "https://packetstormsecurity.com/files/rss.xml"),
+    ("exploitdb_rss2", "https://www.exploit-db.com/rss.xml"),
+    ("zdi_blog", "https://www.zerodayinitiative.com/blog?format=rss"),
+    ("hackmag_alt", "https://hackmag.com/rss"),
+    ("securitylab_c", "https://www.securitylab.ru/rss/all.xml"),
+    ("securitylab_d", "https://www.securitylab.ru/news/rss.xml"),
+    ("anquanke_rss2", "https://www.anquanke.com/rss/news"),
+    ("anquanke_rss3", "https://www.anquanke.com/rss?lang=zh"),
+    ("freebuf_alt", "https://www.freebuf.com/"),
+    ("sploitus_rss", "https://sploitus.com/rss"),
+    ("cvefeed_alt", "https://cvefeed.io/rssfeed/latest.xml"),
+    ("reddit_blueteamsec", "https://www.reddit.com/r/blueteamsec/.json?limit=25"),
+    ("reddit_malware", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reverseengineering", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_exploitdev", "https://www.reddit.com/r/ExploitDev/.json?limit=25"),
+    ("reddit_threatintel", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+    ("reddit_purpleteamsec", "https://www.reddit.com/r/purpleteamsec/.json?limit=25"),
+    ("serverfault_sec", "https://security.stackexchange.com/feeds/tag?tagnames=vulnerability&sort=newest"),
+]
+
+
+def probe(url, adapter="rss", extra=None, detail=False):
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=adapter,
+                 tags=(), limit=5, detail=detail, extra=extra or {})
+    r = net.get(url, timeout=25)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "st": r.status, "ct": (r.headers or {}).get("content-type", "")[:30],
+            "n": len(items), "first": (items[0].title[:70] if items else "")}
+
+
+def run(t):
+    name, url = t
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    try:
+        d = probe(url, ad)
+    except Exception as e:
+        d = {"u": url, "err": str(e)[:80], "n": 0}
+    d["name"] = name
+    return d
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, NEW):
+        print(json.dumps(r, ensure_ascii=False))
+
+# debug openwall prefix filter
+r = net.get("https://www.openwall.com/lists/oss-security/", timeout=25)
+soup = BeautifulSoup(r.text or "", "lxml")
+paths = []
+for a in soup.find_all("a", href=True)[:40]:
+    paths.append(urllib.parse.urlsplit(urllib.parse.urljoin("https://www.openwall.com", str(a["href"]))).path)
+print(json.dumps({"openwall_sample": paths[:20]}, ensure_ascii=False))
+
+# detail=True variants
+for u, extra in (("https://www.t00ls.com/", {"include_prefixes": ["/articles-", "/Vuls-"]}),
+                 ("https://www.debian.org/security/", {"include_prefixes": ["/tracker/DSA-"]})):
+    d = probe(u, "html_listing", extra, detail=True)
+    d["url"] = u
+    print(json.dumps(d, ensure_ascii=False))