abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000

.local/diag6.py

@@ -0,0 +1,69 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds, net
+from omni.sources import Source
+
+TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
+      "breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
+      "hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
+      "cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
+      "osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]
+
+RSS = [
+    ("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
+    ("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
+    ("industrialcyber", "https://industrialcyber.co/feed/"),
+    ("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
+    ("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
+    ("malwaretech_blog", "https://www.malwaretech.com/feed"),
+    ("scworld", "https://www.scworld.com/rss.xml"),
+    ("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
+    ("securityweek_alt", "https://www.securityweek.com/feed/"),
+    ("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
+    ("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
+    ("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
+    ("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
+    ("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
+]
+
+
+def tg(url):
+    r = net.get(url, timeout=25)
+    t = r.text or ""
+    if t.count("tgme_widget_message_wrap") < 3:
+        return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
+    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+def rss(url):
+    ad = "reddit_listing" if "reddit.com" in url else "rss"
+    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
+    items = feeds.fetch(src, 5)
+    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
+
+
+jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]
+
+
+def run(j):
+    kind, u = j
+    try:
+        return tg(u) if kind == "tg" else rss(u)
+    except Exception as e:
+        return {"u": u, "n": 0, "err": str(e)[:70]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, jobs):
+        print(json.dumps(r, ensure_ascii=False))
+
+# openwall: почему фильтр не срабатывает
+src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
+             adapter="html_listing", tags=(), limit=5,
+             extra={"include_prefixes": ["/lists/oss-security/"]})
+print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
+r = net.get(src.url, timeout=25)
+print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/"))