abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)

.local/diag.py

@@ -0,0 +1,28 @@
+import sys, json
+sys.path.insert(0,'.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net
+
+URLS = [
+ "https://darknetlive.com/feed/", "https://darknetlive.com/rss/", "https://darknetlive.com/rss.xml",
+ "https://www.volexity.com/blog/feed/", "https://cxsecurity.com/rss/",
+ "https://www.debian.org/security/", "https://www.openwall.com/lists/oss-security/",
+ "https://www.t00ls.com/", "https://xz.aliyun.com/",
+ "https://t.me/s/TheHackersNews", "https://t.me/s/thehackernews", "https://t.me/s/SecAtor",
+ "https://t.me/s/true_secator", "https://t.me/s/databreaches", "https://t.me/s/cybersecuritynews",
+]
+def one(u):
+    try:
+        r = net.get(u, timeout=25)
+        t = r.text or ""
+        return {"u":u,"st":r.status,"n":len(t),
+                "ct":(r.headers or {}).get("content-type","")[:40],
+                "item":t.count("<item"),"entry":t.count("<entry"),
+                "tgmsg":t.count("tgme_widget_message_wrap"),
+                "hdr":t.count('tgme_channel_info_header_title'),
+                "head": " ".join(t[:200].split())[:120]}
+    except Exception as e:
+        return {"u":u,"st":0,"err":str(e)[:90]}
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(one, URLS):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag2.py

@@ -0,0 +1,33 @@
+import sys, json, re
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net, feeds
+from omni.sources import Source
+
+TG = ["thehackernews", "true_secator", "cybersecuritynews", "ransomwatch", "ransomware_news", "leakbase",
+      "CVEnew", "cvefeed", "hackernews", "cyberdefensemagazine", "securityweek", "therecord",
+      "ciscotalos", "unit42", "projectzero", "huntress", "sentinelone", "recordedfuture",
+      "threatmon", "kalilinux", "darkwebinformer", "darkwebinformer_ru", "breachdetector",
+      "databreach", "ruhack", "expl0it", "infosec_ru", "securitylab_ru", "itsecforu_ru",
+      "xakep", "XakepRU", "bo0om_ru", "mycopypaste", "cnews_ru", "bdu_fstec", "kaspersky",
+      "kaspersky_ru", "talos", "smd_ru", "kernelmode", "hitech_crime", "codeby"]
+
+
+def tg_one(h, url):
+    try:
+        r = net.get(url, timeout=25)
+        t = r.text or ""
+        title = re.search(r"<title>(.*?)</title>", t, re.S)
+        src = Source(name="probe_" + h, category="security", kind="html", url=url, adapter="tg_web",
+                     tags=("tg", "leak"), limit=5)
+        items = feeds.fetch(src, 5)
+        return {"h": h, "st": r.status, "n": len(t), "msg": t.count("tgme_widget_message_wrap"),
+                "items": len(items), "title": (title.group(1).strip()[:60] if title else ""),
+                "first": (items[0].title[:70] if items else "")}
+    except Exception as e:
+        return {"h": h, "st": 0, "err": str(e)[:80]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(lambda h: tg_one(h, "https://t.me/s/" + h), TG):
+        print(json.dumps(r, ensure_ascii=False))

.local/diag3.py

@@ -0,0 +1,41 @@
+import sys, json, re, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net
+from bs4 import BeautifulSoup
+
+URLS = [
+    "https://www.debian.org/security/",
+    "https://www.openwall.com/lists/oss-security/",
+    "https://www.t00ls.com/",
+    "https://www.anquanke.com/",
+    "https://www.freebuf.com/",
+    "https://bbs.kanxue.com/",
+    "https://www.secrss.com/",
+    "https://www.ichunqiu.com/",
+]
+
+
+def one(u):
+    try:
+        r = net.get(u, timeout=25)
+        soup = BeautifulSoup(r.text or "", "lxml")
+        hrefs = []
+        for a in soup.find_all("a", href=True):
+            h = urllib.parse.urljoin(u, str(a["href"]))
+            p = urllib.parse.urlsplit(h).path
+            hrefs.append(p)
+        pat = {}
+        for p in hrefs:
+            seg = re.sub(r"\d+", "N", p)[:60]
+            pat[seg] = pat.get(seg, 0) + 1
+        top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
+        return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
+                "paths": top}
+    except Exception as e:
+        return {"u": u, "st": 0, "err": str(e)[:90]}
+
+
+with ThreadPoolExecutor(4) as ex:
+    for r in ex.map(one, URLS):
+        print(json.dumps(r, ensure_ascii=False))