abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)

.local/diag3.py

@@ -0,0 +1,41 @@
+import sys, json, re, urllib.parse
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import net
+from bs4 import BeautifulSoup
+
+URLS = [
+    "https://www.debian.org/security/",
+    "https://www.openwall.com/lists/oss-security/",
+    "https://www.t00ls.com/",
+    "https://www.anquanke.com/",
+    "https://www.freebuf.com/",
+    "https://bbs.kanxue.com/",
+    "https://www.secrss.com/",
+    "https://www.ichunqiu.com/",
+]
+
+
+def one(u):
+    try:
+        r = net.get(u, timeout=25)
+        soup = BeautifulSoup(r.text or "", "lxml")
+        hrefs = []
+        for a in soup.find_all("a", href=True):
+            h = urllib.parse.urljoin(u, str(a["href"]))
+            p = urllib.parse.urlsplit(h).path
+            hrefs.append(p)
+        pat = {}
+        for p in hrefs:
+            seg = re.sub(r"\d+", "N", p)[:60]
+            pat[seg] = pat.get(seg, 0) + 1
+        top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
+        return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
+                "paths": top}
+    except Exception as e:
+        return {"u": u, "st": 0, "err": str(e)[:90]}
+
+
+with ThreadPoolExecutor(4) as ex:
+    for r in ex.map(one, URLS):
+        print(json.dumps(r, ensure_ascii=False))