41 lines
1.2 KiB
Python
41 lines
1.2 KiB
Python
import sys, json, re, urllib.parse
|
|
sys.path.insert(0, '.')
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from omni import net
|
|
from bs4 import BeautifulSoup
|
|
|
|
URLS = [
|
|
"https://www.debian.org/security/",
|
|
"https://www.openwall.com/lists/oss-security/",
|
|
"https://www.t00ls.com/",
|
|
"https://www.anquanke.com/",
|
|
"https://www.freebuf.com/",
|
|
"https://bbs.kanxue.com/",
|
|
"https://www.secrss.com/",
|
|
"https://www.ichunqiu.com/",
|
|
]
|
|
|
|
|
|
def one(u):
|
|
try:
|
|
r = net.get(u, timeout=25)
|
|
soup = BeautifulSoup(r.text or "", "lxml")
|
|
hrefs = []
|
|
for a in soup.find_all("a", href=True):
|
|
h = urllib.parse.urljoin(u, str(a["href"]))
|
|
p = urllib.parse.urlsplit(h).path
|
|
hrefs.append(p)
|
|
pat = {}
|
|
for p in hrefs:
|
|
seg = re.sub(r"\d+", "N", p)[:60]
|
|
pat[seg] = pat.get(seg, 0) + 1
|
|
top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
|
|
return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
|
|
"paths": top}
|
|
except Exception as e:
|
|
return {"u": u, "st": 0, "err": str(e)[:90]}
|
|
|
|
|
|
with ThreadPoolExecutor(4) as ex:
|
|
for r in ex.map(one, URLS):
|
|
print(json.dumps(r, ensure_ascii=False)) |