import sys, json, re, urllib.parse
sys.path.insert(0, '.')
from concurrent.futures import ThreadPoolExecutor
from omni import net
from bs4 import BeautifulSoup

URLS = [
    "https://www.debian.org/security/",
    "https://www.openwall.com/lists/oss-security/",
    "https://www.t00ls.com/",
    "https://www.anquanke.com/",
    "https://www.freebuf.com/",
    "https://bbs.kanxue.com/",
    "https://www.secrss.com/",
    "https://www.ichunqiu.com/",
]


def one(u):
    try:
        r = net.get(u, timeout=25)
        soup = BeautifulSoup(r.text or "", "lxml")
        hrefs = []
        for a in soup.find_all("a", href=True):
            h = urllib.parse.urljoin(u, str(a["href"]))
            p = urllib.parse.urlsplit(h).path
            hrefs.append(p)
        pat = {}
        for p in hrefs:
            seg = re.sub(r"\d+", "N", p)[:60]
            pat[seg] = pat.get(seg, 0) + 1
        top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
        return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
                "paths": top}
    except Exception as e:
        return {"u": u, "st": 0, "err": str(e)[:90]}


with ThreadPoolExecutor(4) as ex:
    for r in ex.map(one, URLS):
        print(json.dumps(r, ensure_ascii=False))