import sys, json, urllib.parse
sys.path.insert(0, '.')
from concurrent.futures import ThreadPoolExecutor
from omni import feeds, net
from omni.sources import Source
from bs4 import BeautifulSoup

NEW = [
    ("helpnetsecurity", "https://www.helpnetsecurity.com/feed/"),
    ("infosecurity_magazine", "https://www.infosecurity-magazine.com/rss/news/"),
    ("cybersecuritydive", "https://www.cybersecuritydive.com/feeds/news/"),
    ("theregister_security", "https://www.theregister.com/security/headlines.atom"),
    ("arstechnica_security", "https://arstechnica.com/security/feed/"),
    ("csoonline", "https://www.csoonline.com/feed/"),
    ("hackerone_blog", "https://www.hackerone.com/blog.rss"),
    ("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
    ("semgrep_blog", "https://semgrep.dev/blog/rss.xml"),
    ("acunetix_blog", "https://www.acunetix.com/blog/feed/"),
    ("trustwave_spiderlabs", "https://www.trustwave.com/en-us/resources/blogs/spiderlabs-blog/rss.xml"),
    ("cybereason_blog", "https://www.cybereason.com/blog/rss.xml"),
    ("sentinelone_blog", "https://www.sentinelone.com/blog/feed/"),
    ("microsoft_security_blog", "https://www.microsoft.com/en-us/security/blog/feed/"),
    ("ncsc_uk", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml"),
    ("sonatype_blog", "https://www.sonatype.com/blog/rss.xml"),
    ("invicti_blog", "https://www.invicti.com/blog/feed/"),
    ("govinfosecurity", "https://www.govinfosecurity.com/rss"),
    ("bankinfosecurity", "https://www.bankinfosecurity.com/rss"),
    ("healthcareinfosecurity", "https://www.healthcareinfosecurity.com/rss"),
    ("cve_circl_alt", "https://cve.circl.lu/api/last/40"),
    ("vulmon_alt", "https://vulmon.com/rss/latest"),
    ("packetstorm_files", "https://packetstormsecurity.com/files/rss.xml"),
    ("exploitdb_rss2", "https://www.exploit-db.com/rss.xml"),
    ("zdi_blog", "https://www.zerodayinitiative.com/blog?format=rss"),
    ("hackmag_alt", "https://hackmag.com/rss"),
    ("securitylab_c", "https://www.securitylab.ru/rss/all.xml"),
    ("securitylab_d", "https://www.securitylab.ru/news/rss.xml"),
    ("anquanke_rss2", "https://www.anquanke.com/rss/news"),
    ("anquanke_rss3", "https://www.anquanke.com/rss?lang=zh"),
    ("freebuf_alt", "https://www.freebuf.com/"),
    ("sploitus_rss", "https://sploitus.com/rss"),
    ("cvefeed_alt", "https://cvefeed.io/rssfeed/latest.xml"),
    ("reddit_blueteamsec", "https://www.reddit.com/r/blueteamsec/.json?limit=25"),
    ("reddit_malware", "https://www.reddit.com/r/Malware/.json?limit=25"),
    ("reddit_reverseengineering", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
    ("reddit_exploitdev", "https://www.reddit.com/r/ExploitDev/.json?limit=25"),
    ("reddit_threatintel", "https://www.reddit.com/r/threatintel/.json?limit=25"),
    ("reddit_purpleteamsec", "https://www.reddit.com/r/purpleteamsec/.json?limit=25"),
    ("serverfault_sec", "https://security.stackexchange.com/feeds/tag?tagnames=vulnerability&sort=newest"),
]


def probe(url, adapter="rss", extra=None, detail=False):
    src = Source(name="p", category="security", kind="rss", url=url, adapter=adapter,
                 tags=(), limit=5, detail=detail, extra=extra or {})
    r = net.get(url, timeout=25)
    items = feeds.fetch(src, 5)
    return {"u": url, "st": r.status, "ct": (r.headers or {}).get("content-type", "")[:30],
            "n": len(items), "first": (items[0].title[:70] if items else "")}


def run(t):
    name, url = t
    ad = "reddit_listing" if "reddit.com" in url else "rss"
    try:
        d = probe(url, ad)
    except Exception as e:
        d = {"u": url, "err": str(e)[:80], "n": 0}
    d["name"] = name
    return d


with ThreadPoolExecutor(6) as ex:
    for r in ex.map(run, NEW):
        print(json.dumps(r, ensure_ascii=False))

# debug openwall prefix filter
r = net.get("https://www.openwall.com/lists/oss-security/", timeout=25)
soup = BeautifulSoup(r.text or "", "lxml")
paths = []
for a in soup.find_all("a", href=True)[:40]:
    paths.append(urllib.parse.urlsplit(urllib.parse.urljoin("https://www.openwall.com", str(a["href"]))).path)
print(json.dumps({"openwall_sample": paths[:20]}, ensure_ascii=False))

# detail=True variants
for u, extra in (("https://www.t00ls.com/", {"include_prefixes": ["/articles-", "/Vuls-"]}),
                 ("https://www.debian.org/security/", {"include_prefixes": ["/tracker/DSA-"]})):
    d = probe(u, "html_listing", extra, detail=True)
    d["url"] = u
    print(json.dumps(d, ensure_ascii=False))