import sys, json, re
sys.path.insert(0, '.')
from concurrent.futures import ThreadPoolExecutor
from omni import feeds, net
from omni.sources import Source

TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
      "breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
      "hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
      "cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
      "osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]

RSS = [
    ("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
    ("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
    ("industrialcyber", "https://industrialcyber.co/feed/"),
    ("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
    ("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
    ("malwaretech_blog", "https://www.malwaretech.com/feed"),
    ("scworld", "https://www.scworld.com/rss.xml"),
    ("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
    ("securityweek_alt", "https://www.securityweek.com/feed/"),
    ("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
    ("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
    ("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
    ("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
    ("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
]


def tg(url):
    r = net.get(url, timeout=25)
    t = r.text or ""
    if t.count("tgme_widget_message_wrap") < 3:
        return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
    src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
    items = feeds.fetch(src, 5)
    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}


def rss(url):
    ad = "reddit_listing" if "reddit.com" in url else "rss"
    src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
    items = feeds.fetch(src, 5)
    return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}


jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]


def run(j):
    kind, u = j
    try:
        return tg(u) if kind == "tg" else rss(u)
    except Exception as e:
        return {"u": u, "n": 0, "err": str(e)[:70]}


with ThreadPoolExecutor(6) as ex:
    for r in ex.map(run, jobs):
        print(json.dumps(r, ensure_ascii=False))

# openwall: почему фильтр не срабатывает
src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
             adapter="html_listing", tags=(), limit=5,
             extra={"include_prefixes": ["/lists/oss-security/"]})
print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
r = net.get(src.url, timeout=25)
print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/"))