69 lines
3.3 KiB
Python
69 lines
3.3 KiB
Python
import sys, json, re
|
|
sys.path.insert(0, '.')
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from omni import feeds, net
|
|
from omni.sources import Source
|
|
|
|
TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
|
|
"breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
|
|
"hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
|
|
"cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
|
|
"osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]
|
|
|
|
RSS = [
|
|
("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
|
|
("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
|
|
("industrialcyber", "https://industrialcyber.co/feed/"),
|
|
("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
|
|
("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
|
|
("malwaretech_blog", "https://www.malwaretech.com/feed"),
|
|
("scworld", "https://www.scworld.com/rss.xml"),
|
|
("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
|
|
("securityweek_alt", "https://www.securityweek.com/feed/"),
|
|
("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
|
|
("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
|
|
("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
|
|
("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
|
|
("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
|
|
]
|
|
|
|
|
|
def tg(url):
|
|
r = net.get(url, timeout=25)
|
|
t = r.text or ""
|
|
if t.count("tgme_widget_message_wrap") < 3:
|
|
return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
|
|
src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
|
|
items = feeds.fetch(src, 5)
|
|
return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
|
|
|
|
def rss(url):
|
|
ad = "reddit_listing" if "reddit.com" in url else "rss"
|
|
src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
|
|
items = feeds.fetch(src, 5)
|
|
return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
|
|
|
|
jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]
|
|
|
|
|
|
def run(j):
|
|
kind, u = j
|
|
try:
|
|
return tg(u) if kind == "tg" else rss(u)
|
|
except Exception as e:
|
|
return {"u": u, "n": 0, "err": str(e)[:70]}
|
|
|
|
|
|
with ThreadPoolExecutor(6) as ex:
|
|
for r in ex.map(run, jobs):
|
|
print(json.dumps(r, ensure_ascii=False))
|
|
|
|
# openwall: почему фильтр не срабатывает
|
|
src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
|
|
adapter="html_listing", tags=(), limit=5,
|
|
extra={"include_prefixes": ["/lists/oss-security/"]})
|
|
print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
|
|
r = net.get(src.url, timeout=25)
|
|
print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/")) |