abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)
This commit is contained in:
Vlad Gothbreach
149
tools/fresh_candidates.py
Normal file
149
tools/fresh_candidates.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""Кандидаты для расширения реестра omni: security/CVE/leak/CN/RU/tg.
|
||||
|
||||
Источник истины для tools/probe_fresh.py. Живые строки после прогона
|
||||
переезжают в omni/sources_fresh.py (генерируется `--emit`).
|
||||
|
||||
Формат dict совпадает с Source(**d). Адаптеры — только из feeds.ADAPTERS.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
|
||||
def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
||||
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
||||
|
||||
|
||||
# NVD recent: окно считается на импорте (API требует обе даты и <=120 дней)
|
||||
_NOW = datetime.datetime.utcnow()
|
||||
_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
|
||||
_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
|
||||
|
||||
CANDIDATES = [
|
||||
# ---------------- 1. security news / research (RSS) ----------------
|
||||
d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),
|
||||
d("gbhackers", "security", "rss", "https://gbhackers.com/feed/", "rss", ("news",)),
|
||||
d("cybersecuritynews", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", ("news",)),
|
||||
d("scmedia", "security", "rss", "https://www.scmagazine.com/rss.xml", "rss", ("news",)),
|
||||
d("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ("news",)),
|
||||
d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
|
||||
d("seclists_bugtraq", "security", "rss", "https://seclists.org/rss/bugtraq.rss", "rss", ("mailing", "exploit")),
|
||||
d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
|
||||
d("huntress_blog", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", ("research", "dfir")),
|
||||
d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("research", "cti")),
|
||||
d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("research", "cti")),
|
||||
d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
|
||||
d("sophos_news", "security", "rss", "https://news.sophos.com/en-us/feed/", "rss", ("research",)),
|
||||
d("trendmicro_si", "security", "rss", "https://www.trendmicro.com/rss/security-intelligence.xml", "rss", ("research",)),
|
||||
d("volexity_blog", "security", "rss", "https://www.volexity.com/blog/feed/", "rss", ("research", "dfir")),
|
||||
d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
|
||||
d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
|
||||
d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
|
||||
d("intezer_blog", "security", "rss", "https://intezer.com/blog/feed/", "rss", ("malware",)),
|
||||
d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
|
||||
d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
|
||||
d("zscaler_research", "security", "rss",
|
||||
"https://www.zscaler.com/blogs/security-research/rss.xml", "rss", ("research",)),
|
||||
d("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
|
||||
d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/resources/blog/feed/", "rss", ("cti",)),
|
||||
d("dragos_blog", "security", "rss", "https://www.dragos.com/feed/", "rss", ("ics",)),
|
||||
d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("research", "apt")),
|
||||
d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")),
|
||||
d("jfrog_blog", "security", "rss", "https://jfrog.com/blog/feed/", "rss", ("supply-chain",)),
|
||||
d("gitguardian_blog", "security", "rss", "https://blog.gitguardian.com/rss/", "rss", ("secrets",)),
|
||||
d("truffle_security", "security", "rss", "https://trufflesecurity.com/blog/rss.xml", "rss", ("secrets",)),
|
||||
d("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
|
||||
d("hudsonrock_blog", "security", "rss", "https://www.hudsonrock.com/blog/rss.xml", "rss", ("stealer",)),
|
||||
d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("blog", "writeup")),
|
||||
d("0xdf", "security", "rss", "https://0xdf.gitlab.io/feed.xml", "rss", ("writeup", "htb")),
|
||||
d("hackmag", "security", "rss", "https://hackmag.com/feed/", "rss", ("ru",)),
|
||||
d("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("cloud",)),
|
||||
|
||||
# ---------------- 2. CVE / vulnerabilities ----------------
|
||||
d("nvd_recent", "security", "json",
|
||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
|
||||
% (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")),
|
||||
d("nvd_exploited_kw", "security", "json",
|
||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
|
||||
"nvd", ("cve", "exploited")),
|
||||
d("github_advisories_api", "security", "json",
|
||||
"https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
|
||||
d("cve_circl_last", "security", "json", "https://cve.circl.lu/api/last/40", "cve_circl", ("cve",)),
|
||||
d("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
|
||||
d("gentoo_glsa", "security", "rss", "https://security.gentoo.org/glsa/feed.xml", "rss", ("cve", "linux")),
|
||||
d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/feed.atom", "rss", ("cve", "linux")),
|
||||
d("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing", ("cve", "linux"),
|
||||
detail=False, extra={"include_prefixes": ["/security/2026/"], "exclude_exact": ["/security/"]}),
|
||||
d("oss_security_openwall", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
|
||||
("mailing", "cve"), extra={"include_prefixes": ["/lists/oss-security/2026/"]}),
|
||||
d("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing",
|
||||
("cve", "browser"), extra={"include_prefixes": ["/en-US/security/advisories/mfsa"]}),
|
||||
d("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default",
|
||||
"rss", ("cve", "browser")),
|
||||
d("cvefeed_latest", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ("cve",)),
|
||||
d("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
|
||||
d("cxsecurity", "security", "rss", "https://cxsecurity.com/rss/", "rss", ("exploit",)),
|
||||
d("vulmon_recent", "security", "rss", "https://vulmon.com/rss/recent", "rss", ("cve",)),
|
||||
d("cisa_ics_advisories", "security", "rss",
|
||||
"https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml", "rss", ("ics", "cve")),
|
||||
d("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
|
||||
d("epss_recent", "security", "json", "https://api.first.org/data/v1/epss?limit=25&order=!epss", "generic_json",
|
||||
("cve", "epss")),
|
||||
|
||||
# ---------------- 3. leaks / ransomware / underground ----------------
|
||||
d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet", "news")),
|
||||
d("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
|
||||
d("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
|
||||
d("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
|
||||
d("ransomware_live_recent", "abuse", "json", "https://api.ransomware.live/recentvictims", "generic_json",
|
||||
("ransomware",)),
|
||||
d("ransomlook_recent", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", ("ransomware",)),
|
||||
d("ransomware_live_groups", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", ("ransomware",)),
|
||||
d("shadowserver_news", "abuse", "rss", "https://www.shadowserver.org/news/feed/", "rss", ("infra",)),
|
||||
|
||||
# ---------------- 4. китайские security ----------------
|
||||
d("anquanke_rss", "security", "rss", "https://www.anquanke.com/rss", "rss", ("cn", "news")),
|
||||
d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")),
|
||||
d("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
|
||||
d("seebug_paper", "security", "rss", "https://paper.seebug.org/rss/", "rss", ("cn", "research")),
|
||||
d("xz_aliyun", "security", "html", "https://xz.aliyun.com/", "html_listing", ("cn", "research"),
|
||||
extra={"include_prefixes": ["/t/"]}),
|
||||
d("aqniu", "security", "rss", "https://www.aqniu.com/feed", "rss", ("cn", "news")),
|
||||
d("knownsec_blog", "security", "rss", "https://blog.knownsec.com/feed/", "rss", ("cn", "research")),
|
||||
d("butian_forum", "forum", "html", "https://forum.butian.net/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/share/"]}),
|
||||
d("kanxue_news_html", "forum", "html", "https://bbs.kanxue.com/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/thread-"]}),
|
||||
d("t00ls_html", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/article-"]}),
|
||||
|
||||
# ---------------- 5. русскоязычные ----------------
|
||||
d("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
|
||||
d("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
|
||||
d("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
|
||||
d("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
|
||||
d("kaspersky_ru_blog", "security", "rss", "https://www.kaspersky.ru/blog/feed/", "rss", ("ru", "research")),
|
||||
d("securitylab_rss", "security", "rss", "https://www.securitylab.ru/rss/all.xml", "rss", ("ru", "news")),
|
||||
d("anti_malware_ru", "security", "rss", "https://www.anti-malware.ru/rss.xml", "rss", ("ru", "news")),
|
||||
d("cnews_security", "security", "rss", "https://www.cnews.ru/inc/rss/news.xml", "rss", ("ru", "news")),
|
||||
d("bdu_fstec", "security", "rss", "https://bdu.fstec.ru/rss/vulxml", "rss", ("ru", "cve")),
|
||||
d("codeby_forum", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", ("ru", "forum")),
|
||||
|
||||
# ---------------- 6. telegram (t.me/s, публичное превью) ----------------
|
||||
d("tg_thehackersnews", "security", "html", "https://t.me/s/TheHackersNews", "tg_web", ("tg", "news")),
|
||||
d("tg_bleepingcomputer", "security", "html", "https://t.me/s/bleepingcomputer", "tg_web", ("tg", "news")),
|
||||
d("tg_vxunderground", "security", "html", "https://t.me/s/vxunderground", "tg_web", ("tg", "malware")),
|
||||
d("tg_secator", "security", "html", "https://t.me/s/SecAtor", "tg_web", ("tg", "ru")),
|
||||
d("tg_itsecforu", "security", "html", "https://t.me/s/itsecforu", "tg_web", ("tg", "ru")),
|
||||
d("tg_hitech_crime", "security", "html", "https://t.me/s/hitech_crime", "tg_web", ("tg", "ru")),
|
||||
d("tg_codeby", "security", "html", "https://t.me/s/codeby", "tg_web", ("tg", "ru")),
|
||||
d("tg_darkwebinformer", "abuse", "html", "https://t.me/s/DarkWebInformer", "tg_web", ("tg", "leak")),
|
||||
d("tg_breachdetector", "abuse", "html", "https://t.me/s/breachdetector", "tg_web", ("tg", "leak")),
|
||||
d("tg_databreaches", "abuse", "html", "https://t.me/s/databreaches", "tg_web", ("tg", "leak")),
|
||||
d("tg_ransomware_live", "abuse", "html", "https://t.me/s/ransomware_live", "tg_web", ("tg", "ransomware")),
|
||||
d("tg_cybersecuritynews", "security", "html", "https://t.me/s/cybersecuritynews", "tg_web", ("tg", "news")),
|
||||
d("tg_ai_newz", "community", "html", "https://t.me/s/ai_newz", "tg_web", ("tg", "ai", "ru")),
|
||||
d("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web",
|
||||
("tg", "ai", "ru")),
|
||||
d("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
|
||||
d("tg_bo0om", "abuse", "html", "https://t.me/s/bo0om", "tg_web", ("tg", "leak", "ru")),
|
||||
]
|
||||
Reference in New Issue
Block a user