abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000
This commit is contained in:
Vlad Gothbreach
@@ -141,24 +141,27 @@ def _qualifies(source, title, text, tags):
|
||||
sc -= 26
|
||||
if not hits:
|
||||
return 0, []
|
||||
raw_key = bool(re.search(r"(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
|
||||
if NOISE.search(source or "") and not raw_key:
|
||||
raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
|
||||
src = str(source or "")
|
||||
known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_"))
|
||||
if NOISE.search(src) and not (raw_key or known_prefix):
|
||||
return 0, []
|
||||
if cat is None:
|
||||
# старые записи хранят категорию вместо имени источника
|
||||
if not raw_key and not str(source or "").startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo")):
|
||||
return 0, []
|
||||
elif cat not in ABUSE_CATS and not raw_key:
|
||||
if cat is None and not raw_key and not known_prefix:
|
||||
return 0, []
|
||||
if cat == "abuse" and sc >= ABUSE_SRC_SC:
|
||||
if cat is not None and cat not in ABUSE_CATS and not (raw_key or known_prefix):
|
||||
return 0, []
|
||||
# панели new-api, github/hn и abuse-источники считаем абузными по факту источника
|
||||
if cat == "abuse" or known_prefix:
|
||||
if sc >= ABUSE_SRC_SC or strong or len(hits) >= 2:
|
||||
return sc or 1, hits
|
||||
if strong and sc >= 16:
|
||||
return sc, hits
|
||||
if strong and sc >= 24:
|
||||
if len(hits) >= 2 and sc >= 30:
|
||||
return sc, hits
|
||||
if len(hits) >= 2 and sc >= 40:
|
||||
if len(hits) >= 3 and sc >= 20:
|
||||
return sc, hits
|
||||
return 0, []
|
||||
|
||||
|
||||
def _connect():
|
||||
con = sqlite3.connect(str(DB), timeout=60)
|
||||
con.execute("PRAGMA busy_timeout=60000")
|
||||
@@ -248,5 +251,26 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
|
||||
for it in items:
|
||||
for t in it["tags"]:
|
||||
counts[t] = counts.get(t, 0) + 1
|
||||
return {"items": items[:limit], "total": len(items), "tags": counts,
|
||||
# total = реальный размер выдачи в индексе, не размер окна чтения
|
||||
total = len(items)
|
||||
try:
|
||||
con = _connect()
|
||||
sql = "SELECT COUNT(*) FROM abuse_idx a JOIN items i ON i.id = a.id"
|
||||
where, args = [], []
|
||||
if tag:
|
||||
where.append("(',' || a.tags || ',') LIKE ?")
|
||||
args.append("%," + tag + ",%")
|
||||
if q:
|
||||
where.append("(i.title LIKE ? OR i.text LIKE ? OR i.source LIKE ?)")
|
||||
args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q]
|
||||
if since_days:
|
||||
where.append("i.published >= date('now', ?)")
|
||||
args.append("-%d day" % int(since_days))
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
total = con.execute(sql, args).fetchone()[0] or total
|
||||
con.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"items": items[:limit], "total": total, "tags": counts,
|
||||
"labels": TAG_LABEL}
|
||||
@@ -30,9 +30,17 @@ def _clean(v, n=0):
|
||||
|
||||
# ---------- обзор ----------
|
||||
|
||||
def timeline(days=30, cat=None):
|
||||
"""Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями."""
|
||||
BUCKET_SEEN = "date(seen,'unixepoch')"
|
||||
|
||||
|
||||
def timeline(days=30, cat=None, mode="pub"):
|
||||
"""Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями.
|
||||
|
||||
mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
|
||||
mode="seen" — строго по времени импорта: «что комбайн принёс за день».
|
||||
"""
|
||||
days = max(1, min(365, int(days or 30)))
|
||||
bucket = BUCKET_SEEN if mode == "seen" else BUCKET
|
||||
names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
|
||||
where, args = "", []
|
||||
if names is not None:
|
||||
@@ -40,7 +48,7 @@ def timeline(days=30, cat=None):
|
||||
args = list(names)
|
||||
sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
|
||||
" SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
|
||||
" FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
|
||||
" FROM (SELECT " + bucket + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
|
||||
got = {}
|
||||
with store.db() as c:
|
||||
for b, n, cve, sev in c.execute(sql, args):
|
||||
@@ -52,8 +60,9 @@ def timeline(days=30, cat=None):
|
||||
d = (today - _dt.timedelta(days=i)).isoformat()
|
||||
n, cve, sev = got.get(d, (0, 0, 0))
|
||||
out.append({"day": d, "n": n, "cve": cve, "sev": sev})
|
||||
return {"ok": True, "days": days, "points": out,
|
||||
return {"ok": True, "days": days, "mode": mode, "points": out,
|
||||
"total": sum(p["n"] for p in out),
|
||||
"last24": (out[-1] or {}).get("n", 0),
|
||||
"max": max([p["n"] for p in out] or [0])}
|
||||
|
||||
|
||||
|
||||
@@ -287,9 +287,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if tag in ("all", "все"):
|
||||
tag = None
|
||||
try:
|
||||
lim = _intv(q, "limit", 40, 1, 200)
|
||||
lim = _intv(q, "limit", 120, 1, 3000)
|
||||
except ValueError:
|
||||
lim = 40
|
||||
lim = 120
|
||||
return self._json(_ab.scan(limit=lim, tag=tag))
|
||||
|
||||
if path == "/api/summary":
|
||||
@@ -403,7 +403,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if path == "/api/timeline":
|
||||
days = _intv(q, "days", 30, 1, 365)
|
||||
cat = (q.get("category") or [None])[0]
|
||||
return self._json(panel_api.timeline(days=days, cat=cat))
|
||||
mode = (q.get("mode") or ["pub"])[0]
|
||||
return self._json(panel_api.timeline(days=days, cat=cat, mode=mode if mode in ("pub", "seen") else "pub"))
|
||||
|
||||
if path == "/api/suggest":
|
||||
return self._json(panel_api.suggest((q.get("q") or [""])[0], _intv(q, "limit", 8, 1, 20)))
|
||||
|
||||
80
omni/sources_fresh.py
Normal file
80
omni/sources_fresh.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.
|
||||
|
||||
Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).
|
||||
Отклонённые кандидаты и причины — в хвостовом комментарии.
|
||||
Импортов из omni.sources нет намеренно (циклический импорт).
|
||||
"""
|
||||
|
||||
|
||||
def fresh(S):
|
||||
"""S — конструктор Source; возвращает только проверенные сетью источники."""
|
||||
return [
|
||||
S("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
|
||||
S("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
|
||||
S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach")),
|
||||
S("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
|
||||
S("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")),
|
||||
S("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
|
||||
S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup")),
|
||||
S("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", ("tg", "ai", "ru")),
|
||||
S("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
|
||||
S("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing", ("reddit", "blue-team")),
|
||||
S("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing", ("reddit", "exploit")),
|
||||
S("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25", "reddit_listing", ("reddit", "purple-team")),
|
||||
S("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum")),
|
||||
S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec")),
|
||||
S("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news")),
|
||||
S("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")),
|
||||
S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti")),
|
||||
S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news")),
|
||||
S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research")),
|
||||
S("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", "rss", ("cve", "browser")),
|
||||
S("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
|
||||
S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
|
||||
S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
|
||||
S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
|
||||
S("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research")),
|
||||
S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
|
||||
S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
|
||||
S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
|
||||
S("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")),
|
||||
S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news")),
|
||||
S("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")),
|
||||
S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news")),
|
||||
S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research")),
|
||||
S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
|
||||
S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
|
||||
S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
|
||||
S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
|
||||
S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
|
||||
S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
|
||||
S("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news")),
|
||||
S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research")),
|
||||
S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research")),
|
||||
S("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
|
||||
S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti")),
|
||||
S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain")),
|
||||
S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit")),
|
||||
S("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")),
|
||||
S("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")),
|
||||
S("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")),
|
||||
S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news")),
|
||||
S("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
|
||||
S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware")),
|
||||
S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve")),
|
||||
S("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
|
||||
]
|
||||
|
||||
|
||||
# --- отклонено прогоном (не отдают записи без ключа/логина) ---
|
||||
# darknetlive https://darknetlive.com/feed/ http 200, 127811b
|
||||
# ransomlook_recent https://www.ransomlook.io/api/recent http 0, err:ReadTimeout: The read operation timed out, 0b
|
||||
# ransomware_live_recent https://api.ransomware.live/recentvictims http 200, 76549b
|
||||
# codeby_forum https://codeby.net/forums/ http 0, err:ConnectTimeout: timed out, 0b
|
||||
# anti_malware_ru https://www.anti-malware.ru/rss.xml http 0, err:ConnectTimeout: timed out, 0b
|
||||
# bdu_fstec https://bdu.fstec.ru/rss/vulxml http 0, err:ConnectTimeout: timed out, 0b
|
||||
# cnews_security https://www.cnews.ru/inc/rss/news.xml http 0, err:ConnectTimeout: timed out, 0b
|
||||
# freebuf_feed https://www.freebuf.com/feed http 405, 2569b
|
||||
# kaspersky_ru_blog https://www.kaspersky.ru/blog/feed/ http 0, err:ConnectTimeout: timed out, 0b
|
||||
# securitylab_rss https://www.securitylab.ru/rss/all.xml http 404, 17877b
|
||||
# seebug_paper https://paper.seebug.org/rss/ http 0, err:ConnectTimeout: _ssl.c:989: The handshake operation timed out, 0b
|
||||
Reference in New Issue
Block a user