абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days
This commit is contained in:
Vlad Gothbreach
@@ -1,28 +0,0 @@
|
|||||||
import sys, json
|
|
||||||
sys.path.insert(0,'.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import net
|
|
||||||
|
|
||||||
URLS = [
|
|
||||||
"https://darknetlive.com/feed/", "https://darknetlive.com/rss/", "https://darknetlive.com/rss.xml",
|
|
||||||
"https://www.volexity.com/blog/feed/", "https://cxsecurity.com/rss/",
|
|
||||||
"https://www.debian.org/security/", "https://www.openwall.com/lists/oss-security/",
|
|
||||||
"https://www.t00ls.com/", "https://xz.aliyun.com/",
|
|
||||||
"https://t.me/s/TheHackersNews", "https://t.me/s/thehackernews", "https://t.me/s/SecAtor",
|
|
||||||
"https://t.me/s/true_secator", "https://t.me/s/databreaches", "https://t.me/s/cybersecuritynews",
|
|
||||||
]
|
|
||||||
def one(u):
|
|
||||||
try:
|
|
||||||
r = net.get(u, timeout=25)
|
|
||||||
t = r.text or ""
|
|
||||||
return {"u":u,"st":r.status,"n":len(t),
|
|
||||||
"ct":(r.headers or {}).get("content-type","")[:40],
|
|
||||||
"item":t.count("<item"),"entry":t.count("<entry"),
|
|
||||||
"tgmsg":t.count("tgme_widget_message_wrap"),
|
|
||||||
"hdr":t.count('tgme_channel_info_header_title'),
|
|
||||||
"head": " ".join(t[:200].split())[:120]}
|
|
||||||
except Exception as e:
|
|
||||||
return {"u":u,"st":0,"err":str(e)[:90]}
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(one, URLS):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
import sys, json, re
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import net, feeds
|
|
||||||
from omni.sources import Source
|
|
||||||
|
|
||||||
TG = ["thehackernews", "true_secator", "cybersecuritynews", "ransomwatch", "ransomware_news", "leakbase",
|
|
||||||
"CVEnew", "cvefeed", "hackernews", "cyberdefensemagazine", "securityweek", "therecord",
|
|
||||||
"ciscotalos", "unit42", "projectzero", "huntress", "sentinelone", "recordedfuture",
|
|
||||||
"threatmon", "kalilinux", "darkwebinformer", "darkwebinformer_ru", "breachdetector",
|
|
||||||
"databreach", "ruhack", "expl0it", "infosec_ru", "securitylab_ru", "itsecforu_ru",
|
|
||||||
"xakep", "XakepRU", "bo0om_ru", "mycopypaste", "cnews_ru", "bdu_fstec", "kaspersky",
|
|
||||||
"kaspersky_ru", "talos", "smd_ru", "kernelmode", "hitech_crime", "codeby"]
|
|
||||||
|
|
||||||
|
|
||||||
def tg_one(h, url):
|
|
||||||
try:
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
t = r.text or ""
|
|
||||||
title = re.search(r"<title>(.*?)</title>", t, re.S)
|
|
||||||
src = Source(name="probe_" + h, category="security", kind="html", url=url, adapter="tg_web",
|
|
||||||
tags=("tg", "leak"), limit=5)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"h": h, "st": r.status, "n": len(t), "msg": t.count("tgme_widget_message_wrap"),
|
|
||||||
"items": len(items), "title": (title.group(1).strip()[:60] if title else ""),
|
|
||||||
"first": (items[0].title[:70] if items else "")}
|
|
||||||
except Exception as e:
|
|
||||||
return {"h": h, "st": 0, "err": str(e)[:80]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(lambda h: tg_one(h, "https://t.me/s/" + h), TG):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
import sys, json, re, urllib.parse
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import net
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
URLS = [
|
|
||||||
"https://www.debian.org/security/",
|
|
||||||
"https://www.openwall.com/lists/oss-security/",
|
|
||||||
"https://www.t00ls.com/",
|
|
||||||
"https://www.anquanke.com/",
|
|
||||||
"https://www.freebuf.com/",
|
|
||||||
"https://bbs.kanxue.com/",
|
|
||||||
"https://www.secrss.com/",
|
|
||||||
"https://www.ichunqiu.com/",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def one(u):
|
|
||||||
try:
|
|
||||||
r = net.get(u, timeout=25)
|
|
||||||
soup = BeautifulSoup(r.text or "", "lxml")
|
|
||||||
hrefs = []
|
|
||||||
for a in soup.find_all("a", href=True):
|
|
||||||
h = urllib.parse.urljoin(u, str(a["href"]))
|
|
||||||
p = urllib.parse.urlsplit(h).path
|
|
||||||
hrefs.append(p)
|
|
||||||
pat = {}
|
|
||||||
for p in hrefs:
|
|
||||||
seg = re.sub(r"\d+", "N", p)[:60]
|
|
||||||
pat[seg] = pat.get(seg, 0) + 1
|
|
||||||
top = sorted(pat.items(), key=lambda kv: -kv[1])[:12]
|
|
||||||
return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs),
|
|
||||||
"paths": top}
|
|
||||||
except Exception as e:
|
|
||||||
return {"u": u, "st": 0, "err": str(e)[:90]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(4) as ex:
|
|
||||||
for r in ex.map(one, URLS):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
import sys, json
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import feeds
|
|
||||||
from omni.sources import Source
|
|
||||||
|
|
||||||
C = [
|
|
||||||
("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing",
|
|
||||||
{"include_prefixes": ["/tracker/DSA-"]}),
|
|
||||||
("debian_dsa2", "security", "html", "https://www.debian.org/security/", "html_listing",
|
|
||||||
{"include_prefixes": ["/debian-security-announce/"]}),
|
|
||||||
("openwall_a", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
|
|
||||||
{"include_prefixes": ["/lists/oss-security/2026/"]}),
|
|
||||||
("openwall_b", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
|
|
||||||
{"include_prefixes": ["/lists/oss-security/"]}),
|
|
||||||
("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing",
|
|
||||||
{"include_prefixes": ["/articles-", "/Vuls-"]}),
|
|
||||||
("anquanke_html", "security", "html", "https://www.anquanke.com/", "html_listing",
|
|
||||||
{"include_prefixes": ["/post/id/"]}),
|
|
||||||
("secrss", "security", "html", "https://www.secrss.com/", "html_listing",
|
|
||||||
{"include_prefixes": ["/articles/"], "exclude_exact": ["/articles"]}),
|
|
||||||
("freebuf_root", "security", "html", "https://www.freebuf.com/", "html_listing",
|
|
||||||
{"include_prefixes": ["/articles/", "/news/"]}),
|
|
||||||
("freebuf_news", "security", "html", "https://www.freebuf.com/news", "html_listing",
|
|
||||||
{"include_prefixes": ["/articles/", "/news/"]}),
|
|
||||||
("freebuf_rss2", "security", "rss", "https://www.freebuf.com/rss", "rss", {}),
|
|
||||||
("freebuf_rss3", "security", "rss", "https://www.freebuf.com/feed/", "rss", {}),
|
|
||||||
("securitylab_a", "security", "rss", "https://www.securitylab.ru/rss/news.xml", "rss", {}),
|
|
||||||
("securitylab_b", "security", "rss", "https://www.securitylab.ru/rss/", "rss", {}),
|
|
||||||
("kaspersky_a", "security", "rss", "https://www.kaspersky.com/blog/feed/", "rss", {}),
|
|
||||||
("anti_malware_a", "security", "rss", "https://www.anti-malware.ru/rss/news", "rss", {}),
|
|
||||||
("xakep_tech", "security", "rss", "https://xakep.ru/feed/", "rss", {}),
|
|
||||||
("codeby_x", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", {}),
|
|
||||||
("seebug_retry", "security", "rss", "https://paper.seebug.org/rss/", "rss", {}),
|
|
||||||
("ransomlook_retry", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", {}),
|
|
||||||
("ransomware_live_g", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", {}),
|
|
||||||
("cybersecnews_retry", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", {}),
|
|
||||||
("gbhackers_retry", "security", "rss", "https://gbhackers.com/feed/", "rss", {}),
|
|
||||||
("huntress_retry", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", {}),
|
|
||||||
("scmedia_a", "security", "rss", "https://www.scmagazine.com/rss/news", "rss", {}),
|
|
||||||
("trendmicro_a", "security", "rss", "https://www.trendmicro.com/en_us/research.rss", "rss", {}),
|
|
||||||
("dragos_a", "security", "rss", "https://www.dragos.com/blog/feed/", "rss", {}),
|
|
||||||
("arcticwolf_a", "security", "rss", "https://arcticwolf.com/feed/", "rss", {}),
|
|
||||||
("zscaler_a", "security", "rss", "https://www.zscaler.com/blogs/rss.xml", "rss", {}),
|
|
||||||
("arch_a", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", {}),
|
|
||||||
("gentoo_a", "security", "rss", "https://www.gentoo.org/glsa/feed.xml", "rss", {}),
|
|
||||||
("cisa_all_retry", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", {}),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def run(t):
|
|
||||||
name, cat, kind, url, ad, extra = t
|
|
||||||
src = Source(name=name, category=cat, kind=kind, url=url, adapter=ad, tags=(), limit=5, extra=extra)
|
|
||||||
try:
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"name": name, "n": len(items), "first": (items[0].title[:70] if items else "")}
|
|
||||||
except Exception as e:
|
|
||||||
return {"name": name, "n": 0, "err": str(e)[:80]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(run, C):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
import sys, json, urllib.parse
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import feeds, net
|
|
||||||
from omni.sources import Source
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
NEW = [
|
|
||||||
("helpnetsecurity", "https://www.helpnetsecurity.com/feed/"),
|
|
||||||
("infosecurity_magazine", "https://www.infosecurity-magazine.com/rss/news/"),
|
|
||||||
("cybersecuritydive", "https://www.cybersecuritydive.com/feeds/news/"),
|
|
||||||
("theregister_security", "https://www.theregister.com/security/headlines.atom"),
|
|
||||||
("arstechnica_security", "https://arstechnica.com/security/feed/"),
|
|
||||||
("csoonline", "https://www.csoonline.com/feed/"),
|
|
||||||
("hackerone_blog", "https://www.hackerone.com/blog.rss"),
|
|
||||||
("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
|
|
||||||
("semgrep_blog", "https://semgrep.dev/blog/rss.xml"),
|
|
||||||
("acunetix_blog", "https://www.acunetix.com/blog/feed/"),
|
|
||||||
("trustwave_spiderlabs", "https://www.trustwave.com/en-us/resources/blogs/spiderlabs-blog/rss.xml"),
|
|
||||||
("cybereason_blog", "https://www.cybereason.com/blog/rss.xml"),
|
|
||||||
("sentinelone_blog", "https://www.sentinelone.com/blog/feed/"),
|
|
||||||
("microsoft_security_blog", "https://www.microsoft.com/en-us/security/blog/feed/"),
|
|
||||||
("ncsc_uk", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml"),
|
|
||||||
("sonatype_blog", "https://www.sonatype.com/blog/rss.xml"),
|
|
||||||
("invicti_blog", "https://www.invicti.com/blog/feed/"),
|
|
||||||
("govinfosecurity", "https://www.govinfosecurity.com/rss"),
|
|
||||||
("bankinfosecurity", "https://www.bankinfosecurity.com/rss"),
|
|
||||||
("healthcareinfosecurity", "https://www.healthcareinfosecurity.com/rss"),
|
|
||||||
("cve_circl_alt", "https://cve.circl.lu/api/last/40"),
|
|
||||||
("vulmon_alt", "https://vulmon.com/rss/latest"),
|
|
||||||
("packetstorm_files", "https://packetstormsecurity.com/files/rss.xml"),
|
|
||||||
("exploitdb_rss2", "https://www.exploit-db.com/rss.xml"),
|
|
||||||
("zdi_blog", "https://www.zerodayinitiative.com/blog?format=rss"),
|
|
||||||
("hackmag_alt", "https://hackmag.com/rss"),
|
|
||||||
("securitylab_c", "https://www.securitylab.ru/rss/all.xml"),
|
|
||||||
("securitylab_d", "https://www.securitylab.ru/news/rss.xml"),
|
|
||||||
("anquanke_rss2", "https://www.anquanke.com/rss/news"),
|
|
||||||
("anquanke_rss3", "https://www.anquanke.com/rss?lang=zh"),
|
|
||||||
("freebuf_alt", "https://www.freebuf.com/"),
|
|
||||||
("sploitus_rss", "https://sploitus.com/rss"),
|
|
||||||
("cvefeed_alt", "https://cvefeed.io/rssfeed/latest.xml"),
|
|
||||||
("reddit_blueteamsec", "https://www.reddit.com/r/blueteamsec/.json?limit=25"),
|
|
||||||
("reddit_malware", "https://www.reddit.com/r/Malware/.json?limit=25"),
|
|
||||||
("reddit_reverseengineering", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
|
|
||||||
("reddit_exploitdev", "https://www.reddit.com/r/ExploitDev/.json?limit=25"),
|
|
||||||
("reddit_threatintel", "https://www.reddit.com/r/threatintel/.json?limit=25"),
|
|
||||||
("reddit_purpleteamsec", "https://www.reddit.com/r/purpleteamsec/.json?limit=25"),
|
|
||||||
("serverfault_sec", "https://security.stackexchange.com/feeds/tag?tagnames=vulnerability&sort=newest"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def probe(url, adapter="rss", extra=None, detail=False):
|
|
||||||
src = Source(name="p", category="security", kind="rss", url=url, adapter=adapter,
|
|
||||||
tags=(), limit=5, detail=detail, extra=extra or {})
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"u": url, "st": r.status, "ct": (r.headers or {}).get("content-type", "")[:30],
|
|
||||||
"n": len(items), "first": (items[0].title[:70] if items else "")}
|
|
||||||
|
|
||||||
|
|
||||||
def run(t):
|
|
||||||
name, url = t
|
|
||||||
ad = "reddit_listing" if "reddit.com" in url else "rss"
|
|
||||||
try:
|
|
||||||
d = probe(url, ad)
|
|
||||||
except Exception as e:
|
|
||||||
d = {"u": url, "err": str(e)[:80], "n": 0}
|
|
||||||
d["name"] = name
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(run, NEW):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
|
|
||||||
# debug openwall prefix filter
|
|
||||||
r = net.get("https://www.openwall.com/lists/oss-security/", timeout=25)
|
|
||||||
soup = BeautifulSoup(r.text or "", "lxml")
|
|
||||||
paths = []
|
|
||||||
for a in soup.find_all("a", href=True)[:40]:
|
|
||||||
paths.append(urllib.parse.urlsplit(urllib.parse.urljoin("https://www.openwall.com", str(a["href"]))).path)
|
|
||||||
print(json.dumps({"openwall_sample": paths[:20]}, ensure_ascii=False))
|
|
||||||
|
|
||||||
# detail=True variants
|
|
||||||
for u, extra in (("https://www.t00ls.com/", {"include_prefixes": ["/articles-", "/Vuls-"]}),
|
|
||||||
("https://www.debian.org/security/", {"include_prefixes": ["/tracker/DSA-"]})):
|
|
||||||
d = probe(u, "html_listing", extra, detail=True)
|
|
||||||
d["url"] = u
|
|
||||||
print(json.dumps(d, ensure_ascii=False))
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
import sys, json, re
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import feeds, net
|
|
||||||
from omni.sources import Source
|
|
||||||
|
|
||||||
TG = ["DarkWebInformer", "darkwebinformernews", "ransomwarelive", "ransom_live", "leak_insider",
|
|
||||||
"breachforum", "cyberpolice_ru", "threatintel", "dfir", "dfir_channel", "malwareanalysis",
|
|
||||||
"hackernews_ru", "CyberSecurityRu", "cybersecurityru", "ibsecurity", "secnews", "securitynews",
|
|
||||||
"cve", "cvewatch", "vulns", "0day", "0dayfans", "pentest", "bugbounty", "redteam",
|
|
||||||
"osint", "cti", "apt", "hackers", "cyberhack", "infosec", "инфобез", "hackersnews"]
|
|
||||||
|
|
||||||
RSS = [
|
|
||||||
("virustotal_blog", "https://blog.virustotal.com/feeds/posts/default"),
|
|
||||||
("vulncheck_blog", "https://vulncheck.com/blog/rss.xml"),
|
|
||||||
("industrialcyber", "https://industrialcyber.co/feed/"),
|
|
||||||
("silentpush_blog", "https://www.silentpush.com/blog/feed/"),
|
|
||||||
("cybersixgill_blog", "https://www.cybersixgill.com/feed/"),
|
|
||||||
("malwaretech_blog", "https://www.malwaretech.com/feed"),
|
|
||||||
("scworld", "https://www.scworld.com/rss.xml"),
|
|
||||||
("hackread_leaks", "https://hackread.com/category/data-breaches/feed/"),
|
|
||||||
("securityweek_alt", "https://www.securityweek.com/feed/"),
|
|
||||||
("reddit_netsec", "https://www.reddit.com/r/netsec/.json?limit=25"),
|
|
||||||
("reddit_cybersecurity_dup", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
|
|
||||||
("reddit_malware2", "https://www.reddit.com/r/Malware/.json?limit=25"),
|
|
||||||
("reddit_reveng2", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25"),
|
|
||||||
("reddit_threatintel2", "https://www.reddit.com/r/threatintel/.json?limit=25"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def tg(url):
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
t = r.text or ""
|
|
||||||
if t.count("tgme_widget_message_wrap") < 3:
|
|
||||||
return {"u": url, "n": 0, "why": "no preview", "title": (re.search(r"<title>(.*?)</title>", t, re.S) or [None, ""])[1][:50]}
|
|
||||||
src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
||||||
|
|
||||||
|
|
||||||
def rss(url):
|
|
||||||
ad = "reddit_listing" if "reddit.com" in url else "rss"
|
|
||||||
src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
||||||
|
|
||||||
|
|
||||||
jobs = [("tg", "https://t.me/s/" + h) for h in TG] + [("rss", u) for _, u in RSS]
|
|
||||||
|
|
||||||
|
|
||||||
def run(j):
|
|
||||||
kind, u = j
|
|
||||||
try:
|
|
||||||
return tg(u) if kind == "tg" else rss(u)
|
|
||||||
except Exception as e:
|
|
||||||
return {"u": u, "n": 0, "err": str(e)[:70]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(run, jobs):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
|
|
||||||
# openwall: почему фильтр не срабатывает
|
|
||||||
src = Source(name="ow", category="security", kind="html", url="https://www.openwall.com/lists/oss-security/",
|
|
||||||
adapter="html_listing", tags=(), limit=5,
|
|
||||||
extra={"include_prefixes": ["/lists/oss-security/"]})
|
|
||||||
print("openwall direct:", len(feeds._html_listing(src, 5)), len(feeds._html_listing(src, 40)))
|
|
||||||
r = net.get(src.url, timeout=25)
|
|
||||||
print("openwall status:", r.status, len(r.text or ""), "count /lists/oss-security/", (r.text or "").count("/lists/oss-security/"))
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
import sys, json
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import feeds, net
|
|
||||||
from omni.sources import Source
|
|
||||||
|
|
||||||
C = [
|
|
||||||
("cisa_news", "https://www.cisa.gov/news-events/news/rss.xml", "rss", {}),
|
|
||||||
("fortinet_blog", "https://www.fortinet.com/blog/rss.xml", "rss", {}),
|
|
||||||
("mcafee_blogs", "https://www.mcafee.com/blogs/feed/", "rss", {}),
|
|
||||||
("avast_blog", "https://www.avast.com/en-us/blog/feed", "rss", {}),
|
|
||||||
("eset_press", "https://www.eset.com/int/about/newsroom/press-releases/feed/", "rss", {}),
|
|
||||||
("gti_threat_intel", "https://cloud.google.com/blog/topics/threat-intelligence/rss/", "rss", {}),
|
|
||||||
("checkpoint_blog", "https://www.checkpoint.com/blog/feed/", "rss", {}),
|
|
||||||
("anquanke_news", "https://www.anquanke.com/news", "html_listing", {"include_prefixes": ["/post/id/"]}),
|
|
||||||
("secrss_vuln", "https://www.secrss.com/articles", "html_listing", {"include_prefixes": ["/articles/"]}),
|
|
||||||
("bleeping_dup_check", "https://www.bleepingcomputer.com/feed/", "rss", {}),
|
|
||||||
("therecord_dup_check", "https://therecord.media/feed", "rss", {}),
|
|
||||||
("deepwatch_blog", "https://www.deepwatch.com/feed/", "rss", {}),
|
|
||||||
("huntress_alt", "https://www.huntress.com/blog/feed", "rss", {}),
|
|
||||||
("dragos_alt", "https://www.dragos.com/feed/", "rss", {}),
|
|
||||||
("scmedia_alt", "https://www.scmagazine.com/feed", "rss", {}),
|
|
||||||
("trendmicro_alt", "https://www.trendmicro.com/en_us/research/feed", "rss", {}),
|
|
||||||
("zscaler_alt", "https://www.zscaler.com/blogs/security-research/rss", "rss", {}),
|
|
||||||
("gbhackers_alt", "https://gbhackers.com/feed", "rss", {}),
|
|
||||||
("cybersecnews_alt", "https://cybersecuritynews.com/feed", "rss", {}),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def run(t):
|
|
||||||
name, url, ad, extra = t
|
|
||||||
src = Source(name=name, category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5, extra=extra)
|
|
||||||
try:
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"name": name, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
||||||
except Exception as e:
|
|
||||||
return {"name": name, "n": 0, "err": str(e)[:70]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(run, C):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
import sys, json, re
|
|
||||||
sys.path.insert(0, '.')
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from omni import feeds, net
|
|
||||||
from omni.sources import Source
|
|
||||||
|
|
||||||
RSS = [
|
|
||||||
("itsecurityguru", "https://www.itsecurityguru.co.uk/feed/"),
|
|
||||||
("securityledger", "https://securityledger.com/feed/"),
|
|
||||||
("zdnet_security", "https://www.zdnet.com/topic/security/rss.xml"),
|
|
||||||
("grahamcluley", "https://www.grahamcluley.com/feed/"),
|
|
||||||
("cyberdefensemagazine", "https://www.cyberdefensemagazine.com/feed/"),
|
|
||||||
("hackersonlineclub", "https://www.hackersonlineclub.com/feed"),
|
|
||||||
("netspi", "https://www.netspi.com/feed/"),
|
|
||||||
("hackingarticles", "https://www.hackingarticles.in/feed/"),
|
|
||||||
("ired_team", "https://www.ired.team/feed.xml"),
|
|
||||||
("pentestpartners", "https://www.pentestpartners.com/feed/"),
|
|
||||||
("trustedsec", "https://www.trustedsec.com/feed/"),
|
|
||||||
("specterops", "https://posts.specterops.io/feed"),
|
|
||||||
("secjuice", "https://www.secjuice.com/feed/"),
|
|
||||||
("varonis_blog", "https://www.varonis.com/blog/feed"),
|
|
||||||
("netenrich", "https://netenrich.com/feed/"),
|
|
||||||
("opennet_ru", "https://opennet.ru/rss.xml"),
|
|
||||||
("linuxorg_ru", "https://www.linux.org.ru/feed.jsp"),
|
|
||||||
("zd3news_security", "https://3dnews.ru/security/rss"),
|
|
||||||
("securelist_ru", "https://securelist.ru/feed/"),
|
|
||||||
("hackware_ru", "https://hackware.ru/feed/"),
|
|
||||||
("cyberint_blog", "https://cyberint.com/blog/feed/"),
|
|
||||||
("gbhackers_rss", "https://gbhackers.com/rss/"),
|
|
||||||
("cybersecnews_rss", "https://cybersecuritynews.com/rss/"),
|
|
||||||
("chaitin_blog", "https://www.chaitin.cn/zh/blog/feed"),
|
|
||||||
("riskivy_blog", "https://blog.riskivy.com/feed"),
|
|
||||||
("qianxin_news", "https://www.qianxin.com/news/rss"),
|
|
||||||
("secwiki_news", "https://www.sec-wiki.com/news/rss"),
|
|
||||||
("bleeping_ransomware", "https://www.bleepingcomputer.com/tag/ransomware/feed/"),
|
|
||||||
("hackread_cybercrime", "https://hackread.com/category/cyber-crime/feed/"),
|
|
||||||
("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
|
|
||||||
("riskybiz", "https://risky.biz/feeds/risky-business-news/"),
|
|
||||||
]
|
|
||||||
|
|
||||||
TG = ["ITSecForU", "CyberSecurityNews", "securitylab", "xakepnews", "exploitdb", "hacktips",
|
|
||||||
"pentesttools", "sysadmin_ru", "ai_papay", "gonzo_ai", "llm_ru", "malwareunpacked",
|
|
||||||
"databreachnews", "cve2026", "hackersnewsru", "kasperskyclub", "SecNewsRu", "websec",
|
|
||||||
"cyber_ru", "hackernews_ru", "redos", "anti_malware", "Antimalware", "sec_news"]
|
|
||||||
|
|
||||||
|
|
||||||
def rss(url):
|
|
||||||
ad = "rss"
|
|
||||||
src = Source(name="p", category="security", kind="rss", url=url, adapter=ad, tags=(), limit=5)
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"u": url, "st": r.status, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
||||||
|
|
||||||
|
|
||||||
def tg(h):
|
|
||||||
url = "https://t.me/s/" + h
|
|
||||||
r = net.get(url, timeout=25)
|
|
||||||
t = r.text or ""
|
|
||||||
if t.count("tgme_widget_message_wrap") < 3:
|
|
||||||
return {"u": url, "n": 0, "why": "no preview"}
|
|
||||||
src = Source(name="p", category="security", kind="html", url=url, adapter="tg_web", tags=("tg",), limit=5)
|
|
||||||
items = feeds.fetch(src, 5)
|
|
||||||
return {"u": url, "n": len(items), "first": (items[0].title[:60] if items else "")}
|
|
||||||
|
|
||||||
|
|
||||||
jobs = [("rss", u) for _, u in RSS] + [("tg", h) for h in TG]
|
|
||||||
|
|
||||||
|
|
||||||
def run(j):
|
|
||||||
kind, x = j
|
|
||||||
try:
|
|
||||||
return tg(x) if kind == "tg" else rss(x)
|
|
||||||
except Exception as e:
|
|
||||||
return {"u": x, "n": 0, "err": str(e)[:70]}
|
|
||||||
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(6) as ex:
|
|
||||||
for r in ex.map(run, jobs):
|
|
||||||
print(json.dumps(r, ensure_ascii=False))
|
|
||||||
2496
.local/have.txt
2496
.local/have.txt
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
|||||||
21311348-6c5d-4922-b48b-df65f227ec1f
|
|
||||||
@@ -235,9 +235,11 @@ def _rows(limit=400, tag=None, q=None, since_days=None):
|
|||||||
out = []
|
out = []
|
||||||
for r in con.execute(sql, args):
|
for r in con.execute(sql, args):
|
||||||
hits = [t for t in (r[6] or "").split(",") if t]
|
hits = [t for t in (r[6] or "").split(",") if t]
|
||||||
|
body = (r[4] or "")
|
||||||
out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
|
out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
|
||||||
"text": (r[4] or "")[:600], "published": r[5],
|
"text": body[:600], "published": r[5],
|
||||||
"tags": hits, "abuse_score": r[8], "rank": r[7]})
|
"tags": hits, "abuse_score": r[8], "rank": r[7],
|
||||||
|
"find": extract((r[3] or "") + " " + body)})
|
||||||
return out
|
return out
|
||||||
finally:
|
finally:
|
||||||
con.close()
|
con.close()
|
||||||
@@ -273,4 +275,43 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return {"items": items[:limit], "total": total, "tags": counts,
|
return {"items": items[:limit], "total": total, "tags": counts,
|
||||||
"labels": TAG_LABEL}
|
"labels": TAG_LABEL}
|
||||||
|
|
||||||
|
_RE_CODE = re.compile(r"(?:промокод|промо-?код|promo\s*code|coupon|купо[нш]|code)[^\n:：]{0,12}[:：\s]+([A-Z0-9][A-Z0-9_\-]{3,23})", re.I)
|
||||||
|
_RE_CODE2 = re.compile(r"\b([A-Z]{2,}[0-9]{2,}[A-Z0-9_\-]{0,14})\b")
|
||||||
|
_RE_AMT = re.compile(r"(\d[\d\s.,]{0,12})\s*(млрд|млн|тыс|billion|million|k|m|b|токен\w*|tokens?|credits?|\$|₽|юан\w*)", re.I)
|
||||||
|
_RE_LINK = re.compile(r"(?:https?://|t\.me/)[^\s)\]\"'<>]{4,120}")
|
||||||
|
_RE_DAYS = re.compile(r"(\d{1,3})\s*(дн\w*|day[s]?|час\w*|hour[s]?|недел\w*|week[s]?)", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(text):
|
||||||
|
"""Достаёт из текста абуза полезное: промокоды, объём, ссылки, срок."""
|
||||||
|
t = (text or "")[:4000]
|
||||||
|
codes = []
|
||||||
|
for m in _RE_CODE.finditer(t):
|
||||||
|
c = (m.group(1) or "").strip(" .,:;")
|
||||||
|
if c and c.upper() not in {x.upper() for x in codes}:
|
||||||
|
codes.append(c)
|
||||||
|
if not codes and re.search(r"промо|promo|code|код|gift|купо|coupon|подар", t, re.I):
|
||||||
|
for m in _RE_CODE2.finditer(t):
|
||||||
|
c = m.group(1)
|
||||||
|
if c and c.upper() not in {x.upper() for x in codes}:
|
||||||
|
codes.append(c)
|
||||||
|
if len(codes) >= 3:
|
||||||
|
break
|
||||||
|
amt = None
|
||||||
|
m = _RE_AMT.search(t)
|
||||||
|
if m:
|
||||||
|
amt = ("%s %s" % (m.group(1).strip(), m.group(2))).replace(" ", " ")
|
||||||
|
links = []
|
||||||
|
for u in _RE_LINK.findall(t):
|
||||||
|
u = u.rstrip(".,);")
|
||||||
|
if u not in links:
|
||||||
|
links.append(u)
|
||||||
|
if len(links) >= 4:
|
||||||
|
break
|
||||||
|
days = None
|
||||||
|
m = _RE_DAYS.search(t)
|
||||||
|
if m:
|
||||||
|
days = "%s %s" % (m.group(1), m.group(2))
|
||||||
|
return {"codes": codes[:4], "amount": amt, "links": links, "term": days}
|
||||||
|
|||||||
@@ -21,9 +21,11 @@ def _clean(v, n=0):
|
|||||||
import html as _h
|
import html as _h
|
||||||
if not v:
|
if not v:
|
||||||
return ""
|
return ""
|
||||||
t = re.sub(r"<[^>]{1,200}>", " ", str(v))
|
t = str(v)
|
||||||
for _ in range(2):
|
# unescape и снятие тегов в цикле: иначе <b> после unescape снова станет тегом
|
||||||
|
for _ in range(3):
|
||||||
t = _h.unescape(t)
|
t = _h.unescape(t)
|
||||||
|
t = re.sub(r"<[^>]{1,200}>", " ", t)
|
||||||
t = re.sub(r"\s+", " ", t).strip()
|
t = re.sub(r"\s+", " ", t).strip()
|
||||||
return t[:n].rstrip() if n else t
|
return t[:n].rstrip() if n else t
|
||||||
|
|
||||||
@@ -39,7 +41,8 @@ def timeline(days=30, cat=None, mode="pub"):
|
|||||||
mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
|
mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
|
||||||
mode="seen" — строго по времени импорта: «что комбайн принёс за день».
|
mode="seen" — строго по времени импорта: «что комбайн принёс за день».
|
||||||
"""
|
"""
|
||||||
days = max(1, min(365, int(days or 30)))
|
days = 30 if days is None else int(days or 0)
|
||||||
|
days = max(1, min(365, days))
|
||||||
bucket = BUCKET_SEEN if mode == "seen" else BUCKET
|
bucket = BUCKET_SEEN if mode == "seen" else BUCKET
|
||||||
names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
|
names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
|
||||||
where, args = "", []
|
where, args = "", []
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from .abuse_sources import ABUSE as ABUSE_DEEP
|
|||||||
from .sources_tg_abuse import TG_ABUSE
|
from .sources_tg_abuse import TG_ABUSE
|
||||||
from .sources_ext4 import EXTRA5
|
from .sources_ext4 import EXTRA5
|
||||||
from .sources_ext5 import EXTRA6
|
from .sources_ext5 import EXTRA6
|
||||||
|
from .sources_fresh import fresh as _fresh_sources
|
||||||
from .sources_x import X_SOURCES
|
from .sources_x import X_SOURCES
|
||||||
|
|
||||||
|
|
||||||
@@ -298,7 +299,16 @@ def _gen_sources():
|
|||||||
|
|
||||||
|
|
||||||
def all_sources():
|
def all_sources():
|
||||||
"""Реестр + пользовательские источники из панели, без дублей по url."""
|
"""Реестр + пользовательские источники из панели, без дублей по url и по имени.
|
||||||
|
|
||||||
|
Дедуп по двум ключам обязателен:
|
||||||
|
* url — одна площадка не должна обходиться дважды (двойные записи в базе);
|
||||||
|
* name — имя источника является первичным ключом в source_flags/runs, поэтому
|
||||||
|
зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
|
||||||
|
www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
|
||||||
|
Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна →
|
||||||
|
пользовательские → abuse-radar → panels.json.
|
||||||
|
"""
|
||||||
from . import source_state
|
from . import source_state
|
||||||
extra = []
|
extra = []
|
||||||
for d in source_state.custom():
|
for d in source_state.custom():
|
||||||
@@ -307,19 +317,20 @@ def all_sources():
|
|||||||
d = {k: v for k, v in d.items() if k in
|
d = {k: v for k, v in d.items() if k in
|
||||||
("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
|
("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
|
||||||
extra.append(Source(**d))
|
extra.append(Source(**d))
|
||||||
seen, out = set(), []
|
seen_url, seen_name, out = set(), set(), []
|
||||||
for s in SOURCES + _gen_sources() + extra + radar_sources() + panel_sources():
|
for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources():
|
||||||
# дедуп строго по url: одна площадка = один источник, даже если пришла из
|
|
||||||
# реестра, radar_sources и panels.json одновременно (иначе двойной обход и
|
|
||||||
# двойные записи в базе). Побеждает первое вхождение — порядок списка выше.
|
|
||||||
key = (s.url or "").rstrip("/").lower()
|
key = (s.url or "").rstrip("/").lower()
|
||||||
if key and key in seen:
|
name = (s.name or "").strip().lower()
|
||||||
|
if (key and key in seen_url) or (name and name in seen_name):
|
||||||
continue
|
continue
|
||||||
if key:
|
if key:
|
||||||
seen.add(key)
|
seen_url.add(key)
|
||||||
|
if name:
|
||||||
|
seen_name.add(name)
|
||||||
out.append(s)
|
out.append(s)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def enabled_sources(include_opt_in=False):
|
def enabled_sources(include_opt_in=False):
|
||||||
"""Только включённые: выключенные в панели источники не дёргаем.
|
"""Только включённые: выключенные в панели источники не дёргаем.
|
||||||
|
|
||||||
|
|||||||
@@ -71,4 +71,22 @@ ABUSE_LIVE = [
|
|||||||
A("ab_hn_account_generator", "abuse", "site", "https://hn.algolia.com/?query=account+generator", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "account generator"}),
|
A("ab_hn_account_generator", "abuse", "site", "https://hn.algolia.com/?query=account+generator", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "account generator"}),
|
||||||
A("ab_hn_api_key_leak", "abuse", "site", "https://hn.algolia.com/?query=api+key+leak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "api key leak"}),
|
A("ab_hn_api_key_leak", "abuse", "site", "https://hn.algolia.com/?query=api+key+leak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "api key leak"}),
|
||||||
A("ab_hn_chatgpt_free", "abuse", "site", "https://hn.algolia.com/?query=chatgpt+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "chatgpt free"}),
|
A("ab_hn_chatgpt_free", "abuse", "site", "https://hn.algolia.com/?query=chatgpt+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "chatgpt free"}),
|
||||||
|
|
||||||
|
# ---- партия 27.10: проверенные вручную (каждый отдаёт >0 items) ----
|
||||||
|
A("ab_tg_gpt_frees", "abuse", "site", "https://t.me/s/gpt_frees", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_promokody", "abuse", "site", "https://t.me/s/promokody", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_promokodq", "abuse", "site", "https://t.me/s/promokodq", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_cracksoftwux", "abuse", "site", "https://t.me/s/crackSoftiowux", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_taynikstore", "abuse", "site", "https://t.me/s/taynikstoree", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_nodelocrus", "abuse", "site", "https://t.me/s/NODELOCRUS", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_aivision", "abuse", "site", "https://t.me/s/a_vision_ai", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_tg_sms_free", "abuse", "site", "https://t.me/s/freenumbers", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
|
||||||
|
A("ab_reddit_freebies2", "abuse", "site", "https://www.reddit.com/r/freebies/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20),
|
||||||
|
A("ab_reddit_beermoney", "abuse", "site", "https://www.reddit.com/r/beermoney/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20),
|
||||||
|
A("ab_dis_n8n", "abuse", "site", "https://community.n8n.io/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||||
|
A("ab_dis_vercel", "abuse", "site", "https://community.vercel.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||||
|
A("ab_rss_slickdeals2", "abuse", "site", "https://slickdeals.net/newsearch.php?mode=frontpage&searcharea=deals&searchin=first&rss=1", "rss", ('abuse', 'abuse'), limit=20),
|
||||||
|
A("ab_gh_freeapi_keys2", "abuse", "site", "https://github.com/search?q=free+api+keys+pool&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free api keys pool", "sort": "updated"}),
|
||||||
|
A("ab_gh_leak_pool2", "abuse", "site", "https://github.com/search?q=leaked+api+keys+checker&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "leaked api keys checker", "sort": "updated"}),
|
||||||
|
A("ab_hn_free_tokens2", "abuse", "site", "https://hn.algolia.com/?query=free+tokens+api", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "free tokens api"}),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -5,6 +5,16 @@
|
|||||||
Импортов из omni.sources нет намеренно (циклический импорт).
|
Импортов из omni.sources нет намеренно (циклический импорт).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import datetime as _dt
|
||||||
|
|
||||||
|
|
||||||
|
def _nvd_week():
|
||||||
|
"""NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""
|
||||||
|
now = _dt.datetime.utcnow()
|
||||||
|
since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
|
||||||
|
return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"
|
||||||
|
"&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))
|
||||||
|
|
||||||
|
|
||||||
def fresh(S):
|
def fresh(S):
|
||||||
"""S — конструктор Source; возвращает только проверенные сетью источники."""
|
"""S — конструктор Source; возвращает только проверенные сетью источники."""
|
||||||
@@ -33,7 +43,6 @@ def fresh(S):
|
|||||||
S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
|
S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
|
||||||
S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
|
S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
|
||||||
S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
|
S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
|
||||||
S("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research")),
|
|
||||||
S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
|
S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
|
||||||
S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
|
S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
|
||||||
S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
|
S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
|
||||||
@@ -45,6 +54,8 @@ def fresh(S):
|
|||||||
S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
|
S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
|
||||||
S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
|
S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
|
||||||
S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
|
S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
|
||||||
|
S("nvd_exploited_kw", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", "nvd", ("cve", "exploited")),
|
||||||
|
S("nvd_recent", "security", "json", _nvd_week(), "nvd", ("cve", "nvd")),
|
||||||
S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
|
S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
|
||||||
S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
|
S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
|
||||||
S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
|
S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),
|
||||||
|
|||||||
207
tests/test_panel_api.py
Normal file
207
tests/test_panel_api.py
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
"""Панель v2: API-слой panel_api + регистрация роутов в serve. Без сети."""
|
||||||
|
import inspect
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _tmp_item(tag=None):
|
||||||
|
from omni import store
|
||||||
|
from omni.model import Item
|
||||||
|
|
||||||
|
tag = tag or str(time.time_ns())
|
||||||
|
it = Item(provider="pytest", kind="post", source="pytest-panel",
|
||||||
|
title="pytest panel %s" % tag, text="текст <b>%s</b> & xss" % tag,
|
||||||
|
url="https://example.invalid/p/%s" % tag, published=time.strftime("%Y-%m-%d"))
|
||||||
|
store.add([it])
|
||||||
|
with store.db() as c:
|
||||||
|
row = c.execute("SELECT id FROM items WHERE url=?", (it.url,)).fetchone()
|
||||||
|
return int(row[0]), tag
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- timeline ----------
|
||||||
|
def test_timeline_shape_and_modes():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
for mode in ("pub", "seen"):
|
||||||
|
d = panel_api.timeline(days=7, mode=mode)
|
||||||
|
assert d["ok"] and d["mode"] == mode
|
||||||
|
assert len(d["points"]) == 7, "ровно N дней, пустые заполнены нулями"
|
||||||
|
assert d["total"] == sum(p["n"] for p in d["points"])
|
||||||
|
for p in d["points"]:
|
||||||
|
assert set(p) == {"day", "n", "cve", "sev"}
|
||||||
|
assert p["cve"] <= p["n"] and p["sev"] <= p["n"]
|
||||||
|
assert "last24" in panel_api.timeline(days=2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_timeline_days_clamped():
|
||||||
|
"""0/отрицательное → 1 день, None → 30, огромное → потолок 365."""
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
assert len(panel_api.timeline(days=None)["points"]) == 30
|
||||||
|
assert len(panel_api.timeline(days=0)["points"]) == 1
|
||||||
|
assert len(panel_api.timeline(days=-5)["points"]) == 1
|
||||||
|
assert len(panel_api.timeline(days=10_000)["points"]) == 365
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- детали записи ----------
|
||||||
|
def test_item_detail_cleans_html_and_finds_siblings():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
iid, tag = _tmp_item()
|
||||||
|
d = panel_api.item_detail(iid)
|
||||||
|
assert d["ok"]
|
||||||
|
it = d["item"]
|
||||||
|
text = it["text"]
|
||||||
|
assert "<b>" not in text and "<" not in text, "сырые теги остались"
|
||||||
|
assert "&" not in text, "двойное экранирование"
|
||||||
|
assert tag in text
|
||||||
|
assert isinstance(it["related_same_source"], list)
|
||||||
|
assert all(r["source"] == "pytest-panel" for r in it["related_same_source"])
|
||||||
|
assert panel_api.item_detail(10 ** 12) == {"error": "not found"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- подсказки ----------
|
||||||
|
def test_suggest_needs_two_chars_and_finds_source():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
assert panel_api.suggest("a") == {"titles": [], "sources": [], "categories": []}
|
||||||
|
d = panel_api.suggest("firecrawl")
|
||||||
|
assert isinstance(d["titles"], list) and isinstance(d["sources"], list)
|
||||||
|
if d["sources"]:
|
||||||
|
assert {"name", "category"} <= set(d["sources"][0])
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- watchlist ----------
|
||||||
|
def test_watch_add_list_remove():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
q = "pytest-watch-%d" % time.time_ns()
|
||||||
|
r = panel_api.watch_add(q)
|
||||||
|
assert r["ok"] and r["rule"]["q"] == q
|
||||||
|
rid = r["rule"]["id"]
|
||||||
|
try:
|
||||||
|
rules = panel_api.watch_list()["rules"]
|
||||||
|
assert any(x["id"] == rid for x in rules)
|
||||||
|
assert panel_api.watch_remove(rid)["ok"]
|
||||||
|
assert not any(x["id"] == rid for x in panel_api.watch_list()["rules"])
|
||||||
|
finally:
|
||||||
|
panel_api.watch_remove(rid)
|
||||||
|
assert "error" in panel_api.watch_add("x")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- детали источника ----------
|
||||||
|
def test_source_detail_shape():
|
||||||
|
from omni import panel_api, sources
|
||||||
|
|
||||||
|
s = sources.all_sources()[0]
|
||||||
|
d = panel_api.source_detail(s.name)
|
||||||
|
assert d["ok"] and d["source"]["name"] == s.name
|
||||||
|
for k in ("category", "adapter", "kind", "url", "tags", "limit"):
|
||||||
|
assert k in d["source"], "нет поля source." + k
|
||||||
|
for k in ("state", "stored", "latest", "runs"):
|
||||||
|
assert k in d, "нет поля " + k
|
||||||
|
assert isinstance(d["latest"], list) and isinstance(d["runs"], list)
|
||||||
|
assert d["stored"] >= 0
|
||||||
|
assert panel_api.source_detail("нет-такого-источника") == {"error": "not found"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- health ----------
|
||||||
|
def test_health_counts():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
d = panel_api.health()
|
||||||
|
assert d["ok"]
|
||||||
|
assert d["items"] > 0 and d["sources"] > 0
|
||||||
|
assert {"alive", "runs", "uptime", "db_mb"} <= set(d)
|
||||||
|
assert 0 <= d["alive"] <= d["sources"], "живых не может быть больше, чем всех"
|
||||||
|
assert d["db_mb"] >= 0 and d["uptime"] >= 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- tg ----------
|
||||||
|
def test_tg_status_shape_and_flag():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
d = panel_api.tg_status()
|
||||||
|
assert {"enabled", "me", "error", "limit"} <= set(d)
|
||||||
|
assert isinstance(d["enabled"], bool)
|
||||||
|
if d["enabled"]:
|
||||||
|
assert d["session"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- digest ----------
|
||||||
|
def test_digest_shape_and_day_window():
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
d = panel_api.digest(days=3, limit=10)
|
||||||
|
assert d["ok"] and d["days"] == 3 and d["n"] >= 0
|
||||||
|
md = d["md"]
|
||||||
|
assert md.startswith("# omni · дайджест")
|
||||||
|
for head in ("## Главное по score", "## Уязвимости", "## Сливы и утечки", "## Абузы"):
|
||||||
|
assert head in md, "в дайджесте нет раздела " + head
|
||||||
|
p = Path(d["path"])
|
||||||
|
assert p.exists() and p.read_text(encoding="utf-8").startswith("# omni · дайджест")
|
||||||
|
# clamp: 0 → 1 день, 999 → 30
|
||||||
|
assert panel_api.digest(days=0)["days"] == 1
|
||||||
|
assert panel_api.digest(days=999)["days"] == 30
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- export ----------
|
||||||
|
def test_export_respects_filters_and_cap():
|
||||||
|
from omni import panel_api
|
||||||
|
|
||||||
|
d = panel_api.export_json(q="firecrawl", limit=5)
|
||||||
|
assert d["ok"] and d["count"] == len(d["items"]) <= 5
|
||||||
|
assert d["total"] >= d["count"], "total — это всё по фильтру, count — срез"
|
||||||
|
for it in d["items"]:
|
||||||
|
assert {"id", "title", "url", "source", "published", "types",
|
||||||
|
"price", "currency", "score"} <= set(it)
|
||||||
|
d2 = panel_api.export_json(typ="leak", limit=3)
|
||||||
|
assert d2["ok"] and all("leak" in (it["types"] or "") for it in d2["items"])
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- serve: роуты зарегистрированы ----------
|
||||||
|
def test_serve_registers_panel_routes():
|
||||||
|
from omni import serve
|
||||||
|
|
||||||
|
src = inspect.getsource(serve)
|
||||||
|
for path in ("/api/item/", "/api/timeline", "/api/suggest", "/api/health",
|
||||||
|
"/api/tg/status", "/api/tg/search", "/api/tg/pull", "/api/source",
|
||||||
|
"/api/watch", "/api/digest", "/api/export.json"):
|
||||||
|
assert path in src, "роут не зарегистрирован: " + path
|
||||||
|
assert "browse2" in src, "browse2 не используется в serve"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- web: панель v2 содержит новые разделы ----------
|
||||||
|
def test_web_panel_v2_has_new_sections():
|
||||||
|
html = (ROOT / "web" / "index.html").read_text(encoding="utf-8")
|
||||||
|
for frag in ('id:"overview"', 'id:"feed"', 'id:"abuse"', 'id:"sources"', 'id:"runs"',
|
||||||
|
'id:"tg"', 'id:"keys"', 'id:"digest"', 'id:"watch"',
|
||||||
|
"/api/timeline", "/api/tg/status", "/api/suggest", "watchlist",
|
||||||
|
"/api/export.json"):
|
||||||
|
assert frag in html, "в панели нет " + frag
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- store.browse2: since/src ----------
|
||||||
|
def test_browse2_filters_since_and_src():
|
||||||
|
from omni import store
|
||||||
|
|
||||||
|
rows, total = store.browse2(q="", limit=5, sort="seen", since="2026-01-01")
|
||||||
|
assert isinstance(rows, list) and total >= len(rows)
|
||||||
|
rows2, total2 = store.browse2(q="", limit=5, sort="seen", since="2099-01-01")
|
||||||
|
assert total2 == 0 and rows2 == [], "since не применился"
|
||||||
|
rows3, _ = store.browse2(q="", limit=200, sort="seen")
|
||||||
|
assert rows3, "нет строк для проверки src-фильтра"
|
||||||
|
name = rows3[0]["source"]
|
||||||
|
rows4, total4 = store.browse2(q="", limit=200, src=name)
|
||||||
|
assert total4 > 0 and all(x["source"] == name for x in rows4), "src не применился"
|
||||||
|
# src + since вместе
|
||||||
|
rows5, total5 = store.browse2(q="", limit=5, src=name, since="2099-01-01")
|
||||||
|
assert total5 == 0 and rows5 == []
|
||||||
79
tools/check_candidates.py
Normal file
79
tools/check_candidates.py
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
"""Проверка кандидатов в источники абузов: сколько items реально отдаёт адаптер."""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, ".")
|
||||||
|
|
||||||
|
from omni import feeds
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
CANDS = [
|
||||||
|
# TG-превью
|
||||||
|
("ab_tg_free_apis", "tg_web", "https://t.me/s/free_apis", {}),
|
||||||
|
("ab_tg_besplatno_chatgpt", "tg_web", "https://t.me/s/besplatno_chatgpt", {}),
|
||||||
|
("ab_tg_gpt_frees", "tg_web", "https://t.me/s/gpt_frees", {}),
|
||||||
|
("ab_tg_promokody", "tg_web", "https://t.me/s/promokody", {}),
|
||||||
|
("ab_tg_promokodq", "tg_web", "https://t.me/s/promokodq", {}),
|
||||||
|
("ab_tg_free_softiki", "tg_web", "https://t.me/s/free_softiki", {}),
|
||||||
|
("ab_tg_baza_dannykh", "tg_web", "https://t.me/s/baza_dannykh_ru", {}),
|
||||||
|
("ab_tg_donzo", "tg_web", "https://t.me/s/virtualnyydondon_aktivatsiadonzo", {}),
|
||||||
|
("ab_tg_cracksoftiki", "tg_web", "https://t.me/s/cracksoftiki", {}),
|
||||||
|
("ab_tg_cracksoftwux", "tg_web", "https://t.me/s/crackSoftiowux", {}),
|
||||||
|
("ab_tg_taynikstore", "tg_web", "https://t.me/s/taynikstoree", {}),
|
||||||
|
("ab_tg_nodelocrus", "tg_web", "https://t.me/s/NODELOCRUS", {}),
|
||||||
|
("ab_tg_aivision", "tg_web", "https://t.me/s/a_vision_ai", {}),
|
||||||
|
("ab_tg_freeapi_chat", "tg_web", "https://t.me/s/freeapi_chat", {}),
|
||||||
|
("ab_tg_vpn_free", "tg_web", "https://t.me/s/vpn_free_servers", {}),
|
||||||
|
("ab_tg_sms_free", "tg_web", "https://t.me/s/freenumbers", {}),
|
||||||
|
# reddit
|
||||||
|
("ab_reddit_freebies2", "reddit_listing", "https://www.reddit.com/r/freebies/.json?limit=25", {}),
|
||||||
|
("ab_reddit_selfhosted", "reddit_listing", "https://www.reddit.com/r/selfhosted/.json?limit=25", {}),
|
||||||
|
("ab_reddit_localllama", "reddit_listing", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", {}),
|
||||||
|
("ab_reddit_chatgptcoding", "reddit_listing", "https://www.reddit.com/r/ChatGPTCoding/.json?limit=25", {}),
|
||||||
|
("ab_reddit_beermoney", "reddit_listing", "https://www.reddit.com/r/beermoney/.json?limit=25", {}),
|
||||||
|
("ab_reddit_sideloaded", "reddit_listing", "https://www.reddit.com/r/sideloaded/.json?limit=25", {}),
|
||||||
|
("ab_reddit_piracy2", "reddit_listing", "https://www.reddit.com/r/Piracy/.json?limit=25", {}),
|
||||||
|
("ab_reddit_freegame", "reddit_listing", "https://www.reddit.com/r/FreeGameFindings/.json?limit=25", {}),
|
||||||
|
# discourse
|
||||||
|
("ab_dis_n8n", "discourse", "https://community.n8n.io/latest.json", {}),
|
||||||
|
("ab_dis_jina", "discourse", "https://discourse.jina.ai/latest.json", {}),
|
||||||
|
("ab_dis_cloudflare", "discourse", "https://community.cloudflare.com/latest.json", {}),
|
||||||
|
("ab_dis_vercel", "discourse", "https://community.vercel.com/latest.json", {}),
|
||||||
|
# rss
|
||||||
|
("ab_rss_slickdeals2", "rss", "https://slickdeals.net/newsearch.php?mode=frontpage&searcharea=deals&searchin=first&rss=1", {}),
|
||||||
|
("ab_rss_dealnews", "rss", "https://www.dealnews.com/rss/", {}),
|
||||||
|
("ab_rss_freeStuffTimes", "rss", "https://feeds.feedburner.com/free-stuff-times", {}),
|
||||||
|
("ab_rss_humble", "rss", "https://www.humblebundle.com/rss", {}),
|
||||||
|
# sitemap
|
||||||
|
("ab_sitemap_cracked", "sitemap_recent", "https://cracked.io/sitemap.xml", {"days": 30}),
|
||||||
|
("ab_sitemap_nulled", "sitemap_recent", "https://nulled.to/sitemap.xml", {"days": 30}),
|
||||||
|
# github / hn
|
||||||
|
("ab_gh_freeapi_keys", "github_search", "https://api.github.com/search/repositories", {"q": "free api keys pool", "sort": "updated"}),
|
||||||
|
("ab_gh_leak_pool", "github_search", "https://api.github.com/search/repositories", {"q": "leaked api keys checker", "sort": "updated"}),
|
||||||
|
("ab_hn_free_tokens", "hn_search", "https://hn.algolia.com/api/v1/search_by_date", {"q": "free tokens api"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for name, adapter, url, extra in CANDS:
|
||||||
|
fn = feeds.ADAPTERS.get(adapter)
|
||||||
|
if not fn:
|
||||||
|
out.append((name, adapter, -1, "нет адаптера"))
|
||||||
|
continue
|
||||||
|
src = SimpleNamespace(name=name, category="abuse", kind="site", url=url,
|
||||||
|
adapter=adapter, tags=("abuse",), limit=8, detail=False,
|
||||||
|
extra=dict(extra))
|
||||||
|
try:
|
||||||
|
items = fn(src, 8)
|
||||||
|
n = len(items)
|
||||||
|
sample = (items[0].title or "")[:60] if items else ""
|
||||||
|
except Exception as e:
|
||||||
|
n, sample = -2, "ERR %s" % str(e)[:60]
|
||||||
|
out.append((name, adapter, n, sample))
|
||||||
|
|
||||||
|
ok = [x for x in out if x[2] > 0]
|
||||||
|
print("проверено:", len(out), "| живых:", len(ok))
|
||||||
|
for name, adapter, n, sample in out:
|
||||||
|
print("%-28s %-16s %3s %s" % (name, adapter, n, sample))
|
||||||
|
json.dump([{"name": n, "adapter": a, "url": u, "extra": e, "items": c}
|
||||||
|
for (n, a, c, _), (_, _, u, e) in zip(out, CANDS)], open("data/_cand_checked.json", "w", encoding="utf-8"),
|
||||||
|
ensure_ascii=False, indent=1)
|
||||||
@@ -7,9 +7,11 @@
|
|||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
|
|
||||||
def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
|
def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None, **kw):
|
||||||
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
row = {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
||||||
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
||||||
|
row.update(kw)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
_NOW = datetime.datetime.utcnow()
|
_NOW = datetime.datetime.utcnow()
|
||||||
@@ -35,7 +37,8 @@ CANDIDATES = [
|
|||||||
d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
|
d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")),
|
||||||
d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
|
d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
|
||||||
d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
|
d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
|
||||||
d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
|
# elastic_security_labs — в реестре уже есть https://www.elastic.co/security-labs/rss/feed.xml
|
||||||
|
|
||||||
d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
|
d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
|
||||||
d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
|
d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
|
||||||
d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
|
d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
|
||||||
@@ -56,9 +59,10 @@ CANDIDATES = [
|
|||||||
d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
|
d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)),
|
||||||
|
|
||||||
# ---------------- 2. CVE / vulnerabilities ----------------
|
# ---------------- 2. CVE / vulnerabilities ----------------
|
||||||
d("nvd_recent", "security", "json",
|
# nvd_recent: окно last-modified считается в момент вызова (см. emit -> dynamic),
|
||||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
|
# статическая дата протухает уже через неделю.
|
||||||
% (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")),
|
d("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0", "nvd",
|
||||||
|
("cve", "nvd"), dynamic="NVD_LAST_WEEK"),
|
||||||
d("nvd_exploited_kw", "security", "json",
|
d("nvd_exploited_kw", "security", "json",
|
||||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
|
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
|
||||||
"nvd", ("cve", "exploited")),
|
"nvd", ("cve", "exploited")),
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ def probe_one(src, limit):
|
|||||||
if not items and not err:
|
if not items and not err:
|
||||||
err = _diag(src)
|
err = _diag(src)
|
||||||
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
||||||
"url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items),
|
"url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items), "dynamic": src.extra.get("_dynamic", ""),
|
||||||
"err": err, "ms": ms,
|
"err": err, "ms": ms,
|
||||||
"samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
|
"samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
|
||||||
|
|
||||||
@@ -69,11 +69,11 @@ def _norm(u):
|
|||||||
|
|
||||||
def _registry_urls():
|
def _registry_urls():
|
||||||
from omni.sources import all_sources
|
from omni.sources import all_sources
|
||||||
out = set()
|
return {_norm(s.url) for s in all_sources()}
|
||||||
for s in all_sources():
|
|
||||||
out.add(_norm(s.url))
|
|
||||||
out.add(_norm(s.url).split("?")[0])
|
# контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же)
|
||||||
return out
|
CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"}
|
||||||
|
|
||||||
|
|
||||||
def emit(rows, dead, path, dedupe=True):
|
def emit(rows, dead, path, dedupe=True):
|
||||||
@@ -84,9 +84,9 @@ def emit(rows, dead, path, dedupe=True):
|
|||||||
if dedupe:
|
if dedupe:
|
||||||
have = _registry_urls()
|
have = _registry_urls()
|
||||||
kept, skipped = [], []
|
kept, skipped = [], []
|
||||||
for r in rows:
|
for r in [x for x in rows if x["name"] not in CONTENT_DUP]:
|
||||||
n = _norm(r["url"])
|
n = _norm(r["url"])
|
||||||
(skipped if (n in have or n.split("?")[0] in have) else kept).append(r)
|
(skipped if n in have else kept).append(r)
|
||||||
print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
|
print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped)))
|
||||||
rows = kept
|
rows = kept
|
||||||
|
|
||||||
@@ -98,6 +98,16 @@ def emit(rows, dead, path, dedupe=True):
|
|||||||
"Импортов из omni.sources нет намеренно (циклический импорт).",
|
"Импортов из omni.sources нет намеренно (циклический импорт).",
|
||||||
'"""',
|
'"""',
|
||||||
"",
|
"",
|
||||||
|
"import datetime as _dt",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"def _nvd_week():",
|
||||||
|
' """NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""',
|
||||||
|
" now = _dt.datetime.utcnow()",
|
||||||
|
' since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")',
|
||||||
|
' return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"',
|
||||||
|
' "&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))',
|
||||||
|
"",
|
||||||
"",
|
"",
|
||||||
"def fresh(S):",
|
"def fresh(S):",
|
||||||
' """S — конструктор Source; возвращает только проверенные сетью источники."""',
|
' """S — конструктор Source; возвращает только проверенные сетью источники."""',
|
||||||
@@ -111,6 +121,10 @@ def emit(rows, dead, path, dedupe=True):
|
|||||||
lim = "" if limit == 15 else ", limit=%d" % limit
|
lim = "" if limit == 15 else ", limit=%d" % limit
|
||||||
det = ", detail=True" if r.get("detail") else ""
|
det = ", detail=True" if r.get("detail") else ""
|
||||||
tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
|
tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
|
||||||
|
if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent":
|
||||||
|
lines.append(' S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % (
|
||||||
|
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det))
|
||||||
|
continue
|
||||||
lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
|
lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
|
||||||
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
|
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
|
||||||
tags, lim, det, extra))
|
tags, lim, det, extra))
|
||||||
@@ -147,7 +161,9 @@ def main():
|
|||||||
srcs = []
|
srcs = []
|
||||||
for c in cands:
|
for c in cands:
|
||||||
try:
|
try:
|
||||||
srcs.append(Source(**c))
|
cc = dict(c)
|
||||||
|
cc.pop("dynamic", None)
|
||||||
|
srcs.append(Source(**cc))
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
print("skip %s: %s" % (c.get("name"), e))
|
print("skip %s: %s" % (c.get("name"), e))
|
||||||
|
|
||||||
|
|||||||
57
tools/run_fresh.py
Normal file
57
tools/run_fresh.py
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
"""Прогон свежей волны источников: фактическая отдача и сколько легло в базу.
|
||||||
|
|
||||||
|
python tools/run_fresh.py [--limit 10] [--workers 8]
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||||||
|
|
||||||
|
from omni import feeds, rank, sources, source_state, store
|
||||||
|
from omni.sources_fresh import fresh as _fresh
|
||||||
|
|
||||||
|
|
||||||
|
class _Fake:
|
||||||
|
def __init__(self, *a, **k):
|
||||||
|
self.name = a[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--limit", type=int, default=10)
|
||||||
|
ap.add_argument("--workers", type=int, default=8)
|
||||||
|
ap.add_argument("--out", default="data/fresh_run.json")
|
||||||
|
a = ap.parse_args()
|
||||||
|
|
||||||
|
names = {f.name for f in _fresh(_Fake)}
|
||||||
|
picked = [s for s in sources.all_sources() if s.name in names]
|
||||||
|
print("источников свежей волны:", len(picked), "| лимит", a.limit)
|
||||||
|
t0 = time.time()
|
||||||
|
items, report = feeds.fetch_many(picked, limit=a.limit, workers=a.workers)
|
||||||
|
items = rank.rank(items)
|
||||||
|
new, dup = store.add(items)
|
||||||
|
rep = dict(report)
|
||||||
|
ok = {k: v for k, v in rep.items() if v}
|
||||||
|
bad = sorted(k for k, v in rep.items() if not v)
|
||||||
|
by_src = {}
|
||||||
|
for it in items:
|
||||||
|
by_src[it.source] = by_src.get(it.source, 0) + 1
|
||||||
|
# состояние реестра: что реально отдало записи
|
||||||
|
for name, n in rep.items():
|
||||||
|
source_state.record_check(name, 200 if n else 0, n, "ok" if n else "пусто при проверке")
|
||||||
|
out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked),
|
||||||
|
"alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup,
|
||||||
|
"by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad}
|
||||||
|
json.dump(out, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||||||
|
print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad))
|
||||||
|
print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup)
|
||||||
|
print("топ:", list(out["by_source"].items())[:12])
|
||||||
|
if bad:
|
||||||
|
print("пустые:", ", ".join(bad))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
16
tools/shot_abuse.py
Normal file
16
tools/shot_abuse.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
"""Скриншот вкладки «Абузы» панели omni-parse."""
|
||||||
|
import sys
|
||||||
|
from camoufox.sync_api import Camoufox
|
||||||
|
|
||||||
|
OUT = sys.argv[1] if len(sys.argv) > 1 else "data/dash_abuse.png"
|
||||||
|
URL = "http://127.0.0.1:8078/#/abuse"
|
||||||
|
|
||||||
|
with Camoufox(headless=True, geoip=False, humanize=True) as b:
|
||||||
|
p = b.new_page(viewport={"width": 1400, "height": 1000})
|
||||||
|
p.goto(URL, wait_until="domcontentloaded", timeout=60000)
|
||||||
|
p.wait_for_selector("#view", timeout=60000)
|
||||||
|
p.wait_for_timeout(7000)
|
||||||
|
txt = p.inner_text("#view")[:900]
|
||||||
|
p.screenshot(path=OUT, full_page=False)
|
||||||
|
print(txt)
|
||||||
|
print("--- saved:", OUT)
|
||||||
@@ -7,7 +7,7 @@
|
|||||||
<style>
|
<style>
|
||||||
:root{
|
:root{
|
||||||
--bg:#0a0e13; --bg2:#0c1219; --panel:#111823; --panel2:#0e151e; --line:#1c2735; --line2:#27364a;
|
--bg:#0a0e13; --bg2:#0c1219; --panel:#111823; --panel2:#0e151e; --line:#1c2735; --line2:#27364a;
|
||||||
--text:#e9eff7; --muted:#93a3b8; --dim:#65758b;
|
--text:#e9eff7; --muted:#93a3b8; --dim:#7b8ba1;
|
||||||
--acc:#37d67a; --acc2:#5aa9ff; --warn:#f5b544; --bad:#ef5b5b; --violet:#a78bfa;
|
--acc:#37d67a; --acc2:#5aa9ff; --warn:#f5b544; --bad:#ef5b5b; --violet:#a78bfa;
|
||||||
--r:10px; --r2:14px;
|
--r:10px; --r2:14px;
|
||||||
--mono:"Cascadia Mono",Consolas,"SF Mono",monospace;
|
--mono:"Cascadia Mono",Consolas,"SF Mono",monospace;
|
||||||
@@ -105,6 +105,15 @@ nav{display:flex;flex-direction:column;gap:1px;padding:6px 8px;overflow:auto}
|
|||||||
.chip.tool{background:#132028;color:#8fd8e8;border-color:#234450}
|
.chip.tool{background:#132028;color:#8fd8e8;border-color:#234450}
|
||||||
.chip.model{background:#1b1630;color:#c2b2f7;border-color:#332a52}
|
.chip.model{background:#1b1630;color:#c2b2f7;border-color:#332a52}
|
||||||
.chip.promo{background:#12251a;color:#9fe6b4;border-color:#245035}
|
.chip.promo{background:#12251a;color:#9fe6b4;border-color:#245035}
|
||||||
|
.chip.news{background:#151f2b;color:#a9bdd2;border-color:#26374b}
|
||||||
|
.chip.trial{background:#221a33;color:#cbb3f2;border-color:#3a2c58}
|
||||||
|
.chip.resale{background:#2a1626;color:#f0a8cf;border-color:#4a2340}
|
||||||
|
.chip.release{background:#12233a;color:#9dc4f5;border-color:#21405f}
|
||||||
|
.chip.guide{background:#12282a;color:#96dede;border-color:#1f4a4c}
|
||||||
|
.chip.dataset{background:#1a2438;color:#a9bdf0;border-color:#2c3c5c}
|
||||||
|
.chip.job{background:#2a2413;color:#ecd08a;border-color:#4a3f1f}
|
||||||
|
.chip.price{background:#12251a;color:#9fe6b4;border-color:#245035}
|
||||||
|
.chip.question{background:#1b2029;color:#b6c2d4;border-color:#2c3746}
|
||||||
.chip.on{background:#12351f;border-color:#1f5c33;color:#b9f5cd}
|
.chip.on{background:#12351f;border-color:#1f5c33;color:#b9f5cd}
|
||||||
.chip.off{background:#211517;border-color:#4a2429;color:#f0a9a9}
|
.chip.off{background:#211517;border-color:#4a2429;color:#f0a9a9}
|
||||||
.chip.click{cursor:pointer}
|
.chip.click{cursor:pointer}
|
||||||
@@ -127,10 +136,10 @@ td .u{color:var(--dim);font-size:11px;display:block;max-width:420px;overflow:hid
|
|||||||
/* feed */
|
/* feed */
|
||||||
.item{border-bottom:1px solid #151f2b;padding:11px 13px;cursor:pointer;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:10px}
|
.item{border-bottom:1px solid #151f2b;padding:11px 13px;cursor:pointer;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:10px}
|
||||||
.item:hover{background:#0f1822}
|
.item:hover{background:#0f1822}
|
||||||
.item .t{font-size:14px;font-weight:600;letter-spacing:.1px;line-height:1.35;margin-bottom:3px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
|
.item .t{font-size:14.5px;font-weight:600;letter-spacing:.1px;line-height:1.35;margin-bottom:3px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
|
||||||
.item .m{display:flex;gap:8px;align-items:center;flex-wrap:wrap;color:var(--dim);font-size:11.5px}
|
.item .m{display:flex;gap:8px;align-items:center;flex-wrap:wrap;color:var(--dim);font-size:11.5px}
|
||||||
.item .src{color:#9db7d4}
|
.item .src{color:#9db7d4}
|
||||||
.item .x{color:var(--muted);font-size:12px;margin-top:4px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
|
.item .x{color:var(--muted);font-size:12.5px;margin-top:5px;display:-webkit-box;-webkit-line-clamp:3;-webkit-box-orient:vertical;overflow:hidden}
|
||||||
.item .side{display:flex;flex-direction:column;align-items:flex-end;gap:6px;justify-content:center}
|
.item .side{display:flex;flex-direction:column;align-items:flex-end;gap:6px;justify-content:center}
|
||||||
.sc{font:11px var(--mono);color:var(--muted);border:1px solid var(--line2);border-radius:6px;padding:1px 5px}
|
.sc{font:11px var(--mono);color:var(--muted);border:1px solid var(--line2);border-radius:6px;padding:1px 5px}
|
||||||
.sc.hi{color:#b9f5cd;border-color:#1f5c33;background:#12251a}
|
.sc.hi{color:#b9f5cd;border-color:#1f5c33;background:#12251a}
|
||||||
@@ -188,6 +197,7 @@ label.chk{display:inline-flex;align-items:center;gap:6px;color:var(--muted);font
|
|||||||
.md code{font-family:var(--mono);font-size:11.5px;background:#132029;padding:1px 5px;border-radius:5px;color:#9fd7c8}
|
.md code{font-family:var(--mono);font-size:11.5px;background:#132029;padding:1px 5px;border-radius:5px;color:#9fd7c8}
|
||||||
.md a{color:#8fc2ff}
|
.md a{color:#8fc2ff}
|
||||||
.pager{display:flex;gap:8px;justify-content:center;padding:14px}
|
.pager{display:flex;gap:8px;justify-content:center;padding:14px}
|
||||||
|
.legend{font-size:11px;color:var(--dim);display:flex;gap:14px;align-items:center;flex-wrap:wrap}
|
||||||
.hint{font-size:11.5px;color:var(--dim)}
|
.hint{font-size:11.5px;color:var(--dim)}
|
||||||
.sr{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0)}
|
.sr{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0)}
|
||||||
@media (max-width:1100px){.g4{grid-template-columns:repeat(2,1fr)}.g21{grid-template-columns:1fr}}
|
@media (max-width:1100px){.g4{grid-template-columns:repeat(2,1fr)}.g21{grid-template-columns:1fr}}
|
||||||
@@ -343,6 +353,14 @@ async function boot(){
|
|||||||
$("#q").placeholder=`Поиск по ${num(st.items)} записям: CVE, ключи, сливы, модели…`;
|
$("#q").placeholder=`Поиск по ${num(st.items)} записям: CVE, ключи, сливы, модели…`;
|
||||||
}catch(e){ $("#health").innerHTML=`<span class="dot bad"></span><span>API недоступен</span>`; }
|
}catch(e){ $("#health").innerHTML=`<span class="dot bad"></span><span>API недоступен</span>`; }
|
||||||
route();
|
route();
|
||||||
|
setInterval(refreshHealth,60000);
|
||||||
|
}
|
||||||
|
async function refreshHealth(){
|
||||||
|
try{
|
||||||
|
const h=await api("/api/health");
|
||||||
|
$("#health").innerHTML=`<span class="dot ${h.alive>0?"":"bad"}"></span><span>${num(h.items)} записей · ${num(h.alive)}/${num(h.sources)} живых · аптайм ${Math.round(h.uptime/60)} мин</span>`;
|
||||||
|
$("#q").placeholder=`Поиск по ${num(h.items)} записям: CVE, ключи, сливы, модели…`;
|
||||||
|
}catch(e){ $("#health").innerHTML=`<span class="dot bad"></span><span>API недоступен</span>`; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ============================= views ============================= */
|
/* ============================= views ============================= */
|
||||||
@@ -354,12 +372,12 @@ V.overview=async()=>{
|
|||||||
box.innerHTML=`<div class="grid g4">${Array(4).fill(`<div class="card"><div class="skel" style="width:40%"></div><div class="skel" style="width:70%;height:26px;margin-top:10px"></div><div class="skel" style="width:55%;margin-top:10px"></div></div>`).join("")}</div>
|
box.innerHTML=`<div class="grid g4">${Array(4).fill(`<div class="card"><div class="skel" style="width:40%"></div><div class="skel" style="width:70%;height:26px;margin-top:10px"></div><div class="skel" style="width:55%;margin-top:10px"></div></div>`).join("")}</div>
|
||||||
<div class="grid g21" style="margin-top:14px"><div class="card"><div class="skel" style="height:130px"></div></div><div class="card"><div class="skel" style="height:130px"></div></div></div>`;
|
<div class="grid g21" style="margin-top:14px"><div class="card"><div class="skel" style="height:130px"></div></div><div class="card"><div class="skel" style="height:130px"></div></div></div>`;
|
||||||
const [st,tl,runs,watch,ab]=await Promise.all([
|
const [st,tl,runs,watch,ab]=await Promise.all([
|
||||||
api("/api/stats"), api("/api/timeline?days=30"), api("/api/runs?limit=6"),
|
api("/api/stats"), api("/api/timeline?days=30&mode=seen"), api("/api/runs?limit=6"),
|
||||||
api("/api/watch").catch(()=>({rules:[]})), api("/api/abuse?limit=8").catch(()=>({items:[]}))
|
api("/api/watch").catch(()=>({rules:[]})), api("/api/abuse?limit=8").catch(()=>({items:[]}))
|
||||||
]);
|
]);
|
||||||
S.stats=st;
|
S.stats=st;
|
||||||
const day=tl.points[tl.points.length-1]?.n||0, prev=tl.points[tl.points.length-2]?.n||0;
|
const day=tl.last24||tl.points[tl.points.length-1]?.n||0, prev=tl.points[tl.points.length-2]?.n||0;
|
||||||
const dPct=prev?Math.round((day-prev)/prev*100):0;
|
const dPct=prev?Math.round((day-prev)/prev*100):null;
|
||||||
const live=(st.sources?(st.alive/st.sources*100):0).toFixed(0);
|
const live=(st.sources?(st.alive/st.sources*100):0).toFixed(0);
|
||||||
const okRate=st.runs?(st.runs_ok/st.runs*100).toFixed(0):0;
|
const okRate=st.runs?(st.runs_ok/st.runs*100).toFixed(0):0;
|
||||||
const kpi=(label,val,sub,accent)=>`<div class="card hov"><div class="sub">${label}</div>
|
const kpi=(label,val,sub,accent)=>`<div class="card hov"><div class="sub">${label}</div>
|
||||||
@@ -372,16 +390,17 @@ V.overview=async()=>{
|
|||||||
<button class="btn ${S.auto?"primary":""}" data-act="auto">${S.auto?"Автообновление вкл":"Автообновление выкл"}</button>
|
<button class="btn ${S.auto?"primary":""}" data-act="auto">${S.auto?"Автообновление вкл":"Автообновление выкл"}</button>
|
||||||
</div></div>
|
</div></div>
|
||||||
<div class="grid g4">
|
<div class="grid g4">
|
||||||
${kpi("Записей всего",num(st.items),`+${num(day)} за последний день <span class="${dPct>=0?"up":"down"}">${dPct>=0?"▲":"▼"} ${Math.abs(dPct)}%</span>`)}
|
${kpi("Записей всего",num(st.items),`+${num(day)} за сутки импорта${dPct===null?"":` <span class="${dPct>=0?"up":"down"}">${dPct>=0?"▲":"▼"} ${Math.abs(dPct)}%</span>`}`)}
|
||||||
${kpi("Источников живых",`${num(st.alive)}<small>/ ${num(st.sources)}</small>`,`${live}% реестра отдают записи`)}
|
${kpi("Источников живых",`${num(st.alive)}<small>/ ${num(st.sources)}</small>`,`${live}% реестра отдают записи`)}
|
||||||
${kpi("Ключей готово",`${num(st.keys?.ready||0)}<small>/ ${num(st.keys?.keys||0)}</small>`,`${num(st.keys?.services||0)} сервисов · ротация и остывание`)}
|
${kpi("Ключей готово",`${num(st.keys?.ready||0)}<small>/ ${num(st.keys?.keys||0)}</small>`,`${num(st.keys?.services||0)} сервисов · ротация и остывание`)}
|
||||||
${kpi("Watchlist",num((watch.rules||[]).length),`${num(hits)} совпадений по правилам`)}
|
${kpi("Watchlist",num((watch.rules||[]).length),`${num(hits)} совпадений по правилам`)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid g21" style="margin-top:14px">
|
<div class="grid g21" style="margin-top:14px">
|
||||||
<div class="card"><h3>Поток за 30 дней <span class="r">${num(tl.total)} записей · пик ${num(tl.max)}</span></h3>
|
<div class="card"><h3>Импорт за 30 дней <span class="r">${num(tl.total)} записей · пик ${num(tl.max)}/день</span></h3>
|
||||||
${chart(tl.points)}
|
${chart(tl.points)}
|
||||||
<div class="chart-x"><span>${dt(tl.points[0]?.day)}</span><span>${dt(tl.points[Math.floor(tl.points.length/2)]?.day)}</span><span>сегодня</span></div>
|
<div class="chart-x"><span>${dt(tl.points[0]?.day)}</span><span>${dt(tl.points[Math.floor(tl.points.length/2)]?.day)}</span><span>сегодня</span></div>
|
||||||
|
<div class="legend" style="margin-top:9px"><span>столбец = записей за день импорта</span><span>наведение — дата и число CVE</span><span>переключатель «сутки» в ленте фильтрует по дате публикации</span></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="card"><h3>Источники по категориям</h3>
|
<div class="card"><h3>Источники по категориям</h3>
|
||||||
${(st.categories||[]).slice(0,8).map(c=>{const r=c.total?Math.round((c.alive||0)/c.total*100):0;
|
${(st.categories||[]).slice(0,8).map(c=>{const r=c.total?Math.round((c.alive||0)/c.total*100):0;
|
||||||
@@ -422,8 +441,8 @@ V.feed=async()=>{
|
|||||||
const box=$("#view");
|
const box=$("#view");
|
||||||
box.innerHTML=`<div class="page-h"><div><h2>Лента</h2><p>Все записи реестра: фильтры по типу, периоду, сортировке; клик — детали и соседи.</p></div>
|
box.innerHTML=`<div class="page-h"><div><h2>Лента</h2><p>Все записи реестра: фильтры по типу, периоду, сортировке; клик — детали и соседи.</p></div>
|
||||||
<div class="right">
|
<div class="right">
|
||||||
<a class="btn sm" href="/api/export.csv?${exQS()}" download>CSV</a>
|
<a class="btn sm" href="/api/export.csv?${exQS()}" download title="Экспорт текущего фильтра, до 1000 записей">CSV</a>
|
||||||
<a class="btn sm" href="/api/export.json?${exQS()}" download>JSON</a>
|
<a class="btn sm" href="/api/export.json?${exQS()}" download title="Экспорт текущего фильтра, до 1000 записей">JSON</a>
|
||||||
</div></div>
|
</div></div>
|
||||||
<div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
|
<div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
|
||||||
<span class="dim" style="font-size:11.5px">Типы:</span>
|
<span class="dim" style="font-size:11.5px">Типы:</span>
|
||||||
@@ -452,6 +471,14 @@ async function loadItems(reset){
|
|||||||
: `<span class="dim">показано ${num(S.items.length)} из ${num(S.total)}</span>`;
|
: `<span class="dim">показано ${num(S.items.length)} из ${num(S.total)}</span>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function findChips(f){
|
||||||
|
if(!f) return "";
|
||||||
|
const p=[];
|
||||||
|
(f.codes||[]).slice(0,2).forEach(c=>p.push(`<span class="chip promo" title="промокод/код">code: ${esc(c)}</span>`));
|
||||||
|
if(f.amount) p.push(`<span class="chip credits">${esc(f.amount)}</span>`);
|
||||||
|
if(f.term) p.push(`<span class="chip invite">${esc(f.term)}</span>`);
|
||||||
|
return p.join("");
|
||||||
|
}
|
||||||
function itemRow(it,compact){
|
function itemRow(it,compact){
|
||||||
const types=(it.types||"").split(",").filter(Boolean).slice(0,3);
|
const types=(it.types||"").split(",").filter(Boolean).slice(0,3);
|
||||||
const labels=typeLabels();
|
const labels=typeLabels();
|
||||||
@@ -463,7 +490,7 @@ function itemRow(it,compact){
|
|||||||
<div class="t">${esc(it.title||"(без заголовка)")}</div>
|
<div class="t">${esc(it.title||"(без заголовка)")}</div>
|
||||||
<div class="m"><span class="src">${esc(it.source||"")}</span>
|
<div class="m"><span class="src">${esc(it.source||"")}</span>
|
||||||
${types.map(t=>`<span class="chip t ${t}">${esc(labels[t]||t)}</span>`).join("")}
|
${types.map(t=>`<span class="chip t ${t}">${esc(labels[t]||t)}</span>`).join("")}
|
||||||
${price}<span>${dt(it.published)}</span>${it.views?`<span>👁 ${num(it.views)}</span>`:""}</div>
|
${price}${findChips(it.find)}<span>${dt(it.published)}</span>${it.views?`<span>👁 ${num(it.views)}</span>`:""}</div>
|
||||||
${compact?"":`<div class="x">${esc(it.text||"")}</div>`}
|
${compact?"":`<div class="x">${esc(it.text||"")}</div>`}
|
||||||
</div>
|
</div>
|
||||||
<div class="side">
|
<div class="side">
|
||||||
@@ -510,7 +537,7 @@ function closeDrawer(){$("#drawer").classList.remove("on");$("#backdrop").classL
|
|||||||
|
|
||||||
/* ---------- abuse ---------- */
|
/* ---------- abuse ---------- */
|
||||||
V.abuse=async()=>{
|
V.abuse=async()=>{
|
||||||
const d=await api("/api/abuse?limit=60");
|
const d=await api("/api/abuse?limit=120");
|
||||||
const tags=Object.entries(d.tags||{}).slice(0,14);
|
const tags=Object.entries(d.tags||{}).slice(0,14);
|
||||||
$("#view").innerHTML=`
|
$("#view").innerHTML=`
|
||||||
<div class="page-h"><div><h2>Абузы</h2><p>Скам/промо/триалы/ключи в постах каналов и форумов. Вес считается по признакам: обещания, цены, «в личку», свежесть.</p></div>
|
<div class="page-h"><div><h2>Абузы</h2><p>Скам/промо/триалы/ключи в постах каналов и форумов. Вес считается по признакам: обещания, цены, «в личку», свежесть.</p></div>
|
||||||
@@ -620,20 +647,28 @@ V.runs=async()=>{
|
|||||||
const ok=(d.items||[]).filter(r=>r.status==="ok").length;
|
const ok=(d.items||[]).filter(r=>r.status==="ok").length;
|
||||||
$("#view").innerHTML=`
|
$("#view").innerHTML=`
|
||||||
<div class="page-h"><div><h2>Прогоны</h2><p>Журнал сборов: что запросили, сколько получили и сколько нового легло в базу.</p></div>
|
<div class="page-h"><div><h2>Прогоны</h2><p>Журнал сборов: что запросили, сколько получили и сколько нового легло в базу.</p></div>
|
||||||
<div class="right"><span class="chip on">ok ${ok}</span><span class="chip off">ошибки ${d.total-ok}</span></div></div>
|
<div class="right">
|
||||||
|
<span class="chip click on" data-rf="all">все <span class="n">${num(d.total)}</span></span>
|
||||||
|
<span class="chip click" data-rf="ok">ok <span class="n">${ok}</span></span>
|
||||||
|
<span class="chip click" data-rf="err">ошибки <span class="n">${d.total-ok}</span></span></div></div>
|
||||||
<div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
|
<div class="card" style="padding:10px 12px;margin-bottom:12px"><div class="row">
|
||||||
<input id="r-q" placeholder="фильтр: источник или запрос" style="min-width:280px">
|
<input id="r-q" placeholder="фильтр: источник или запрос" style="min-width:280px">
|
||||||
<span class="spacer"></span><span class="dim mono">всего ${num(d.total)}</span></div></div>
|
<span class="spacer"></span><span class="dim mono">всего ${num(d.total)}</span></div></div>
|
||||||
<div class="card" style="padding:0"><div class="tbl-wrap"><table id="r-tbl"><thead><tr>
|
<div class="card" style="padding:0"><div class="tbl-wrap"><table id="r-tbl"><thead><tr>
|
||||||
<th>когда</th><th>источник</th><th>запрос</th><th class="right">got</th><th class="right">kept</th><th class="right">байт</th><th>статус</th></tr></thead>
|
<th>когда</th><th>источник</th><th>запрос</th><th class="right">got</th><th class="right">kept</th><th class="right">байт</th><th>статус</th></tr></thead>
|
||||||
<tbody>${(d.items||[]).map(r=>`<tr><td class="mono dim nowrap">${esc(r.when||"")}</td><td>${esc(r.source||"—")}</td>
|
<tbody>${(d.items||[]).map(r=>`<tr data-st="${esc(r.status||"")}"><td class="mono dim nowrap">${esc(r.when||"")}</td><td>${esc(r.source||"—")}</td>
|
||||||
<td class="dim" style="max-width:380px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(r.query||"")}</td>
|
<td class="dim" style="max-width:380px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(r.query||"")}</td>
|
||||||
<td class="right mono">${r.got??""}</td><td class="right mono">${r.kept??""}</td><td class="right mono dim">${r.bytes?num(r.bytes):""}</td>
|
<td class="right mono">${r.got??""}</td><td class="right mono">${r.kept??""}</td><td class="right mono dim">${r.bytes?num(r.bytes):""}</td>
|
||||||
<td><span class="pill ${r.status==="ok"?"ok":"no"}"></span>${esc(r.status||"")}</td></tr>`).join("")}</tbody></table></div></div>`;
|
<td><span class="pill ${r.status==="ok"?"ok":"no"}"></span>${esc(r.status||"")}</td></tr>`).join("")}</tbody></table></div></div>`;
|
||||||
$("#r-q").addEventListener("input",debounce(e=>{
|
S.runQ=""; S.runF="all";
|
||||||
const v=e.target.value.toLowerCase();
|
const applyRuns=()=>{const v=(S.runQ||"").toLowerCase();
|
||||||
$$("#r-tbl tbody tr").forEach(tr=>tr.style.display=tr.textContent.toLowerCase().includes(v)?"":"none");
|
$$("#r-tbl tbody tr").forEach(tr=>{
|
||||||
},120));
|
const okT=!v||tr.textContent.toLowerCase().includes(v);
|
||||||
|
const okS=S.runF==="all"||(S.runF==="ok"?tr.dataset.st==="ok":tr.dataset.st!=="ok");
|
||||||
|
tr.style.display=(okT&&okS)?"":"none";});};
|
||||||
|
$("#r-q").addEventListener("input",debounce(e=>{S.runQ=e.target.value;applyRuns()},120));
|
||||||
|
$$("#view [data-rf]").forEach(c=>c.addEventListener("click",()=>{
|
||||||
|
$$("#view [data-rf]").forEach(x=>x.classList.remove("on"));c.classList.add("on");S.runF=c.dataset.rf;applyRuns();}));
|
||||||
};
|
};
|
||||||
|
|
||||||
/* ---------- telegram ---------- */
|
/* ---------- telegram ---------- */
|
||||||
@@ -749,7 +784,9 @@ V.watch=async()=>{
|
|||||||
<div class="card" style="margin-bottom:14px"><div class="row">
|
<div class="card" style="margin-bottom:14px"><div class="row">
|
||||||
<input id="w-q" placeholder="например: CVE-2026, firecrawl, приватка" style="min-width:320px">
|
<input id="w-q" placeholder="например: CVE-2026, firecrawl, приватка" style="min-width:320px">
|
||||||
<select id="w-type"><option value="all">любой тип</option>${(S.types||[]).map(t=>`<option value="${esc(t.key)}">${esc(t.label)}</option>`).join("")}</select>
|
<select id="w-type"><option value="all">любой тип</option>${(S.types||[]).map(t=>`<option value="${esc(t.key)}">${esc(t.label)}</option>`).join("")}</select>
|
||||||
<button class="btn primary" data-act="watch-add">Добавить правило</button></div></div>
|
<button class="btn primary" data-act="watch-add">Добавить правило</button></div>
|
||||||
|
<div class="row" style="margin-top:10px"><span class="dim">быстро:</span>
|
||||||
|
${["CVE-2026","ключ","firecrawl","слив","промо","52pojie"].map(x=>`<span class="chip click" data-watch-q="${esc(x)}">${esc(x)}</span>`).join("")}</div></div>
|
||||||
<div class="grid g2">${(d.rules||[]).map(r=>`<div class="card hov">
|
<div class="grid g2">${(d.rules||[]).map(r=>`<div class="card hov">
|
||||||
<div class="row"><b>${esc(r.q)}</b><span class="chip">${esc(r.type)}</span><span class="spacer"></span>
|
<div class="row"><b>${esc(r.q)}</b><span class="chip">${esc(r.type)}</span><span class="spacer"></span>
|
||||||
<span class="chip on">${num(r.count)}</span><button class="ico" data-watch-del="${esc(r.id)}">✕</button></div>
|
<span class="chip on">${num(r.count)}</span><button class="ico" data-watch-del="${esc(r.id)}">✕</button></div>
|
||||||
@@ -865,6 +902,8 @@ document.addEventListener("click",async e=>{
|
|||||||
const it=t.closest("[data-id]"); if(it){e.preventDefault();openItem(+it.dataset.id);return}
|
const it=t.closest("[data-id]"); if(it){e.preventDefault();openItem(+it.dataset.id);return}
|
||||||
const wd=t.closest("[data-watch-del]"); if(wd){e.preventDefault();await post("/api/watch/remove",{id:wd.dataset.watchDel});toast("правило удалено");render();return}
|
const wd=t.closest("[data-watch-del]"); if(wd){e.preventDefault();await post("/api/watch/remove",{id:wd.dataset.watchDel});toast("правило удалено");render();return}
|
||||||
const wt=t.closest("[data-watch]"); if(wt){e.preventDefault();await post("/api/watch/add",{q:wt.dataset.watch,type:"all"});toast("в watchlist");return}
|
const wt=t.closest("[data-watch]"); if(wt){e.preventDefault();await post("/api/watch/add",{q:wt.dataset.watch,type:"all"});toast("в watchlist");return}
|
||||||
|
const wq=t.closest("[data-watch-q]"); if(wq){e.preventDefault();
|
||||||
|
post("/api/watch/add",{q:wq.dataset.watchQ,type:"all"}).then(()=>{toast("правило добавлено: "+wq.dataset.watchQ);render()});return}
|
||||||
const po=t.closest("[data-pull-one]"); if(po){e.preventDefault();tgPull([po.dataset.pullOne]);return}
|
const po=t.closest("[data-pull-one]"); if(po){e.preventDefault();tgPull([po.dataset.pullOne]);return}
|
||||||
const ts=t.closest("[data-toggle-src]"); if(ts){e.preventDefault();
|
const ts=t.closest("[data-toggle-src]"); if(ts){e.preventDefault();
|
||||||
await post("/api/sources/toggle",{name:ts.dataset.toggleSrc,enabled:ts.dataset.on==="1"});toast("переключено");srcDetail(ts.dataset.toggleSrc);return}
|
await post("/api/sources/toggle",{name:ts.dataset.toggleSrc,enabled:ts.dataset.on==="1"});toast("переключено");srcDetail(ts.dataset.toggleSrc);return}
|
||||||
|
|||||||
Reference in New Issue
Block a user