abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)
This commit is contained in:
Vlad Gothbreach
149
tools/fresh_candidates.py
Normal file
149
tools/fresh_candidates.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""Кандидаты для расширения реестра omni: security/CVE/leak/CN/RU/tg.
|
||||
|
||||
Источник истины для tools/probe_fresh.py. Живые строки после прогона
|
||||
переезжают в omni/sources_fresh.py (генерируется `--emit`).
|
||||
|
||||
Формат dict совпадает с Source(**d). Адаптеры — только из feeds.ADAPTERS.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
|
||||
def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
||||
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
||||
|
||||
|
||||
# NVD recent: окно считается на импорте (API требует обе даты и <=120 дней)
|
||||
_NOW = datetime.datetime.utcnow()
|
||||
_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
|
||||
_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000")
|
||||
|
||||
CANDIDATES = [
|
||||
# ---------------- 1. security news / research (RSS) ----------------
|
||||
d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)),
|
||||
d("gbhackers", "security", "rss", "https://gbhackers.com/feed/", "rss", ("news",)),
|
||||
d("cybersecuritynews", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", ("news",)),
|
||||
d("scmedia", "security", "rss", "https://www.scmagazine.com/rss.xml", "rss", ("news",)),
|
||||
d("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ("news",)),
|
||||
d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
|
||||
d("seclists_bugtraq", "security", "rss", "https://seclists.org/rss/bugtraq.rss", "rss", ("mailing", "exploit")),
|
||||
d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)),
|
||||
d("huntress_blog", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", ("research", "dfir")),
|
||||
d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("research", "cti")),
|
||||
d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("research", "cti")),
|
||||
d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)),
|
||||
d("sophos_news", "security", "rss", "https://news.sophos.com/en-us/feed/", "rss", ("research",)),
|
||||
d("trendmicro_si", "security", "rss", "https://www.trendmicro.com/rss/security-intelligence.xml", "rss", ("research",)),
|
||||
d("volexity_blog", "security", "rss", "https://www.volexity.com/blog/feed/", "rss", ("research", "dfir")),
|
||||
d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)),
|
||||
d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)),
|
||||
d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)),
|
||||
d("intezer_blog", "security", "rss", "https://intezer.com/blog/feed/", "rss", ("malware",)),
|
||||
d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)),
|
||||
d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)),
|
||||
d("zscaler_research", "security", "rss",
|
||||
"https://www.zscaler.com/blogs/security-research/rss.xml", "rss", ("research",)),
|
||||
d("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)),
|
||||
d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/resources/blog/feed/", "rss", ("cti",)),
|
||||
d("dragos_blog", "security", "rss", "https://www.dragos.com/feed/", "rss", ("ics",)),
|
||||
d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("research", "apt")),
|
||||
d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")),
|
||||
d("jfrog_blog", "security", "rss", "https://jfrog.com/blog/feed/", "rss", ("supply-chain",)),
|
||||
d("gitguardian_blog", "security", "rss", "https://blog.gitguardian.com/rss/", "rss", ("secrets",)),
|
||||
d("truffle_security", "security", "rss", "https://trufflesecurity.com/blog/rss.xml", "rss", ("secrets",)),
|
||||
d("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)),
|
||||
d("hudsonrock_blog", "security", "rss", "https://www.hudsonrock.com/blog/rss.xml", "rss", ("stealer",)),
|
||||
d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("blog", "writeup")),
|
||||
d("0xdf", "security", "rss", "https://0xdf.gitlab.io/feed.xml", "rss", ("writeup", "htb")),
|
||||
d("hackmag", "security", "rss", "https://hackmag.com/feed/", "rss", ("ru",)),
|
||||
d("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("cloud",)),
|
||||
|
||||
# ---------------- 2. CVE / vulnerabilities ----------------
|
||||
d("nvd_recent", "security", "json",
|
||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s"
|
||||
% (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")),
|
||||
d("nvd_exploited_kw", "security", "json",
|
||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild",
|
||||
"nvd", ("cve", "exploited")),
|
||||
d("github_advisories_api", "security", "json",
|
||||
"https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
|
||||
d("cve_circl_last", "security", "json", "https://cve.circl.lu/api/last/40", "cve_circl", ("cve",)),
|
||||
d("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")),
|
||||
d("gentoo_glsa", "security", "rss", "https://security.gentoo.org/glsa/feed.xml", "rss", ("cve", "linux")),
|
||||
d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/feed.atom", "rss", ("cve", "linux")),
|
||||
d("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing", ("cve", "linux"),
|
||||
detail=False, extra={"include_prefixes": ["/security/2026/"], "exclude_exact": ["/security/"]}),
|
||||
d("oss_security_openwall", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
|
||||
("mailing", "cve"), extra={"include_prefixes": ["/lists/oss-security/2026/"]}),
|
||||
d("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing",
|
||||
("cve", "browser"), extra={"include_prefixes": ["/en-US/security/advisories/mfsa"]}),
|
||||
d("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default",
|
||||
"rss", ("cve", "browser")),
|
||||
d("cvefeed_latest", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ("cve",)),
|
||||
d("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)),
|
||||
d("cxsecurity", "security", "rss", "https://cxsecurity.com/rss/", "rss", ("exploit",)),
|
||||
d("vulmon_recent", "security", "rss", "https://vulmon.com/rss/recent", "rss", ("cve",)),
|
||||
d("cisa_ics_advisories", "security", "rss",
|
||||
"https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml", "rss", ("ics", "cve")),
|
||||
d("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)),
|
||||
d("epss_recent", "security", "json", "https://api.first.org/data/v1/epss?limit=25&order=!epss", "generic_json",
|
||||
("cve", "epss")),
|
||||
|
||||
# ---------------- 3. leaks / ransomware / underground ----------------
|
||||
d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet", "news")),
|
||||
d("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")),
|
||||
d("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")),
|
||||
d("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")),
|
||||
d("ransomware_live_recent", "abuse", "json", "https://api.ransomware.live/recentvictims", "generic_json",
|
||||
("ransomware",)),
|
||||
d("ransomlook_recent", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", ("ransomware",)),
|
||||
d("ransomware_live_groups", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", ("ransomware",)),
|
||||
d("shadowserver_news", "abuse", "rss", "https://www.shadowserver.org/news/feed/", "rss", ("infra",)),
|
||||
|
||||
# ---------------- 4. китайские security ----------------
|
||||
d("anquanke_rss", "security", "rss", "https://www.anquanke.com/rss", "rss", ("cn", "news")),
|
||||
d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")),
|
||||
d("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")),
|
||||
d("seebug_paper", "security", "rss", "https://paper.seebug.org/rss/", "rss", ("cn", "research")),
|
||||
d("xz_aliyun", "security", "html", "https://xz.aliyun.com/", "html_listing", ("cn", "research"),
|
||||
extra={"include_prefixes": ["/t/"]}),
|
||||
d("aqniu", "security", "rss", "https://www.aqniu.com/feed", "rss", ("cn", "news")),
|
||||
d("knownsec_blog", "security", "rss", "https://blog.knownsec.com/feed/", "rss", ("cn", "research")),
|
||||
d("butian_forum", "forum", "html", "https://forum.butian.net/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/share/"]}),
|
||||
d("kanxue_news_html", "forum", "html", "https://bbs.kanxue.com/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/thread-"]}),
|
||||
d("t00ls_html", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"),
|
||||
extra={"include_prefixes": ["/article-"]}),
|
||||
|
||||
# ---------------- 5. русскоязычные ----------------
|
||||
d("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")),
|
||||
d("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
|
||||
d("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
|
||||
d("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")),
|
||||
d("kaspersky_ru_blog", "security", "rss", "https://www.kaspersky.ru/blog/feed/", "rss", ("ru", "research")),
|
||||
d("securitylab_rss", "security", "rss", "https://www.securitylab.ru/rss/all.xml", "rss", ("ru", "news")),
|
||||
d("anti_malware_ru", "security", "rss", "https://www.anti-malware.ru/rss.xml", "rss", ("ru", "news")),
|
||||
d("cnews_security", "security", "rss", "https://www.cnews.ru/inc/rss/news.xml", "rss", ("ru", "news")),
|
||||
d("bdu_fstec", "security", "rss", "https://bdu.fstec.ru/rss/vulxml", "rss", ("ru", "cve")),
|
||||
d("codeby_forum", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", ("ru", "forum")),
|
||||
|
||||
# ---------------- 6. telegram (t.me/s, публичное превью) ----------------
|
||||
d("tg_thehackersnews", "security", "html", "https://t.me/s/TheHackersNews", "tg_web", ("tg", "news")),
|
||||
d("tg_bleepingcomputer", "security", "html", "https://t.me/s/bleepingcomputer", "tg_web", ("tg", "news")),
|
||||
d("tg_vxunderground", "security", "html", "https://t.me/s/vxunderground", "tg_web", ("tg", "malware")),
|
||||
d("tg_secator", "security", "html", "https://t.me/s/SecAtor", "tg_web", ("tg", "ru")),
|
||||
d("tg_itsecforu", "security", "html", "https://t.me/s/itsecforu", "tg_web", ("tg", "ru")),
|
||||
d("tg_hitech_crime", "security", "html", "https://t.me/s/hitech_crime", "tg_web", ("tg", "ru")),
|
||||
d("tg_codeby", "security", "html", "https://t.me/s/codeby", "tg_web", ("tg", "ru")),
|
||||
d("tg_darkwebinformer", "abuse", "html", "https://t.me/s/DarkWebInformer", "tg_web", ("tg", "leak")),
|
||||
d("tg_breachdetector", "abuse", "html", "https://t.me/s/breachdetector", "tg_web", ("tg", "leak")),
|
||||
d("tg_databreaches", "abuse", "html", "https://t.me/s/databreaches", "tg_web", ("tg", "leak")),
|
||||
d("tg_ransomware_live", "abuse", "html", "https://t.me/s/ransomware_live", "tg_web", ("tg", "ransomware")),
|
||||
d("tg_cybersecuritynews", "security", "html", "https://t.me/s/cybersecuritynews", "tg_web", ("tg", "news")),
|
||||
d("tg_ai_newz", "community", "html", "https://t.me/s/ai_newz", "tg_web", ("tg", "ai", "ru")),
|
||||
d("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web",
|
||||
("tg", "ai", "ru")),
|
||||
d("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")),
|
||||
d("tg_bo0om", "abuse", "html", "https://t.me/s/bo0om", "tg_web", ("tg", "leak", "ru")),
|
||||
]
|
||||
146
tools/probe_fresh.py
Normal file
146
tools/probe_fresh.py
Normal file
@@ -0,0 +1,146 @@
|
||||
"""Живой прогон кандидатов из tools/fresh_candidates.py через штатные адаптеры omni.
|
||||
|
||||
python tools/probe_fresh.py # прогон + data/fresh_probe.json
|
||||
python tools/probe_fresh.py --emit # + перегенерация omni/sources_fresh.py
|
||||
python tools/probe_fresh.py --only tg_ --workers 4 --limit 5
|
||||
|
||||
Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись.
|
||||
Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина
|
||||
в отчёте была фактической, а не «0 items».
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
|
||||
from omni import feeds, net # noqa: E402
|
||||
from omni.sources import Source # noqa: E402
|
||||
from fresh_candidates import CANDIDATES # noqa: E402
|
||||
|
||||
|
||||
def _diag(src):
|
||||
"""Почему пусто: код ответа, блокировка, размер тела."""
|
||||
try:
|
||||
r = net.get(src.url, timeout=20)
|
||||
bits = ["http %s" % r.status]
|
||||
if getattr(r, "blocked", False):
|
||||
bits.append("blocked")
|
||||
if getattr(r, "error", ""):
|
||||
bits.append("err:%s" % str(r.error)[:120])
|
||||
bits.append("%db" % len(r.text or ""))
|
||||
return ", ".join(bits)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return "exception:%s" % str(e)[:140]
|
||||
|
||||
|
||||
def probe_one(src, limit):
|
||||
t0 = time.time()
|
||||
if src.adapter not in feeds.ADAPTERS:
|
||||
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
||||
"url": src.url, "tags": list(src.tags), "ok": False, "n": 0,
|
||||
"err": "no_adapter:%s" % src.adapter, "ms": 0, "samples": []}
|
||||
try:
|
||||
items = feeds.fetch(src, limit) or []
|
||||
err = ""
|
||||
except Exception as e: # noqa: BLE001
|
||||
items, err = [], "raise:%s" % str(e)[:140]
|
||||
ms = int((time.time() - t0) * 1000)
|
||||
if not items and not err:
|
||||
err = _diag(src)
|
||||
return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter,
|
||||
"url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items),
|
||||
"err": err, "ms": ms,
|
||||
"samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]}
|
||||
|
||||
|
||||
def emit(rows, dead, path):
|
||||
"""omni/sources_fresh.py: только живые, без импортов из omni.sources."""
|
||||
def q(s):
|
||||
return '"%s"' % str(s).replace('"', '\\"')
|
||||
|
||||
lines = [
|
||||
'"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.',
|
||||
"",
|
||||
"Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).",
|
||||
"Отклонённые кандидаты и причины — в хвостовом комментарии.",
|
||||
"Импортов из omni.sources нет намеренно (циклический импорт).",
|
||||
'"""',
|
||||
"",
|
||||
"",
|
||||
"def fresh(S):",
|
||||
' """S — конструктор Source; возвращает только проверенные сетью источники."""',
|
||||
" return [",
|
||||
]
|
||||
for r in rows:
|
||||
extra = ""
|
||||
if r.get("extra"):
|
||||
extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True)
|
||||
limit = r.get("limit") or 15
|
||||
lim = "" if limit == 15 else ", limit=%d" % limit
|
||||
det = ", detail=True" if r.get("detail") else ""
|
||||
tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ())
|
||||
lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % (
|
||||
q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]),
|
||||
tags, lim, det, extra))
|
||||
lines += [" ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"]
|
||||
for r in dead:
|
||||
lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90]))
|
||||
text = "\n".join(lines) + "\n"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit", type=int, default=5)
|
||||
ap.add_argument("--workers", type=int, default=6)
|
||||
ap.add_argument("--only", default="")
|
||||
ap.add_argument("--out", default="data/fresh_probe.json")
|
||||
ap.add_argument("--emit", action="store_true")
|
||||
ap.add_argument("--fresh", action="store_true", help="прогонять только то, что уже в sources_fresh.py")
|
||||
a = ap.parse_args()
|
||||
|
||||
cands = [c for c in CANDIDATES if not a.only or a.only in c["name"]]
|
||||
srcs = []
|
||||
for c in cands:
|
||||
try:
|
||||
srcs.append(Source(**c))
|
||||
except Exception as e: # noqa: BLE001
|
||||
print("skip %s: %s" % (c.get("name"), e))
|
||||
|
||||
t0 = time.time()
|
||||
with ThreadPoolExecutor(max_workers=a.workers) as ex:
|
||||
rows = list(ex.map(lambda s: probe_one(s, a.limit), srcs))
|
||||
|
||||
alive = [r for r in rows if r["ok"]]
|
||||
dead = [r for r in rows if not r["ok"]]
|
||||
cats = {}
|
||||
for r in alive:
|
||||
cats[r["category"]] = cats.get(r["category"], 0) + 1
|
||||
out = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "limit": a.limit, "workers": a.workers,
|
||||
"total": len(rows), "alive": len(alive), "dead": len(dead), "seconds": round(time.time() - t0, 1),
|
||||
"by_category": dict(sorted(cats.items(), key=lambda kv: -kv[1])),
|
||||
"results": sorted(rows, key=lambda r: (not r["ok"], r["category"], r["name"]))}
|
||||
p = ROOT / a.out
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
print("всего %d | живых %d | мёртвых %d | %.1fs" % (len(rows), len(alive), len(dead), time.time() - t0))
|
||||
print("по категориям:", out["by_category"])
|
||||
for r in dead:
|
||||
print(" dead %-24s %s" % (r["name"], r["err"][:110]))
|
||||
|
||||
if a.emit:
|
||||
n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py")
|
||||
print("omni/sources_fresh.py: %d источников" % n)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
96
tools/ui_smoke.py
Normal file
96
tools/ui_smoke.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Смоук панели в реальном Chromium: скриншоты + ошибки консоли.
|
||||
|
||||
python tools/ui_smoke.py [--url http://127.0.0.1:8078/] [--shots data/shots]
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
|
||||
async def run(url, out):
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
errors, report = [], {}
|
||||
async with async_playwright() as p:
|
||||
b = await p.chromium.launch()
|
||||
pg = await b.new_page(viewport={"width": 1500, "height": 950})
|
||||
pg.on("console", lambda m: errors.append([m.type, m.text]) if m.type in ("error",) else None)
|
||||
pg.on("pageerror", lambda e: errors.append(["pageerror", str(e)]))
|
||||
await pg.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await pg.wait_for_selector(".kpi", timeout=20000)
|
||||
await pg.wait_for_timeout(1200)
|
||||
report["title"] = await pg.title()
|
||||
report["nav"] = await pg.locator(".nav-i").count()
|
||||
report["kpi"] = (await pg.locator(".kpi").all_inner_texts())[:4]
|
||||
await pg.screenshot(path=str(out / "v2_overview.png"), full_page=False)
|
||||
|
||||
await pg.click("[data-nav=feed]")
|
||||
await pg.wait_for_selector("#list .item", timeout=20000)
|
||||
report["feed_rows"] = await pg.locator(".item").count()
|
||||
await pg.screenshot(path=str(out / "v2_feed.png"))
|
||||
await pg.locator(".item").first.click()
|
||||
await pg.wait_for_timeout(1200)
|
||||
report["drawer"] = (await pg.locator("#d-title").inner_text())[:100]
|
||||
await pg.screenshot(path=str(out / "v2_drawer.png"))
|
||||
await pg.keyboard.press("Escape")
|
||||
|
||||
await pg.click("[data-nav=sources]")
|
||||
await pg.wait_for_timeout(1500)
|
||||
report["src_rows"] = await pg.locator("#src-tbl tbody tr").count()
|
||||
await pg.screenshot(path=str(out / "v2_sources.png"))
|
||||
|
||||
await pg.click("[data-nav=abuse]")
|
||||
await pg.wait_for_timeout(1500)
|
||||
report["abuse_rows"] = await pg.locator(".item").count()
|
||||
await pg.screenshot(path=str(out / "v2_abuse.png"))
|
||||
|
||||
await pg.click("[data-nav=runs]")
|
||||
await pg.wait_for_timeout(900)
|
||||
report["run_rows"] = await pg.locator("#r-tbl tbody tr").count()
|
||||
await pg.screenshot(path=str(out / "v2_runs.png"))
|
||||
|
||||
await pg.click("[data-nav=tg]")
|
||||
await pg.wait_for_timeout(2000)
|
||||
report["tg"] = (await pg.locator(".card").first.inner_text())[:160].replace("\n", " | ")
|
||||
await pg.screenshot(path=str(out / "v2_tg.png"))
|
||||
|
||||
await pg.click("[data-nav=keys]")
|
||||
await pg.wait_for_timeout(800)
|
||||
report["key_rows"] = await pg.locator("#k-tbl tbody tr").count()
|
||||
await pg.screenshot(path=str(out / "v2_keys.png"))
|
||||
|
||||
await pg.click("[data-nav=digest]")
|
||||
await pg.wait_for_timeout(500)
|
||||
await pg.click("[data-act=digest]")
|
||||
await pg.wait_for_timeout(6000)
|
||||
report["digest_chars"] = len(await pg.locator("#d-out").inner_text())
|
||||
await pg.screenshot(path=str(out / "v2_digest.png"))
|
||||
|
||||
await pg.click("[data-nav=watch]")
|
||||
await pg.wait_for_timeout(600)
|
||||
await pg.screenshot(path=str(out / "v2_watch.png"))
|
||||
|
||||
# мобильный вид
|
||||
await pg.set_viewport_size({"width": 420, "height": 900})
|
||||
await pg.click("[data-nav=feed]")
|
||||
await pg.wait_for_timeout(1200)
|
||||
await pg.screenshot(path=str(out / "v2_mobile.png"))
|
||||
await b.close()
|
||||
report["console"] = errors[:12]
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
return 0 if not errors else 1
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:8078/")
|
||||
ap.add_argument("--shots", default="data/shots")
|
||||
a = ap.parse_args()
|
||||
sys.exit(asyncio.run(run(a.url, pathlib.Path(a.shots))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user