diff --git a/.local/diag.py b/.local/diag.py new file mode 100644 index 0000000..9e64de9 --- /dev/null +++ b/.local/diag.py @@ -0,0 +1,28 @@ +import sys, json +sys.path.insert(0,'.') +from concurrent.futures import ThreadPoolExecutor +from omni import net + +URLS = [ + "https://darknetlive.com/feed/", "https://darknetlive.com/rss/", "https://darknetlive.com/rss.xml", + "https://www.volexity.com/blog/feed/", "https://cxsecurity.com/rss/", + "https://www.debian.org/security/", "https://www.openwall.com/lists/oss-security/", + "https://www.t00ls.com/", "https://xz.aliyun.com/", + "https://t.me/s/TheHackersNews", "https://t.me/s/thehackernews", "https://t.me/s/SecAtor", + "https://t.me/s/true_secator", "https://t.me/s/databreaches", "https://t.me/s/cybersecuritynews", +] +def one(u): + try: + r = net.get(u, timeout=25) + t = r.text or "" + return {"u":u,"st":r.status,"n":len(t), + "ct":(r.headers or {}).get("content-type","")[:40], + "item":t.count("(.*?)", t, re.S) + src = Source(name="probe_" + h, category="security", kind="html", url=url, adapter="tg_web", + tags=("tg", "leak"), limit=5) + items = feeds.fetch(src, 5) + return {"h": h, "st": r.status, "n": len(t), "msg": t.count("tgme_widget_message_wrap"), + "items": len(items), "title": (title.group(1).strip()[:60] if title else ""), + "first": (items[0].title[:70] if items else "")} + except Exception as e: + return {"h": h, "st": 0, "err": str(e)[:80]} + + +with ThreadPoolExecutor(6) as ex: + for r in ex.map(lambda h: tg_one(h, "https://t.me/s/" + h), TG): + print(json.dumps(r, ensure_ascii=False)) \ No newline at end of file diff --git a/.local/diag3.py b/.local/diag3.py new file mode 100644 index 0000000..1dbd2b2 --- /dev/null +++ b/.local/diag3.py @@ -0,0 +1,41 @@ +import sys, json, re, urllib.parse +sys.path.insert(0, '.') +from concurrent.futures import ThreadPoolExecutor +from omni import net +from bs4 import BeautifulSoup + +URLS = [ + "https://www.debian.org/security/", + "https://www.openwall.com/lists/oss-security/", + "https://www.t00ls.com/", + "https://www.anquanke.com/", + "https://www.freebuf.com/", + "https://bbs.kanxue.com/", + "https://www.secrss.com/", + "https://www.ichunqiu.com/", +] + + +def one(u): + try: + r = net.get(u, timeout=25) + soup = BeautifulSoup(r.text or "", "lxml") + hrefs = [] + for a in soup.find_all("a", href=True): + h = urllib.parse.urljoin(u, str(a["href"])) + p = urllib.parse.urlsplit(h).path + hrefs.append(p) + pat = {} + for p in hrefs: + seg = re.sub(r"\d+", "N", p)[:60] + pat[seg] = pat.get(seg, 0) + 1 + top = sorted(pat.items(), key=lambda kv: -kv[1])[:12] + return {"u": u, "st": r.status, "n": len(r.text or ""), "links": len(hrefs), + "paths": top} + except Exception as e: + return {"u": u, "st": 0, "err": str(e)[:90]} + + +with ThreadPoolExecutor(4) as ex: + for r in ex.map(one, URLS): + print(json.dumps(r, ensure_ascii=False)) \ No newline at end of file diff --git a/omni/abuse.py b/omni/abuse.py index 1965338..2400ad2 100644 --- a/omni/abuse.py +++ b/omni/abuse.py @@ -167,14 +167,15 @@ def _connect(): return con -def rebuild(full=False, chunk=4000): +def rebuild(full=False, chunk=4000, auto=False): """Инкрементальный индекс абузов: classify по новым items, без полного перещёлкивания. full=True (или OMNI_ABUSE_REBUILD=1) — пересчитать всю базу после правки правил. + auto=True (вызов из чтения) игнорирует env, иначе каждый scan делал бы полный проход. Возвращает число записей, попавших в индекс на этом проходе. """ import os - if os.environ.get("OMNI_ABUSE_REBUILD") == "1": + if not auto and os.environ.get("OMNI_ABUSE_REBUILD") == "1": full = True con = _connect() try: @@ -207,7 +208,7 @@ def rebuild(full=False, chunk=4000): def _rows(limit=400, tag=None, q=None, since_days=None): """Читает из индекса abuse_idx; индекс добивается свежими записями перед чтением.""" try: - rebuild() + rebuild(auto=True) except Exception: pass sql = ("SELECT i.id, i.source, i.url, i.title, i.text, i.published, a.tags, i.score, a.score " diff --git a/omni/panel_api.py b/omni/panel_api.py new file mode 100644 index 0000000..341eb9f --- /dev/null +++ b/omni/panel_api.py @@ -0,0 +1,316 @@ +"""Данные для панели v2: обзор, детали записи, TG-сессия, дайджест, watchlist. + +Здесь только чтение и небольшие записи в data/*.json; HTTP-обвязка — в serve.py, +чтобы роуты оставались тонкими, а логика — тестируемой без поднятия сервера. +""" +import datetime as _dt +import json +import os +import re +import time +import urllib.parse + +from . import abuse as abuse_mod +from . import config, sources, source_state, store, taxonomy + +BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' " + "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END") + + +def _clean(v, n=0): + import html as _h + if not v: + return "" + t = re.sub(r"<[^>]{1,200}>", " ", str(v)) + for _ in range(2): + t = _h.unescape(t) + t = re.sub(r"\s+", " ", t).strip() + return t[:n].rstrip() if n else t + + +# ---------- обзор ---------- + +def timeline(days=30, cat=None): + """Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями.""" + days = max(1, min(365, int(days or 30))) + names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None + where, args = "", [] + if names is not None: + where = " WHERE source IN (%s)" % ",".join("?" * len(names)) + args = list(names) + sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END)," + " SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)" + " FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b") + got = {} + with store.db() as c: + for b, n, cve, sev in c.execute(sql, args): + if b: + got[b] = (int(n or 0), int(cve or 0), int(sev or 0)) + today = _dt.date.today() + out = [] + for i in range(days - 1, -1, -1): + d = (today - _dt.timedelta(days=i)).isoformat() + n, cve, sev = got.get(d, (0, 0, 0)) + out.append({"day": d, "n": n, "cve": cve, "sev": sev}) + return {"ok": True, "days": days, "points": out, + "total": sum(p["n"] for p in out), + "max": max([p["n"] for p in out] or [0])} + + +def item_detail(iid): + """Полная запись + соседи: свежее из того же источника и с того же домена.""" + iid = int(iid) + keys = ("id", "provider", "kind", "source", "url", "title", "text", "author", "published", + "lang", "tags", "price", "currency", "score", "types") + with store.db() as c: + row = c.execute("SELECT " + ",".join(keys) + " FROM items WHERE id=?", (iid,)).fetchone() + if not row: + return {"error": "not found"} + it: dict = dict(zip(keys, row)) + host = "" + try: + host = urllib.parse.urlsplit(it.get("url") or "").netloc.lower() + except Exception: + host = "" + same_src = c.execute("SELECT id, title, source, published FROM items WHERE source=? AND id!=? " + "ORDER BY id DESC LIMIT 6", (it["source"], iid)).fetchall() + same_host = [] + if host: + same_host = c.execute("SELECT id, title, source, published FROM items " + "WHERE url LIKE ? AND id!=? ORDER BY id DESC LIMIT 6", + ("%//" + host + "/%", iid)).fetchall() + it["title"] = _clean(it.get("title"), 400) + it["text"] = _clean(it.get("text")) + it["types"] = [t for t in (it.get("types") or "").split(",") if t] + it["types_labels"] = taxonomy.labels(it["types"]) + it["tags"] = [t for t in (it.get("tags") or "").split(",") if t] if isinstance(it.get("tags"), str) else (it.get("tags") or []) + it["related_same_source"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]} + for r in same_src] + it["related_same_host"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]} + for r in same_host] + return {"ok": True, "item": it} + + +def suggest(q, limit=8): + """Подсказки поиска: заголовки из базы + имена источников + категории.""" + q = (q or "").strip() + out = {"titles": [], "sources": [], "categories": []} + if len(q) < 2: + return out + like = "%" + q + "%" + with store.db() as c: + for iid, title, src in c.execute("SELECT id,title,source FROM items WHERE title LIKE ? " + "ORDER BY id DESC LIMIT ?", (like, limit)): + out["titles"].append({"id": iid, "title": _clean(title, 120), "source": src}) + ql = q.lower() + for s in sources.all_sources(): + if ql in s.name.lower() or ql in (s.url or "").lower(): + out["sources"].append({"name": s.name, "category": s.category}) + if len(out["sources"]) >= limit: + break + cats = {} + for s in sources.all_sources(): + cats[s.category] = cats.get(s.category, 0) + 1 + out["categories"] = [{"name": k, "n": v} for k, v in sorted(cats.items()) if ql in k.lower()][:5] + return out + + +def health(): + st = store.stats() + fl = source_state.flags() + all_s = sources.all_sources() + return {"ok": True, "items": st["items"], "sources": len(all_s), + "alive": sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0), + "runs": st["runs"], "uptime": int(time.time() - _START), + "db_mb": round(config.DB.stat().st_size / 1048576, 1) if config.DB.exists() else 0} + + +_START = time.time() + + +# ---------- telegram ---------- + +def tg_status(): + from . import tgsess + out = {"enabled": bool(tgsess.enabled()), "limit": 100, "me": None, "error": None} + try: + cfg = config.tg_cfg() + out["limit"] = int(cfg.get("limit") or 100) + out["session"] = os.path.basename(str(cfg.get("session") or "")) or "" + except Exception: + pass + if out["enabled"]: + try: + out["me"] = tgsess.me() + except Exception as e: + out["error"] = "%s: %s" % (type(e).__name__, str(e)[:160]) + return out + + +def tg_search(q, limit=40, min_members=0): + from . import tgsess + if not tgsess.enabled(): + return {"ok": False, "error": "сессия не настроена", "channels": []} + rows = tgsess.search(q, limit=limit) + rows = [r for r in rows if int(r.get("participants") or 0) >= int(min_members or 0)] + rows.sort(key=lambda r: -(r.get("participants") or 0)) + return {"ok": True, "query": q, "channels": rows} + + +def tg_pull(handles, limit=100): + """Тянет посты через сессию и складывает в общий склад. Возвращает счётчики.""" + from . import tgsess + from .model import Item + if not tgsess.enabled(): + return {"ok": False, "error": "сессия не настроена", "new": 0} + chans = tgsess.fetch([h for h in handles if h], limit=limit) + items, errors = [], [] + for ch in chans: + if ch.get("error"): + errors.append({"handle": ch.get("handle"), "error": ch["error"]}) + for p in ch.get("posts") or []: + items.append(Item("tg_sess", "post", url=p.get("url") or "", + title=p.get("title") or "", text=p.get("text") or "", + source="tg_sess_" + str(ch.get("handle") or ""), + published=(p.get("date") or "")[:10], + extra={"views": p.get("views"), "handle": ch.get("handle")})) + new, dup = store.add(items) if items else (0, 0) + return {"ok": True, "channels": len(chans), "fetched": len(items), "new": new, "dup": dup, + "errors": errors[:20]} + + +# ---------- дайджест ---------- + +def digest(days=1, limit=30): + """Markdown-дайджест: главное за период по score, уязвимости, сливы, здоровье реестра.""" + days = max(1, min(30, int(days or 1))) + since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat() + rows, total = store.browse2(limit=limit, offset=0, sort="score", since=since) + cves, n_cve = store.browse2(limit=12, offset=0, sort="fresh", typ="cve", since=since) + leaks, n_leak = store.browse2(limit=10, offset=0, sort="fresh", typ="leak", since=since) + fl = source_state.flags() + all_s = sources.all_sources() + alive = sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0) + ab = abuse_mod.scan(limit=8) + L = [] + L.append("# omni · дайджест за %d дн. (%s)" % (days, _dt.date.today().isoformat())) + L.append("") + L.append("Записей за период: **%d**. Реестр: %d источников, живых %d." % (total, len(all_s), alive)) + L.append("") + L.append("## Главное по score") + for r in rows[:15]: + t = _clean(r.get("title"), 160) + L.append("- [%s](%s) — `%s` · %s" % (t, r.get("url") or "#", r.get("source"), r.get("published") or "")) + L.append("") + L.append("## Уязвимости (%d)" % n_cve) + for r in cves: + L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source"))) + L.append("") + L.append("## Сливы и утечки (%d)" % n_leak) + for r in leaks: + L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source"))) + L.append("") + L.append("## Абузы: промо, триалы, ключи") + for it in (ab.get("items") or [])[:8]: + L.append("- [%s](%s) — `%s` · score %s" % (_clean(it.get("title"), 140), it.get("url") or "#", + it.get("source"), it.get("abuse_score"))) + md = "\n".join(L) + return {"ok": True, "days": days, "md": md, "n": total, "path": str(_write_digest(md))} + + +def _write_digest(md): + p = config.DATA / "digest_latest.md" + try: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(md, encoding="utf-8") + except Exception: + pass + return p + + +# ---------- watchlist ---------- + +WATCH = config.DATA / "watch.json" + + +def _watch_load(): + try: + d = json.loads(WATCH.read_text(encoding="utf-8")) + if isinstance(d, list): + return d + except Exception: + pass + return [] + + +def _watch_save(rows): + try: + WATCH.parent.mkdir(parents=True, exist_ok=True) + WATCH.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8") + except Exception: + pass + + +def watch_list(hits=3): + """Правила + сколько записей подошло после добавления и 3 последних примера.""" + rows = _watch_load() + out = [] + for r in rows: + q = (r.get("q") or "").strip() + typ = r.get("type") or "all" + since = _dt.datetime.fromtimestamp(int(r.get("added") or 0), _dt.timezone.utc).date().isoformat() + items, n = store.browse2(q=q, limit=hits, offset=0, sort="fresh", typ=typ, since=since) + out.append({**r, "count": n, + "latest": [{"id": x.get("id"), "title": _clean(x.get("title"), 120), + "source": x.get("source"), "url": x.get("url")} for x in items]}) + return {"ok": True, "rules": out} + + +def watch_add(q, typ="all"): + q = (q or "").strip() + if len(q) < 2: + return {"error": "нужно минимум 2 символа"} + rows = _watch_load() + for r in rows: + if r.get("q", "").lower() == q.lower() and r.get("type", "all") == (typ or "all"): + return {"ok": True, "rule": r, "existed": True} + rule = {"id": "%x" % int(time.time() * 1000), "q": q, "type": typ or "all", "added": int(time.time())} + rows.append(rule) + _watch_save(rows) + return {"ok": True, "rule": rule} + + +def watch_remove(rid): + rows = [r for r in _watch_load() if r.get("id") != rid] + _watch_save(rows) + return {"ok": True, "left": len(rows)} + + +# ---------- источник подробно ---------- + +def source_detail(name): + fl = source_state.flags() + src = next((s for s in sources.all_sources() if s.name == name), None) + if not src: + return {"error": "not found"} + with store.db() as c: + stored = c.execute("SELECT COUNT(*) FROM items WHERE source=?", (name,)).fetchone()[0] + rows = c.execute("SELECT id,title,url,types,published,score FROM items WHERE source=? " + "ORDER BY id DESC LIMIT 12", (name,)).fetchall() + runs = c.execute("SELECT ts, got, kept, status, query FROM runs WHERE source=? " + "ORDER BY id DESC LIMIT 15", (name,)).fetchall() + return {"ok": True, + "source": {"name": src.name, "category": src.category, "kind": src.kind, "url": src.url, + "adapter": src.adapter, "tags": list(src.tags), "limit": src.limit}, + "state": fl.get(name, {}), "stored": stored, + "latest": [{"id": r[0], "title": _clean(r[1], 160), "url": r[2], "types": r[3], + "published": r[4], "score": r[5]} for r in rows], + "runs": [{"when": time.strftime("%Y-%m-%d %H:%M", time.localtime(r[0] or 0)), + "got": r[1], "kept": r[2], "status": r[3], "query": r[4]} for r in runs]} + + +# ---------- экспорт ---------- + +def export_json(**kw): + rows, total = store.browse2(**kw) + return {"ok": True, "total": total, "count": len(rows), "items": rows} \ No newline at end of file diff --git a/omni/serve.py b/omni/serve.py index 0100dcc..784d958 100644 --- a/omni/serve.py +++ b/omni/serve.py @@ -12,7 +12,7 @@ import urllib.parse from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from . import feeds, net, sources, source_state, store +from . import feeds, net, panel_api, sources, source_state, store WEB = pathlib.Path(__file__).resolve().parent.parent / "web" JOBS = {} @@ -353,8 +353,14 @@ class Handler(BaseHTTPRequestHandler): sort = (q.get("sort") or ["score"])[0] priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on") typ = (q.get("type") or ["all"])[0] + src = (q.get("src") or [""])[0].strip() or None + days = _intv(q, "days", 0, 0, 3650) + since = None + if days: + import datetime as _dt + since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat() rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, - sort=sort, priced=priced, typ=typ) + sort=sort, priced=priced, typ=typ, since=since, src=src) for _r in rows: _r["title"] = _clean_txt(_r.get("title"), 180) _r["text"] = _clean_txt(_r.get("text"), 260) @@ -387,6 +393,71 @@ class Handler(BaseHTTPRequestHandler): self.wfile.write(data) return + if path.startswith("/api/item/"): + try: + iid = int(path.rsplit("/", 1)[-1]) + except ValueError: + return self._json({"error": "bad id"}, 400) + return self._json(panel_api.item_detail(iid)) + + if path == "/api/timeline": + days = _intv(q, "days", 30, 1, 365) + cat = (q.get("category") or [None])[0] + return self._json(panel_api.timeline(days=days, cat=cat)) + + if path == "/api/suggest": + return self._json(panel_api.suggest((q.get("q") or [""])[0], _intv(q, "limit", 8, 1, 20))) + + if path == "/api/health": + return self._json(panel_api.health()) + + if path == "/api/tg/status": + return self._json(panel_api.tg_status()) + + if path == "/api/tg/search": + term = (q.get("q") or [""])[0].strip() + if len(term) < 2: + return self._json({"ok": False, "error": "нужно минимум 2 символа", "channels": []}) + return self._json(panel_api.tg_search(term, limit=_intv(q, "limit", 40, 1, 100), + min_members=_intv(q, "min", 0, 0, 10_000_000))) + + if path == "/api/source": + name = (q.get("name") or [""])[0].strip() + return self._json(panel_api.source_detail(name) if name else {"error": "name required"}) + + if path == "/api/watch": + return self._json(panel_api.watch_list(hits=_intv(q, "hits", 3, 1, 10))) + + if path == "/api/digest": + days = _intv(q, "days", 1, 1, 30) + res = panel_api.digest(days=days, limit=_intv(q, "limit", 30, 5, 100)) + if (q.get("download") or ["0"])[0] in ("1", "true"): + data = res["md"].encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/markdown; charset=utf-8") + self.send_header("Content-Disposition", 'attachment; filename="omni-digest.md"') + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + return self._json(res) + + if path == "/api/export.json": + cat = (q.get("category") or [None])[0] + rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(), + names=_names_in_category(cat), + limit=_intv(q, "limit", 500, 1, 5000), + sort=(q.get("sort") or ["score"])[0], + typ=(q.get("type") or ["all"])[0]) + data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Disposition", 'attachment; filename="omni.json"') + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + if path.startswith("/api/job/"): jid = path.rsplit("/", 1)[-1] return self._json(JOBS.get(jid) or {"state": "unknown"}) @@ -414,6 +485,25 @@ class Handler(BaseHTTPRequestHandler): res = _d.discover(limit_hosts=hosts, workers=8, max_add=add, dry=dry) return self._json(res) + if u.path == "/api/tg/pull": + handles = [h for h in (body.get("handles") or []) if isinstance(h, str)][:20] + if not handles: + return self._json({"error": "handles required"}, 400) + jid = _jid("tg") + _job(jid, state="running") + threading.Thread(target=lambda: _job(jid, **panel_api.tg_pull(handles, int(body.get("limit") or 100))), + daemon=True).start() + return self._json({"job": jid}) + + if u.path == "/api/watch/add": + return self._json(panel_api.watch_add(body.get("q"), body.get("type") or "all")) + + if u.path == "/api/watch/remove": + rid = body.get("id") + if not rid: + return self._json({"error": "id required"}, 400) + return self._json(panel_api.watch_remove(str(rid))) + if u.path == "/api/feed": jid = _jid("feed") _job(jid, state="running") diff --git a/omni/store.py b/omni/store.py index 7ec5c3d..803bcb0 100644 --- a/omni/store.py +++ b/omni/store.py @@ -182,13 +182,24 @@ def browse(category=None, q="", limit=60, offset=0): return [dict(zip(keys, r)) for r in rows] -def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None): - """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total.""" +def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None, + since=None, src=None): + """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total. + + since — дата YYYY-MM-DD (включительно), src — точное имя источника. + """ where = "WHERE 1=1 " args = [] if names: where += "AND source IN (%s) " % ",".join("?" * len(names)) args += list(names) + if src: + where += "AND source = ? " + args += [src] + if since: + where += ("AND (CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' " + "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END) >= ? ") + args += [str(since)[:10]] if q: where += "AND (title LIKE ? OR text LIKE ?) " args += [f"%{q}%", f"%{q}%"] @@ -202,10 +213,10 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty "old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC") with db() as c: total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0] - rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types " + rows = c.execute("SELECT id,provider,kind,source,title,text,author,published,score,price,currency,url,types " "FROM items " + where + order + " LIMIT ? OFFSET ?", args + [limit, offset]).fetchall() - keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types") + keys = ("id", "provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types") return [dict(zip(keys, r)) for r in rows], total diff --git a/omni/tgsess.py b/omni/tgsess.py index dd3a9e2..aba9aa7 100644 --- a/omni/tgsess.py +++ b/omni/tgsess.py @@ -132,3 +132,43 @@ def search(query, limit=40): api_id = int(os.environ["OMNI_TG_API_ID"]) api_hash = os.environ["OMNI_TG_API_HASH"] return asyncio.run(_search(query, limit, session, api_id, api_hash)) + + +_ME = {} + + +async def _me(session, api_id, api_hash, timeout=40): + from telethon import TelegramClient + + client = TelegramClient(session, api_id, api_hash, connection_retries=1, timeout=timeout / 2) + await asyncio.wait_for(client.connect(), timeout) + try: + u = await asyncio.wait_for(client.get_me(), timeout) + if not u: + return {} + return {"id": getattr(u, "id", None), "username": getattr(u, "username", None), + "first_name": getattr(u, "first_name", None), "premium": bool(getattr(u, "premium", False)), + "phone_tail": (str(getattr(u, "phone", "") or "")[-4:] or None)} + finally: + try: + await asyncio.wait_for(client.disconnect(), 10) + except Exception: + pass + + +def me(): + """Аккаунт сессии: {id, username, first_name, premium}. Кэш на процесс.""" + if _ME: + return _ME + if not enabled(): + return {} + try: + from . import config + config.tg_cfg() + except Exception: + pass + session = os.environ["OMNI_TG_SESSION"] + api_id = int(os.environ["OMNI_TG_API_ID"]) + api_hash = os.environ["OMNI_TG_API_HASH"] + _ME.update(asyncio.run(_me(session, api_id, api_hash)) or {}) + return _ME diff --git a/tools/fresh_candidates.py b/tools/fresh_candidates.py new file mode 100644 index 0000000..768eeaa --- /dev/null +++ b/tools/fresh_candidates.py @@ -0,0 +1,149 @@ +"""Кандидаты для расширения реестра omni: security/CVE/leak/CN/RU/tg. + +Источник истины для tools/probe_fresh.py. Живые строки после прогона +переезжают в omni/sources_fresh.py (генерируется `--emit`). + +Формат dict совпадает с Source(**d). Адаптеры — только из feeds.ADAPTERS. +""" +import datetime + + +def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None): + return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter, + "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}} + + +# NVD recent: окно считается на импорте (API требует обе даты и <=120 дней) +_NOW = datetime.datetime.utcnow() +_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000") +_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000") + +CANDIDATES = [ + # ---------------- 1. security news / research (RSS) ---------------- + d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)), + d("gbhackers", "security", "rss", "https://gbhackers.com/feed/", "rss", ("news",)), + d("cybersecuritynews", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", ("news",)), + d("scmedia", "security", "rss", "https://www.scmagazine.com/rss.xml", "rss", ("news",)), + d("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ("news",)), + d("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")), + d("seclists_bugtraq", "security", "rss", "https://seclists.org/rss/bugtraq.rss", "rss", ("mailing", "exploit")), + d("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)), + d("huntress_blog", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", ("research", "dfir")), + d("recordedfuture_blog", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ("research", "cti")), + d("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("research", "cti")), + d("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ("research",)), + d("sophos_news", "security", "rss", "https://news.sophos.com/en-us/feed/", "rss", ("research",)), + d("trendmicro_si", "security", "rss", "https://www.trendmicro.com/rss/security-intelligence.xml", "rss", ("research",)), + d("volexity_blog", "security", "rss", "https://www.volexity.com/blog/feed/", "rss", ("research", "dfir")), + d("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("dfir",)), + d("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research",)), + d("greynoise_blog", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ("research",)), + d("intezer_blog", "security", "rss", "https://intezer.com/blog/feed/", "rss", ("malware",)), + d("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)), + d("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)), + d("zscaler_research", "security", "rss", + "https://www.zscaler.com/blogs/security-research/rss.xml", "rss", ("research",)), + d("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)), + d("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/resources/blog/feed/", "rss", ("cti",)), + d("dragos_blog", "security", "rss", "https://www.dragos.com/feed/", "rss", ("ics",)), + d("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("research", "apt")), + d("dfir_report", "security", "rss", "https://thedfirreport.com/feed/", "rss", ("dfir", "ransomware")), + d("jfrog_blog", "security", "rss", "https://jfrog.com/blog/feed/", "rss", ("supply-chain",)), + d("gitguardian_blog", "security", "rss", "https://blog.gitguardian.com/rss/", "rss", ("secrets",)), + d("truffle_security", "security", "rss", "https://trufflesecurity.com/blog/rss.xml", "rss", ("secrets",)), + d("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)), + d("hudsonrock_blog", "security", "rss", "https://www.hudsonrock.com/blog/rss.xml", "rss", ("stealer",)), + d("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("blog", "writeup")), + d("0xdf", "security", "rss", "https://0xdf.gitlab.io/feed.xml", "rss", ("writeup", "htb")), + d("hackmag", "security", "rss", "https://hackmag.com/feed/", "rss", ("ru",)), + d("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("cloud",)), + + # ---------------- 2. CVE / vulnerabilities ---------------- + d("nvd_recent", "security", "json", + "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s" + % (_NVD_SINCE, _NVD_UNTIL), "nvd", ("cve", "nvd")), + d("nvd_exploited_kw", "security", "json", + "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", + "nvd", ("cve", "exploited")), + d("github_advisories_api", "security", "json", + "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")), + d("cve_circl_last", "security", "json", "https://cve.circl.lu/api/last/40", "cve_circl", ("cve",)), + d("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")), + d("gentoo_glsa", "security", "rss", "https://security.gentoo.org/glsa/feed.xml", "rss", ("cve", "linux")), + d("arch_linux_asa", "security", "rss", "https://security.archlinux.org/feed.atom", "rss", ("cve", "linux")), + d("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing", ("cve", "linux"), + detail=False, extra={"include_prefixes": ["/security/2026/"], "exclude_exact": ["/security/"]}), + d("oss_security_openwall", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing", + ("mailing", "cve"), extra={"include_prefixes": ["/lists/oss-security/2026/"]}), + d("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", + ("cve", "browser"), extra={"include_prefixes": ["/en-US/security/advisories/mfsa"]}), + d("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", + "rss", ("cve", "browser")), + d("cvefeed_latest", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ("cve",)), + d("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)), + d("cxsecurity", "security", "rss", "https://cxsecurity.com/rss/", "rss", ("exploit",)), + d("vulmon_recent", "security", "rss", "https://vulmon.com/rss/recent", "rss", ("cve",)), + d("cisa_ics_advisories", "security", "rss", + "https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml", "rss", ("ics", "cve")), + d("cisa_advisories", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", ("cve",)), + d("epss_recent", "security", "json", "https://api.first.org/data/v1/epss?limit=25&order=!epss", "generic_json", + ("cve", "epss")), + + # ---------------- 3. leaks / ransomware / underground ---------------- + d("darknetlive", "abuse", "rss", "https://darknetlive.com/feed/", "rss", ("darknet", "news")), + d("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")), + d("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")), + d("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")), + d("ransomware_live_recent", "abuse", "json", "https://api.ransomware.live/recentvictims", "generic_json", + ("ransomware",)), + d("ransomlook_recent", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", ("ransomware",)), + d("ransomware_live_groups", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", ("ransomware",)), + d("shadowserver_news", "abuse", "rss", "https://www.shadowserver.org/news/feed/", "rss", ("infra",)), + + # ---------------- 4. китайские security ---------------- + d("anquanke_rss", "security", "rss", "https://www.anquanke.com/rss", "rss", ("cn", "news")), + d("freebuf_feed", "security", "rss", "https://www.freebuf.com/feed", "rss", ("cn", "news")), + d("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")), + d("seebug_paper", "security", "rss", "https://paper.seebug.org/rss/", "rss", ("cn", "research")), + d("xz_aliyun", "security", "html", "https://xz.aliyun.com/", "html_listing", ("cn", "research"), + extra={"include_prefixes": ["/t/"]}), + d("aqniu", "security", "rss", "https://www.aqniu.com/feed", "rss", ("cn", "news")), + d("knownsec_blog", "security", "rss", "https://blog.knownsec.com/feed/", "rss", ("cn", "research")), + d("butian_forum", "forum", "html", "https://forum.butian.net/", "html_listing", ("cn", "forum"), + extra={"include_prefixes": ["/share/"]}), + d("kanxue_news_html", "forum", "html", "https://bbs.kanxue.com/", "html_listing", ("cn", "forum"), + extra={"include_prefixes": ["/thread-"]}), + d("t00ls_html", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum"), + extra={"include_prefixes": ["/article-"]}), + + # ---------------- 5. русскоязычные ---------------- + d("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")), + d("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")), + d("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")), + d("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")), + d("kaspersky_ru_blog", "security", "rss", "https://www.kaspersky.ru/blog/feed/", "rss", ("ru", "research")), + d("securitylab_rss", "security", "rss", "https://www.securitylab.ru/rss/all.xml", "rss", ("ru", "news")), + d("anti_malware_ru", "security", "rss", "https://www.anti-malware.ru/rss.xml", "rss", ("ru", "news")), + d("cnews_security", "security", "rss", "https://www.cnews.ru/inc/rss/news.xml", "rss", ("ru", "news")), + d("bdu_fstec", "security", "rss", "https://bdu.fstec.ru/rss/vulxml", "rss", ("ru", "cve")), + d("codeby_forum", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", ("ru", "forum")), + + # ---------------- 6. telegram (t.me/s, публичное превью) ---------------- + d("tg_thehackersnews", "security", "html", "https://t.me/s/TheHackersNews", "tg_web", ("tg", "news")), + d("tg_bleepingcomputer", "security", "html", "https://t.me/s/bleepingcomputer", "tg_web", ("tg", "news")), + d("tg_vxunderground", "security", "html", "https://t.me/s/vxunderground", "tg_web", ("tg", "malware")), + d("tg_secator", "security", "html", "https://t.me/s/SecAtor", "tg_web", ("tg", "ru")), + d("tg_itsecforu", "security", "html", "https://t.me/s/itsecforu", "tg_web", ("tg", "ru")), + d("tg_hitech_crime", "security", "html", "https://t.me/s/hitech_crime", "tg_web", ("tg", "ru")), + d("tg_codeby", "security", "html", "https://t.me/s/codeby", "tg_web", ("tg", "ru")), + d("tg_darkwebinformer", "abuse", "html", "https://t.me/s/DarkWebInformer", "tg_web", ("tg", "leak")), + d("tg_breachdetector", "abuse", "html", "https://t.me/s/breachdetector", "tg_web", ("tg", "leak")), + d("tg_databreaches", "abuse", "html", "https://t.me/s/databreaches", "tg_web", ("tg", "leak")), + d("tg_ransomware_live", "abuse", "html", "https://t.me/s/ransomware_live", "tg_web", ("tg", "ransomware")), + d("tg_cybersecuritynews", "security", "html", "https://t.me/s/cybersecuritynews", "tg_web", ("tg", "news")), + d("tg_ai_newz", "community", "html", "https://t.me/s/ai_newz", "tg_web", ("tg", "ai", "ru")), + d("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", + ("tg", "ai", "ru")), + d("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")), + d("tg_bo0om", "abuse", "html", "https://t.me/s/bo0om", "tg_web", ("tg", "leak", "ru")), +] \ No newline at end of file diff --git a/tools/probe_fresh.py b/tools/probe_fresh.py new file mode 100644 index 0000000..289b0b5 --- /dev/null +++ b/tools/probe_fresh.py @@ -0,0 +1,146 @@ +"""Живой прогон кандидатов из tools/fresh_candidates.py через штатные адаптеры omni. + + python tools/probe_fresh.py # прогон + data/fresh_probe.json + python tools/probe_fresh.py --emit # + перегенерация omni/sources_fresh.py + python tools/probe_fresh.py --only tg_ --workers 4 --limit 5 + +Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись. +Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина +в отчёте была фактической, а не «0 items». +""" +import argparse +import json +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tools")) + +from omni import feeds, net # noqa: E402 +from omni.sources import Source # noqa: E402 +from fresh_candidates import CANDIDATES # noqa: E402 + + +def _diag(src): + """Почему пусто: код ответа, блокировка, размер тела.""" + try: + r = net.get(src.url, timeout=20) + bits = ["http %s" % r.status] + if getattr(r, "blocked", False): + bits.append("blocked") + if getattr(r, "error", ""): + bits.append("err:%s" % str(r.error)[:120]) + bits.append("%db" % len(r.text or "")) + return ", ".join(bits) + except Exception as e: # noqa: BLE001 + return "exception:%s" % str(e)[:140] + + +def probe_one(src, limit): + t0 = time.time() + if src.adapter not in feeds.ADAPTERS: + return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter, + "url": src.url, "tags": list(src.tags), "ok": False, "n": 0, + "err": "no_adapter:%s" % src.adapter, "ms": 0, "samples": []} + try: + items = feeds.fetch(src, limit) or [] + err = "" + except Exception as e: # noqa: BLE001 + items, err = [], "raise:%s" % str(e)[:140] + ms = int((time.time() - t0) * 1000) + if not items and not err: + err = _diag(src) + return {"name": src.name, "category": src.category, "kind": src.kind, "adapter": src.adapter, + "url": src.url, "tags": list(src.tags), "ok": bool(items), "n": len(items), + "err": err, "ms": ms, + "samples": [{"title": i.title[:160], "url": i.url[:200]} for i in items[:3]]} + + +def emit(rows, dead, path): + """omni/sources_fresh.py: только живые, без импортов из omni.sources.""" + def q(s): + return '"%s"' % str(s).replace('"', '\\"') + + lines = [ + '"""Свежая волна источников omni: security/CVE/leaks/CN/RU/tg.', + "", + "Сгенерировано tools/probe_fresh.py по факту живого fetch (n>0).", + "Отклонённые кандидаты и причины — в хвостовом комментарии.", + "Импортов из omni.sources нет намеренно (циклический импорт).", + '"""', + "", + "", + "def fresh(S):", + ' """S — конструктор Source; возвращает только проверенные сетью источники."""', + " return [", + ] + for r in rows: + extra = "" + if r.get("extra"): + extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True) + limit = r.get("limit") or 15 + lim = "" if limit == 15 else ", limit=%d" % limit + det = ", detail=True" if r.get("detail") else "" + tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ()) + lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % ( + q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]), + tags, lim, det, extra)) + lines += [" ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"] + for r in dead: + lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90])) + text = "\n".join(lines) + "\n" + path.write_text(text, encoding="utf-8") + return len(rows) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=5) + ap.add_argument("--workers", type=int, default=6) + ap.add_argument("--only", default="") + ap.add_argument("--out", default="data/fresh_probe.json") + ap.add_argument("--emit", action="store_true") + ap.add_argument("--fresh", action="store_true", help="прогонять только то, что уже в sources_fresh.py") + a = ap.parse_args() + + cands = [c for c in CANDIDATES if not a.only or a.only in c["name"]] + srcs = [] + for c in cands: + try: + srcs.append(Source(**c)) + except Exception as e: # noqa: BLE001 + print("skip %s: %s" % (c.get("name"), e)) + + t0 = time.time() + with ThreadPoolExecutor(max_workers=a.workers) as ex: + rows = list(ex.map(lambda s: probe_one(s, a.limit), srcs)) + + alive = [r for r in rows if r["ok"]] + dead = [r for r in rows if not r["ok"]] + cats = {} + for r in alive: + cats[r["category"]] = cats.get(r["category"], 0) + 1 + out = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "limit": a.limit, "workers": a.workers, + "total": len(rows), "alive": len(alive), "dead": len(dead), "seconds": round(time.time() - t0, 1), + "by_category": dict(sorted(cats.items(), key=lambda kv: -kv[1])), + "results": sorted(rows, key=lambda r: (not r["ok"], r["category"], r["name"]))} + p = ROOT / a.out + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8") + + print("всего %d | живых %d | мёртвых %d | %.1fs" % (len(rows), len(alive), len(dead), time.time() - t0)) + print("по категориям:", out["by_category"]) + for r in dead: + print(" dead %-24s %s" % (r["name"], r["err"][:110])) + + if a.emit: + n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py") + print("omni/sources_fresh.py: %d источников" % n) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/tools/ui_smoke.py b/tools/ui_smoke.py new file mode 100644 index 0000000..d67b8c7 --- /dev/null +++ b/tools/ui_smoke.py @@ -0,0 +1,96 @@ +"""Смоук панели в реальном Chromium: скриншоты + ошибки консоли. + +python tools/ui_smoke.py [--url http://127.0.0.1:8078/] [--shots data/shots] +""" +import argparse +import asyncio +import json +import pathlib +import sys + +from playwright.async_api import async_playwright + + +async def run(url, out): + out.mkdir(parents=True, exist_ok=True) + errors, report = [], {} + async with async_playwright() as p: + b = await p.chromium.launch() + pg = await b.new_page(viewport={"width": 1500, "height": 950}) + pg.on("console", lambda m: errors.append([m.type, m.text]) if m.type in ("error",) else None) + pg.on("pageerror", lambda e: errors.append(["pageerror", str(e)])) + await pg.goto(url, wait_until="domcontentloaded", timeout=30000) + await pg.wait_for_selector(".kpi", timeout=20000) + await pg.wait_for_timeout(1200) + report["title"] = await pg.title() + report["nav"] = await pg.locator(".nav-i").count() + report["kpi"] = (await pg.locator(".kpi").all_inner_texts())[:4] + await pg.screenshot(path=str(out / "v2_overview.png"), full_page=False) + + await pg.click("[data-nav=feed]") + await pg.wait_for_selector("#list .item", timeout=20000) + report["feed_rows"] = await pg.locator(".item").count() + await pg.screenshot(path=str(out / "v2_feed.png")) + await pg.locator(".item").first.click() + await pg.wait_for_timeout(1200) + report["drawer"] = (await pg.locator("#d-title").inner_text())[:100] + await pg.screenshot(path=str(out / "v2_drawer.png")) + await pg.keyboard.press("Escape") + + await pg.click("[data-nav=sources]") + await pg.wait_for_timeout(1500) + report["src_rows"] = await pg.locator("#src-tbl tbody tr").count() + await pg.screenshot(path=str(out / "v2_sources.png")) + + await pg.click("[data-nav=abuse]") + await pg.wait_for_timeout(1500) + report["abuse_rows"] = await pg.locator(".item").count() + await pg.screenshot(path=str(out / "v2_abuse.png")) + + await pg.click("[data-nav=runs]") + await pg.wait_for_timeout(900) + report["run_rows"] = await pg.locator("#r-tbl tbody tr").count() + await pg.screenshot(path=str(out / "v2_runs.png")) + + await pg.click("[data-nav=tg]") + await pg.wait_for_timeout(2000) + report["tg"] = (await pg.locator(".card").first.inner_text())[:160].replace("\n", " | ") + await pg.screenshot(path=str(out / "v2_tg.png")) + + await pg.click("[data-nav=keys]") + await pg.wait_for_timeout(800) + report["key_rows"] = await pg.locator("#k-tbl tbody tr").count() + await pg.screenshot(path=str(out / "v2_keys.png")) + + await pg.click("[data-nav=digest]") + await pg.wait_for_timeout(500) + await pg.click("[data-act=digest]") + await pg.wait_for_timeout(6000) + report["digest_chars"] = len(await pg.locator("#d-out").inner_text()) + await pg.screenshot(path=str(out / "v2_digest.png")) + + await pg.click("[data-nav=watch]") + await pg.wait_for_timeout(600) + await pg.screenshot(path=str(out / "v2_watch.png")) + + # мобильный вид + await pg.set_viewport_size({"width": 420, "height": 900}) + await pg.click("[data-nav=feed]") + await pg.wait_for_timeout(1200) + await pg.screenshot(path=str(out / "v2_mobile.png")) + await b.close() + report["console"] = errors[:12] + print(json.dumps(report, ensure_ascii=False, indent=1)) + return 0 if not errors else 1 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:8078/") + ap.add_argument("--shots", default="data/shots") + a = ap.parse_args() + sys.exit(asyncio.run(run(a.url, pathlib.Path(a.shots)))) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/web/index.html b/web/index.html index 3317b4c..c398514 100644 --- a/web/index.html +++ b/web/index.html @@ -3,1035 +3,945 @@ -omni · радар источников +omni · комбайн источников
-