abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)
Some checks failed
ci / test (3.10) (push) Has been cancelled
ci / test (3.11) (push) Has been cancelled
ci / test (3.12) (push) Has been cancelled

This commit is contained in:
Vlad Gothbreach
2026-10-27 01:21:01 +03:00
parent d0b2187319
commit 42a864419b
12 changed files with 1844 additions and 983 deletions

View File

@@ -167,14 +167,15 @@ def _connect():
return con
def rebuild(full=False, chunk=4000):
def rebuild(full=False, chunk=4000, auto=False):
"""Инкрементальный индекс абузов: classify по новым items, без полного перещёлкивания.
full=True (или OMNI_ABUSE_REBUILD=1) — пересчитать всю базу после правки правил.
auto=True (вызов из чтения) игнорирует env, иначе каждый scan делал бы полный проход.
Возвращает число записей, попавших в индекс на этом проходе.
"""
import os
if os.environ.get("OMNI_ABUSE_REBUILD") == "1":
if not auto and os.environ.get("OMNI_ABUSE_REBUILD") == "1":
full = True
con = _connect()
try:
@@ -207,7 +208,7 @@ def rebuild(full=False, chunk=4000):
def _rows(limit=400, tag=None, q=None, since_days=None):
"""Читает из индекса abuse_idx; индекс добивается свежими записями перед чтением."""
try:
rebuild()
rebuild(auto=True)
except Exception:
pass
sql = ("SELECT i.id, i.source, i.url, i.title, i.text, i.published, a.tags, i.score, a.score "

316
omni/panel_api.py Normal file
View File

@@ -0,0 +1,316 @@
"""Данные для панели v2: обзор, детали записи, TG-сессия, дайджест, watchlist.
Здесь только чтение и небольшие записи в data/*.json; HTTP-обвязка — в serve.py,
чтобы роуты оставались тонкими, а логика — тестируемой без поднятия сервера.
"""
import datetime as _dt
import json
import os
import re
import time
import urllib.parse
from . import abuse as abuse_mod
from . import config, sources, source_state, store, taxonomy
BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
"THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
def _clean(v, n=0):
import html as _h
if not v:
return ""
t = re.sub(r"<[^>]{1,200}>", " ", str(v))
for _ in range(2):
t = _h.unescape(t)
t = re.sub(r"\s+", " ", t).strip()
return t[:n].rstrip() if n else t
# ---------- обзор ----------
def timeline(days=30, cat=None):
"""Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями."""
days = max(1, min(365, int(days or 30)))
names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
where, args = "", []
if names is not None:
where = " WHERE source IN (%s)" % ",".join("?" * len(names))
args = list(names)
sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
" SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
" FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
got = {}
with store.db() as c:
for b, n, cve, sev in c.execute(sql, args):
if b:
got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
today = _dt.date.today()
out = []
for i in range(days - 1, -1, -1):
d = (today - _dt.timedelta(days=i)).isoformat()
n, cve, sev = got.get(d, (0, 0, 0))
out.append({"day": d, "n": n, "cve": cve, "sev": sev})
return {"ok": True, "days": days, "points": out,
"total": sum(p["n"] for p in out),
"max": max([p["n"] for p in out] or [0])}
def item_detail(iid):
"""Полная запись + соседи: свежее из того же источника и с того же домена."""
iid = int(iid)
keys = ("id", "provider", "kind", "source", "url", "title", "text", "author", "published",
"lang", "tags", "price", "currency", "score", "types")
with store.db() as c:
row = c.execute("SELECT " + ",".join(keys) + " FROM items WHERE id=?", (iid,)).fetchone()
if not row:
return {"error": "not found"}
it: dict = dict(zip(keys, row))
host = ""
try:
host = urllib.parse.urlsplit(it.get("url") or "").netloc.lower()
except Exception:
host = ""
same_src = c.execute("SELECT id, title, source, published FROM items WHERE source=? AND id!=? "
"ORDER BY id DESC LIMIT 6", (it["source"], iid)).fetchall()
same_host = []
if host:
same_host = c.execute("SELECT id, title, source, published FROM items "
"WHERE url LIKE ? AND id!=? ORDER BY id DESC LIMIT 6",
("%//" + host + "/%", iid)).fetchall()
it["title"] = _clean(it.get("title"), 400)
it["text"] = _clean(it.get("text"))
it["types"] = [t for t in (it.get("types") or "").split(",") if t]
it["types_labels"] = taxonomy.labels(it["types"])
it["tags"] = [t for t in (it.get("tags") or "").split(",") if t] if isinstance(it.get("tags"), str) else (it.get("tags") or [])
it["related_same_source"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
for r in same_src]
it["related_same_host"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
for r in same_host]
return {"ok": True, "item": it}
def suggest(q, limit=8):
"""Подсказки поиска: заголовки из базы + имена источников + категории."""
q = (q or "").strip()
out = {"titles": [], "sources": [], "categories": []}
if len(q) < 2:
return out
like = "%" + q + "%"
with store.db() as c:
for iid, title, src in c.execute("SELECT id,title,source FROM items WHERE title LIKE ? "
"ORDER BY id DESC LIMIT ?", (like, limit)):
out["titles"].append({"id": iid, "title": _clean(title, 120), "source": src})
ql = q.lower()
for s in sources.all_sources():
if ql in s.name.lower() or ql in (s.url or "").lower():
out["sources"].append({"name": s.name, "category": s.category})
if len(out["sources"]) >= limit:
break
cats = {}
for s in sources.all_sources():
cats[s.category] = cats.get(s.category, 0) + 1
out["categories"] = [{"name": k, "n": v} for k, v in sorted(cats.items()) if ql in k.lower()][:5]
return out
def health():
st = store.stats()
fl = source_state.flags()
all_s = sources.all_sources()
return {"ok": True, "items": st["items"], "sources": len(all_s),
"alive": sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0),
"runs": st["runs"], "uptime": int(time.time() - _START),
"db_mb": round(config.DB.stat().st_size / 1048576, 1) if config.DB.exists() else 0}
_START = time.time()
# ---------- telegram ----------
def tg_status():
from . import tgsess
out = {"enabled": bool(tgsess.enabled()), "limit": 100, "me": None, "error": None}
try:
cfg = config.tg_cfg()
out["limit"] = int(cfg.get("limit") or 100)
out["session"] = os.path.basename(str(cfg.get("session") or "")) or ""
except Exception:
pass
if out["enabled"]:
try:
out["me"] = tgsess.me()
except Exception as e:
out["error"] = "%s: %s" % (type(e).__name__, str(e)[:160])
return out
def tg_search(q, limit=40, min_members=0):
from . import tgsess
if not tgsess.enabled():
return {"ok": False, "error": "сессия не настроена", "channels": []}
rows = tgsess.search(q, limit=limit)
rows = [r for r in rows if int(r.get("participants") or 0) >= int(min_members or 0)]
rows.sort(key=lambda r: -(r.get("participants") or 0))
return {"ok": True, "query": q, "channels": rows}
def tg_pull(handles, limit=100):
"""Тянет посты через сессию и складывает в общий склад. Возвращает счётчики."""
from . import tgsess
from .model import Item
if not tgsess.enabled():
return {"ok": False, "error": "сессия не настроена", "new": 0}
chans = tgsess.fetch([h for h in handles if h], limit=limit)
items, errors = [], []
for ch in chans:
if ch.get("error"):
errors.append({"handle": ch.get("handle"), "error": ch["error"]})
for p in ch.get("posts") or []:
items.append(Item("tg_sess", "post", url=p.get("url") or "",
title=p.get("title") or "", text=p.get("text") or "",
source="tg_sess_" + str(ch.get("handle") or ""),
published=(p.get("date") or "")[:10],
extra={"views": p.get("views"), "handle": ch.get("handle")}))
new, dup = store.add(items) if items else (0, 0)
return {"ok": True, "channels": len(chans), "fetched": len(items), "new": new, "dup": dup,
"errors": errors[:20]}
# ---------- дайджест ----------
def digest(days=1, limit=30):
"""Markdown-дайджест: главное за период по score, уязвимости, сливы, здоровье реестра."""
days = max(1, min(30, int(days or 1)))
since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
rows, total = store.browse2(limit=limit, offset=0, sort="score", since=since)
cves, n_cve = store.browse2(limit=12, offset=0, sort="fresh", typ="cve", since=since)
leaks, n_leak = store.browse2(limit=10, offset=0, sort="fresh", typ="leak", since=since)
fl = source_state.flags()
all_s = sources.all_sources()
alive = sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0)
ab = abuse_mod.scan(limit=8)
L = []
L.append("# omni · дайджест за %d дн. (%s)" % (days, _dt.date.today().isoformat()))
L.append("")
L.append("Записей за период: **%d**. Реестр: %d источников, живых %d." % (total, len(all_s), alive))
L.append("")
L.append("## Главное по score")
for r in rows[:15]:
t = _clean(r.get("title"), 160)
L.append("- [%s](%s) — `%s` · %s" % (t, r.get("url") or "#", r.get("source"), r.get("published") or ""))
L.append("")
L.append("## Уязвимости (%d)" % n_cve)
for r in cves:
L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
L.append("")
L.append("## Сливы и утечки (%d)" % n_leak)
for r in leaks:
L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
L.append("")
L.append("## Абузы: промо, триалы, ключи")
for it in (ab.get("items") or [])[:8]:
L.append("- [%s](%s) — `%s` · score %s" % (_clean(it.get("title"), 140), it.get("url") or "#",
it.get("source"), it.get("abuse_score")))
md = "\n".join(L)
return {"ok": True, "days": days, "md": md, "n": total, "path": str(_write_digest(md))}
def _write_digest(md):
p = config.DATA / "digest_latest.md"
try:
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(md, encoding="utf-8")
except Exception:
pass
return p
# ---------- watchlist ----------
WATCH = config.DATA / "watch.json"
def _watch_load():
try:
d = json.loads(WATCH.read_text(encoding="utf-8"))
if isinstance(d, list):
return d
except Exception:
pass
return []
def _watch_save(rows):
try:
WATCH.parent.mkdir(parents=True, exist_ok=True)
WATCH.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
except Exception:
pass
def watch_list(hits=3):
"""Правила + сколько записей подошло после добавления и 3 последних примера."""
rows = _watch_load()
out = []
for r in rows:
q = (r.get("q") or "").strip()
typ = r.get("type") or "all"
since = _dt.datetime.fromtimestamp(int(r.get("added") or 0), _dt.timezone.utc).date().isoformat()
items, n = store.browse2(q=q, limit=hits, offset=0, sort="fresh", typ=typ, since=since)
out.append({**r, "count": n,
"latest": [{"id": x.get("id"), "title": _clean(x.get("title"), 120),
"source": x.get("source"), "url": x.get("url")} for x in items]})
return {"ok": True, "rules": out}
def watch_add(q, typ="all"):
q = (q or "").strip()
if len(q) < 2:
return {"error": "нужно минимум 2 символа"}
rows = _watch_load()
for r in rows:
if r.get("q", "").lower() == q.lower() and r.get("type", "all") == (typ or "all"):
return {"ok": True, "rule": r, "existed": True}
rule = {"id": "%x" % int(time.time() * 1000), "q": q, "type": typ or "all", "added": int(time.time())}
rows.append(rule)
_watch_save(rows)
return {"ok": True, "rule": rule}
def watch_remove(rid):
rows = [r for r in _watch_load() if r.get("id") != rid]
_watch_save(rows)
return {"ok": True, "left": len(rows)}
# ---------- источник подробно ----------
def source_detail(name):
fl = source_state.flags()
src = next((s for s in sources.all_sources() if s.name == name), None)
if not src:
return {"error": "not found"}
with store.db() as c:
stored = c.execute("SELECT COUNT(*) FROM items WHERE source=?", (name,)).fetchone()[0]
rows = c.execute("SELECT id,title,url,types,published,score FROM items WHERE source=? "
"ORDER BY id DESC LIMIT 12", (name,)).fetchall()
runs = c.execute("SELECT ts, got, kept, status, query FROM runs WHERE source=? "
"ORDER BY id DESC LIMIT 15", (name,)).fetchall()
return {"ok": True,
"source": {"name": src.name, "category": src.category, "kind": src.kind, "url": src.url,
"adapter": src.adapter, "tags": list(src.tags), "limit": src.limit},
"state": fl.get(name, {}), "stored": stored,
"latest": [{"id": r[0], "title": _clean(r[1], 160), "url": r[2], "types": r[3],
"published": r[4], "score": r[5]} for r in rows],
"runs": [{"when": time.strftime("%Y-%m-%d %H:%M", time.localtime(r[0] or 0)),
"got": r[1], "kept": r[2], "status": r[3], "query": r[4]} for r in runs]}
# ---------- экспорт ----------
def export_json(**kw):
rows, total = store.browse2(**kw)
return {"ok": True, "total": total, "count": len(rows), "items": rows}

View File

@@ -12,7 +12,7 @@ import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from . import feeds, net, sources, source_state, store
from . import feeds, net, panel_api, sources, source_state, store
WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
JOBS = {}
@@ -353,8 +353,14 @@ class Handler(BaseHTTPRequestHandler):
sort = (q.get("sort") or ["score"])[0]
priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
typ = (q.get("type") or ["all"])[0]
src = (q.get("src") or [""])[0].strip() or None
days = _intv(q, "days", 0, 0, 3650)
since = None
if days:
import datetime as _dt
since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
sort=sort, priced=priced, typ=typ)
sort=sort, priced=priced, typ=typ, since=since, src=src)
for _r in rows:
_r["title"] = _clean_txt(_r.get("title"), 180)
_r["text"] = _clean_txt(_r.get("text"), 260)
@@ -387,6 +393,71 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(data)
return
if path.startswith("/api/item/"):
try:
iid = int(path.rsplit("/", 1)[-1])
except ValueError:
return self._json({"error": "bad id"}, 400)
return self._json(panel_api.item_detail(iid))
if path == "/api/timeline":
days = _intv(q, "days", 30, 1, 365)
cat = (q.get("category") or [None])[0]
return self._json(panel_api.timeline(days=days, cat=cat))
if path == "/api/suggest":
return self._json(panel_api.suggest((q.get("q") or [""])[0], _intv(q, "limit", 8, 1, 20)))
if path == "/api/health":
return self._json(panel_api.health())
if path == "/api/tg/status":
return self._json(panel_api.tg_status())
if path == "/api/tg/search":
term = (q.get("q") or [""])[0].strip()
if len(term) < 2:
return self._json({"ok": False, "error": "нужно минимум 2 символа", "channels": []})
return self._json(panel_api.tg_search(term, limit=_intv(q, "limit", 40, 1, 100),
min_members=_intv(q, "min", 0, 0, 10_000_000)))
if path == "/api/source":
name = (q.get("name") or [""])[0].strip()
return self._json(panel_api.source_detail(name) if name else {"error": "name required"})
if path == "/api/watch":
return self._json(panel_api.watch_list(hits=_intv(q, "hits", 3, 1, 10)))
if path == "/api/digest":
days = _intv(q, "days", 1, 1, 30)
res = panel_api.digest(days=days, limit=_intv(q, "limit", 30, 5, 100))
if (q.get("download") or ["0"])[0] in ("1", "true"):
data = res["md"].encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/markdown; charset=utf-8")
self.send_header("Content-Disposition", 'attachment; filename="omni-digest.md"')
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
return self._json(res)
if path == "/api/export.json":
cat = (q.get("category") or [None])[0]
rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(),
names=_names_in_category(cat),
limit=_intv(q, "limit", 500, 1, 5000),
sort=(q.get("sort") or ["score"])[0],
typ=(q.get("type") or ["all"])[0])
data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Disposition", 'attachment; filename="omni.json"')
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
if path.startswith("/api/job/"):
jid = path.rsplit("/", 1)[-1]
return self._json(JOBS.get(jid) or {"state": "unknown"})
@@ -414,6 +485,25 @@ class Handler(BaseHTTPRequestHandler):
res = _d.discover(limit_hosts=hosts, workers=8, max_add=add, dry=dry)
return self._json(res)
if u.path == "/api/tg/pull":
handles = [h for h in (body.get("handles") or []) if isinstance(h, str)][:20]
if not handles:
return self._json({"error": "handles required"}, 400)
jid = _jid("tg")
_job(jid, state="running")
threading.Thread(target=lambda: _job(jid, **panel_api.tg_pull(handles, int(body.get("limit") or 100))),
daemon=True).start()
return self._json({"job": jid})
if u.path == "/api/watch/add":
return self._json(panel_api.watch_add(body.get("q"), body.get("type") or "all"))
if u.path == "/api/watch/remove":
rid = body.get("id")
if not rid:
return self._json({"error": "id required"}, 400)
return self._json(panel_api.watch_remove(str(rid)))
if u.path == "/api/feed":
jid = _jid("feed")
_job(jid, state="running")

View File

@@ -182,13 +182,24 @@ def browse(category=None, q="", limit=60, offset=0):
return [dict(zip(keys, r)) for r in rows]
def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None):
"""Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None,
since=None, src=None):
"""Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total.
since — дата YYYY-MM-DD (включительно), src — точное имя источника.
"""
where = "WHERE 1=1 "
args = []
if names:
where += "AND source IN (%s) " % ",".join("?" * len(names))
args += list(names)
if src:
where += "AND source = ? "
args += [src]
if since:
where += ("AND (CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
"THEN substr(published,1,10) ELSE date(seen,'unixepoch') END) >= ? ")
args += [str(since)[:10]]
if q:
where += "AND (title LIKE ? OR text LIKE ?) "
args += [f"%{q}%", f"%{q}%"]
@@ -202,10 +213,10 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty
"old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
with db() as c:
total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
rows = c.execute("SELECT id,provider,kind,source,title,text,author,published,score,price,currency,url,types "
"FROM items " + where + order + " LIMIT ? OFFSET ?",
args + [limit, offset]).fetchall()
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
keys = ("id", "provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
return [dict(zip(keys, r)) for r in rows], total

View File

@@ -132,3 +132,43 @@ def search(query, limit=40):
api_id = int(os.environ["OMNI_TG_API_ID"])
api_hash = os.environ["OMNI_TG_API_HASH"]
return asyncio.run(_search(query, limit, session, api_id, api_hash))
_ME = {}
async def _me(session, api_id, api_hash, timeout=40):
from telethon import TelegramClient
client = TelegramClient(session, api_id, api_hash, connection_retries=1, timeout=timeout / 2)
await asyncio.wait_for(client.connect(), timeout)
try:
u = await asyncio.wait_for(client.get_me(), timeout)
if not u:
return {}
return {"id": getattr(u, "id", None), "username": getattr(u, "username", None),
"first_name": getattr(u, "first_name", None), "premium": bool(getattr(u, "premium", False)),
"phone_tail": (str(getattr(u, "phone", "") or "")[-4:] or None)}
finally:
try:
await asyncio.wait_for(client.disconnect(), 10)
except Exception:
pass
def me():
"""Аккаунт сессии: {id, username, first_name, premium}. Кэш на процесс."""
if _ME:
return _ME
if not enabled():
return {}
try:
from . import config
config.tg_cfg()
except Exception:
pass
session = os.environ["OMNI_TG_SESSION"]
api_id = int(os.environ["OMNI_TG_API_ID"])
api_hash = os.environ["OMNI_TG_API_HASH"]
_ME.update(asyncio.run(_me(session, api_id, api_hash)) or {})
return _ME