325 lines
14 KiB
Python
325 lines
14 KiB
Python
"""Данные для панели v2: обзор, детали записи, TG-сессия, дайджест, watchlist.
|
||
|
||
Здесь только чтение и небольшие записи в data/*.json; HTTP-обвязка — в serve.py,
|
||
чтобы роуты оставались тонкими, а логика — тестируемой без поднятия сервера.
|
||
"""
|
||
import datetime as _dt
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
import urllib.parse
|
||
|
||
from . import abuse as abuse_mod
|
||
from . import config, sources, source_state, store, taxonomy
|
||
|
||
BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
|
||
"THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
|
||
|
||
|
||
def _clean(v, n=0):
|
||
import html as _h
|
||
if not v:
|
||
return ""
|
||
t = re.sub(r"<[^>]{1,200}>", " ", str(v))
|
||
for _ in range(2):
|
||
t = _h.unescape(t)
|
||
t = re.sub(r"\s+", " ", t).strip()
|
||
return t[:n].rstrip() if n else t
|
||
|
||
|
||
# ---------- обзор ----------
|
||
|
||
BUCKET_SEEN = "date(seen,'unixepoch')"
|
||
|
||
|
||
def timeline(days=30, cat=None, mode="pub"):
|
||
"""Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями.
|
||
|
||
mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
|
||
mode="seen" — строго по времени импорта: «что комбайн принёс за день».
|
||
"""
|
||
days = max(1, min(365, int(days or 30)))
|
||
bucket = BUCKET_SEEN if mode == "seen" else BUCKET
|
||
names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
|
||
where, args = "", []
|
||
if names is not None:
|
||
where = " WHERE source IN (%s)" % ",".join("?" * len(names))
|
||
args = list(names)
|
||
sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
|
||
" SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
|
||
" FROM (SELECT " + bucket + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
|
||
got = {}
|
||
with store.db() as c:
|
||
for b, n, cve, sev in c.execute(sql, args):
|
||
if b:
|
||
got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
|
||
today = _dt.date.today()
|
||
out = []
|
||
for i in range(days - 1, -1, -1):
|
||
d = (today - _dt.timedelta(days=i)).isoformat()
|
||
n, cve, sev = got.get(d, (0, 0, 0))
|
||
out.append({"day": d, "n": n, "cve": cve, "sev": sev})
|
||
return {"ok": True, "days": days, "mode": mode, "points": out,
|
||
"total": sum(p["n"] for p in out),
|
||
"last24": (out[-1] or {}).get("n", 0),
|
||
"max": max([p["n"] for p in out] or [0])}
|
||
|
||
|
||
def item_detail(iid):
|
||
"""Полная запись + соседи: свежее из того же источника и с того же домена."""
|
||
iid = int(iid)
|
||
keys = ("id", "provider", "kind", "source", "url", "title", "text", "author", "published",
|
||
"lang", "tags", "price", "currency", "score", "types")
|
||
with store.db() as c:
|
||
row = c.execute("SELECT " + ",".join(keys) + " FROM items WHERE id=?", (iid,)).fetchone()
|
||
if not row:
|
||
return {"error": "not found"}
|
||
it: dict = dict(zip(keys, row))
|
||
host = ""
|
||
try:
|
||
host = urllib.parse.urlsplit(it.get("url") or "").netloc.lower()
|
||
except Exception:
|
||
host = ""
|
||
same_src = c.execute("SELECT id, title, source, published FROM items WHERE source=? AND id!=? "
|
||
"ORDER BY id DESC LIMIT 6", (it["source"], iid)).fetchall()
|
||
same_host = []
|
||
if host:
|
||
same_host = c.execute("SELECT id, title, source, published FROM items "
|
||
"WHERE url LIKE ? AND id!=? ORDER BY id DESC LIMIT 6",
|
||
("%//" + host + "/%", iid)).fetchall()
|
||
it["title"] = _clean(it.get("title"), 400)
|
||
it["text"] = _clean(it.get("text"))
|
||
it["types"] = [t for t in (it.get("types") or "").split(",") if t]
|
||
it["types_labels"] = taxonomy.labels(it["types"])
|
||
it["tags"] = [t for t in (it.get("tags") or "").split(",") if t] if isinstance(it.get("tags"), str) else (it.get("tags") or [])
|
||
it["related_same_source"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
|
||
for r in same_src]
|
||
it["related_same_host"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
|
||
for r in same_host]
|
||
return {"ok": True, "item": it}
|
||
|
||
|
||
def suggest(q, limit=8):
|
||
"""Подсказки поиска: заголовки из базы + имена источников + категории."""
|
||
q = (q or "").strip()
|
||
out = {"titles": [], "sources": [], "categories": []}
|
||
if len(q) < 2:
|
||
return out
|
||
like = "%" + q + "%"
|
||
with store.db() as c:
|
||
for iid, title, src in c.execute("SELECT id,title,source FROM items WHERE title LIKE ? "
|
||
"ORDER BY id DESC LIMIT ?", (like, limit)):
|
||
out["titles"].append({"id": iid, "title": _clean(title, 120), "source": src})
|
||
ql = q.lower()
|
||
for s in sources.all_sources():
|
||
if ql in s.name.lower() or ql in (s.url or "").lower():
|
||
out["sources"].append({"name": s.name, "category": s.category})
|
||
if len(out["sources"]) >= limit:
|
||
break
|
||
cats = {}
|
||
for s in sources.all_sources():
|
||
cats[s.category] = cats.get(s.category, 0) + 1
|
||
out["categories"] = [{"name": k, "n": v} for k, v in sorted(cats.items()) if ql in k.lower()][:5]
|
||
return out
|
||
|
||
|
||
def health():
|
||
st = store.stats()
|
||
fl = source_state.flags()
|
||
all_s = sources.all_sources()
|
||
return {"ok": True, "items": st["items"], "sources": len(all_s),
|
||
"alive": sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0),
|
||
"runs": st["runs"], "uptime": int(time.time() - _START),
|
||
"db_mb": round(config.DB.stat().st_size / 1048576, 1) if config.DB.exists() else 0}
|
||
|
||
|
||
_START = time.time()
|
||
|
||
|
||
# ---------- telegram ----------
|
||
|
||
def tg_status():
|
||
from . import tgsess
|
||
out = {"enabled": bool(tgsess.enabled()), "limit": 100, "me": None, "error": None}
|
||
try:
|
||
cfg = config.tg_cfg()
|
||
out["limit"] = int(cfg.get("limit") or 100)
|
||
out["session"] = os.path.basename(str(cfg.get("session") or "")) or ""
|
||
except Exception:
|
||
pass
|
||
if out["enabled"]:
|
||
try:
|
||
out["me"] = tgsess.me()
|
||
except Exception as e:
|
||
out["error"] = "%s: %s" % (type(e).__name__, str(e)[:160])
|
||
return out
|
||
|
||
|
||
def tg_search(q, limit=40, min_members=0):
|
||
from . import tgsess
|
||
if not tgsess.enabled():
|
||
return {"ok": False, "error": "сессия не настроена", "channels": []}
|
||
rows = tgsess.search(q, limit=limit)
|
||
rows = [r for r in rows if int(r.get("participants") or 0) >= int(min_members or 0)]
|
||
rows.sort(key=lambda r: -(r.get("participants") or 0))
|
||
return {"ok": True, "query": q, "channels": rows}
|
||
|
||
|
||
def tg_pull(handles, limit=100):
|
||
"""Тянет посты через сессию и складывает в общий склад. Возвращает счётчики."""
|
||
from . import tgsess
|
||
from .model import Item
|
||
if not tgsess.enabled():
|
||
return {"ok": False, "error": "сессия не настроена", "new": 0}
|
||
chans = tgsess.fetch([h for h in handles if h], limit=limit)
|
||
items, errors = [], []
|
||
for ch in chans:
|
||
if ch.get("error"):
|
||
errors.append({"handle": ch.get("handle"), "error": ch["error"]})
|
||
for p in ch.get("posts") or []:
|
||
items.append(Item("tg_sess", "post", url=p.get("url") or "",
|
||
title=p.get("title") or "", text=p.get("text") or "",
|
||
source="tg_sess_" + str(ch.get("handle") or ""),
|
||
published=(p.get("date") or "")[:10],
|
||
extra={"views": p.get("views"), "handle": ch.get("handle")}))
|
||
new, dup = store.add(items) if items else (0, 0)
|
||
return {"ok": True, "channels": len(chans), "fetched": len(items), "new": new, "dup": dup,
|
||
"errors": errors[:20]}
|
||
|
||
|
||
# ---------- дайджест ----------
|
||
|
||
def digest(days=1, limit=30):
|
||
"""Markdown-дайджест: главное за период по score, уязвимости, сливы, здоровье реестра."""
|
||
days = max(1, min(30, int(days or 1)))
|
||
since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
|
||
rows, total = store.browse2(limit=limit, offset=0, sort="score", since=since)
|
||
cves, n_cve = store.browse2(limit=12, offset=0, sort="fresh", typ="cve", since=since)
|
||
leaks, n_leak = store.browse2(limit=10, offset=0, sort="fresh", typ="leak", since=since)
|
||
fl = source_state.flags()
|
||
all_s = sources.all_sources()
|
||
alive = sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0)
|
||
ab = abuse_mod.scan(limit=8)
|
||
L = []
|
||
L.append("# omni · дайджест за %d дн. (%s)" % (days, _dt.date.today().isoformat()))
|
||
L.append("")
|
||
L.append("Записей за период: **%d**. Реестр: %d источников, живых %d." % (total, len(all_s), alive))
|
||
L.append("")
|
||
L.append("## Главное по score")
|
||
for r in rows[:15]:
|
||
t = _clean(r.get("title"), 160)
|
||
L.append("- [%s](%s) — `%s` · %s" % (t, r.get("url") or "#", r.get("source"), r.get("published") or ""))
|
||
L.append("")
|
||
L.append("## Уязвимости (%d)" % n_cve)
|
||
for r in cves:
|
||
L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
|
||
L.append("")
|
||
L.append("## Сливы и утечки (%d)" % n_leak)
|
||
for r in leaks:
|
||
L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
|
||
L.append("")
|
||
L.append("## Абузы: промо, триалы, ключи")
|
||
for it in (ab.get("items") or [])[:8]:
|
||
L.append("- [%s](%s) — `%s` · score %s" % (_clean(it.get("title"), 140), it.get("url") or "#",
|
||
it.get("source"), it.get("abuse_score")))
|
||
md = "\n".join(L)
|
||
return {"ok": True, "days": days, "md": md, "n": total, "path": str(_write_digest(md))}
|
||
|
||
|
||
def _write_digest(md):
|
||
p = config.DATA / "digest_latest.md"
|
||
try:
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text(md, encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
return p
|
||
|
||
|
||
# ---------- watchlist ----------
|
||
|
||
WATCH = config.DATA / "watch.json"
|
||
|
||
|
||
def _watch_load():
|
||
try:
|
||
d = json.loads(WATCH.read_text(encoding="utf-8"))
|
||
if isinstance(d, list):
|
||
return d
|
||
except Exception:
|
||
pass
|
||
return []
|
||
|
||
|
||
def _watch_save(rows):
|
||
try:
|
||
WATCH.parent.mkdir(parents=True, exist_ok=True)
|
||
WATCH.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def watch_list(hits=3):
|
||
"""Правила + сколько записей подошло после добавления и 3 последних примера."""
|
||
rows = _watch_load()
|
||
out = []
|
||
for r in rows:
|
||
q = (r.get("q") or "").strip()
|
||
typ = r.get("type") or "all"
|
||
since = _dt.datetime.fromtimestamp(int(r.get("added") or 0), _dt.timezone.utc).date().isoformat()
|
||
items, n = store.browse2(q=q, limit=hits, offset=0, sort="fresh", typ=typ, since=since)
|
||
out.append({**r, "count": n,
|
||
"latest": [{"id": x.get("id"), "title": _clean(x.get("title"), 120),
|
||
"source": x.get("source"), "url": x.get("url")} for x in items]})
|
||
return {"ok": True, "rules": out}
|
||
|
||
|
||
def watch_add(q, typ="all"):
|
||
q = (q or "").strip()
|
||
if len(q) < 2:
|
||
return {"error": "нужно минимум 2 символа"}
|
||
rows = _watch_load()
|
||
for r in rows:
|
||
if r.get("q", "").lower() == q.lower() and r.get("type", "all") == (typ or "all"):
|
||
return {"ok": True, "rule": r, "existed": True}
|
||
rule = {"id": "%x" % int(time.time() * 1000), "q": q, "type": typ or "all", "added": int(time.time())}
|
||
rows.append(rule)
|
||
_watch_save(rows)
|
||
return {"ok": True, "rule": rule}
|
||
|
||
|
||
def watch_remove(rid):
|
||
rows = [r for r in _watch_load() if r.get("id") != rid]
|
||
_watch_save(rows)
|
||
return {"ok": True, "left": len(rows)}
|
||
|
||
|
||
# ---------- источник подробно ----------
|
||
|
||
def source_detail(name):
|
||
fl = source_state.flags()
|
||
src = next((s for s in sources.all_sources() if s.name == name), None)
|
||
if not src:
|
||
return {"error": "not found"}
|
||
with store.db() as c:
|
||
stored = c.execute("SELECT COUNT(*) FROM items WHERE source=?", (name,)).fetchone()[0]
|
||
rows = c.execute("SELECT id,title,url,types,published,score FROM items WHERE source=? "
|
||
"ORDER BY id DESC LIMIT 12", (name,)).fetchall()
|
||
runs = c.execute("SELECT ts, got, kept, status, query FROM runs WHERE source=? "
|
||
"ORDER BY id DESC LIMIT 15", (name,)).fetchall()
|
||
return {"ok": True,
|
||
"source": {"name": src.name, "category": src.category, "kind": src.kind, "url": src.url,
|
||
"adapter": src.adapter, "tags": list(src.tags), "limit": src.limit},
|
||
"state": fl.get(name, {}), "stored": stored,
|
||
"latest": [{"id": r[0], "title": _clean(r[1], 160), "url": r[2], "types": r[3],
|
||
"published": r[4], "score": r[5]} for r in rows],
|
||
"runs": [{"when": time.strftime("%Y-%m-%d %H:%M", time.localtime(r[0] or 0)),
|
||
"got": r[1], "kept": r[2], "status": r[3], "query": r[4]} for r in runs]}
|
||
|
||
|
||
# ---------- экспорт ----------
|
||
|
||
def export_json(**kw):
|
||
rows, total = store.browse2(**kw)
|
||
return {"ok": True, "total": total, "count": len(rows), "items": rows} |