abuse: полный пересчёт индекса только по явному вызову (env не тормозит scan)

omni/panel_api.py

@@ -0,0 +1,316 @@
+"""Данные для панели v2: обзор, детали записи, TG-сессия, дайджест, watchlist.
+
+Здесь только чтение и небольшие записи в data/*.json; HTTP-обвязка — в serve.py,
+чтобы роуты оставались тонкими, а логика — тестируемой без поднятия сервера.
+"""
+import datetime as _dt
+import json
+import os
+import re
+import time
+import urllib.parse
+
+from . import abuse as abuse_mod
+from . import config, sources, source_state, store, taxonomy
+
+BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' "
+          "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END")
+
+
+def _clean(v, n=0):
+    import html as _h
+    if not v:
+        return ""
+    t = re.sub(r"<[^>]{1,200}>", " ", str(v))
+    for _ in range(2):
+        t = _h.unescape(t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t[:n].rstrip() if n else t
+
+
+# ---------- обзор ----------
+
+def timeline(days=30, cat=None):
+    """Записей по дням + отдельно уязвимости и сливы. Пустые дни заполняются нулями."""
+    days = max(1, min(365, int(days or 30)))
+    names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
+    where, args = "", []
+    if names is not None:
+        where = " WHERE source IN (%s)" % ",".join("?" * len(names))
+        args = list(names)
+    sql = ("SELECT b, COUNT(*), SUM(CASE WHEN types LIKE '%,cve,%' THEN 1 ELSE 0 END),"
+           " SUM(CASE WHEN types LIKE '%,leak,%' OR types LIKE '%,cve,%' THEN 1 ELSE 0 END)"
+           " FROM (SELECT " + BUCKET + " AS b, types FROM items" + where + ") GROUP BY b ORDER BY b")
+    got = {}
+    with store.db() as c:
+        for b, n, cve, sev in c.execute(sql, args):
+            if b:
+                got[b] = (int(n or 0), int(cve or 0), int(sev or 0))
+    today = _dt.date.today()
+    out = []
+    for i in range(days - 1, -1, -1):
+        d = (today - _dt.timedelta(days=i)).isoformat()
+        n, cve, sev = got.get(d, (0, 0, 0))
+        out.append({"day": d, "n": n, "cve": cve, "sev": sev})
+    return {"ok": True, "days": days, "points": out,
+            "total": sum(p["n"] for p in out),
+            "max": max([p["n"] for p in out] or [0])}
+
+
+def item_detail(iid):
+    """Полная запись + соседи: свежее из того же источника и с того же домена."""
+    iid = int(iid)
+    keys = ("id", "provider", "kind", "source", "url", "title", "text", "author", "published",
+            "lang", "tags", "price", "currency", "score", "types")
+    with store.db() as c:
+        row = c.execute("SELECT " + ",".join(keys) + " FROM items WHERE id=?", (iid,)).fetchone()
+        if not row:
+            return {"error": "not found"}
+        it: dict = dict(zip(keys, row))
+        host = ""
+        try:
+            host = urllib.parse.urlsplit(it.get("url") or "").netloc.lower()
+        except Exception:
+            host = ""
+        same_src = c.execute("SELECT id, title, source, published FROM items WHERE source=? AND id!=? "
+                             "ORDER BY id DESC LIMIT 6", (it["source"], iid)).fetchall()
+        same_host = []
+        if host:
+            same_host = c.execute("SELECT id, title, source, published FROM items "
+                                  "WHERE url LIKE ? AND id!=? ORDER BY id DESC LIMIT 6",
+                                  ("%//" + host + "/%", iid)).fetchall()
+    it["title"] = _clean(it.get("title"), 400)
+    it["text"] = _clean(it.get("text"))
+    it["types"] = [t for t in (it.get("types") or "").split(",") if t]
+    it["types_labels"] = taxonomy.labels(it["types"])
+    it["tags"] = [t for t in (it.get("tags") or "").split(",") if t] if isinstance(it.get("tags"), str) else (it.get("tags") or [])
+    it["related_same_source"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
+                                 for r in same_src]
+    it["related_same_host"] = [{"id": r[0], "title": _clean(r[1], 140), "source": r[2], "published": r[3]}
+                               for r in same_host]
+    return {"ok": True, "item": it}
+
+
+def suggest(q, limit=8):
+    """Подсказки поиска: заголовки из базы + имена источников + категории."""
+    q = (q or "").strip()
+    out = {"titles": [], "sources": [], "categories": []}
+    if len(q) < 2:
+        return out
+    like = "%" + q + "%"
+    with store.db() as c:
+        for iid, title, src in c.execute("SELECT id,title,source FROM items WHERE title LIKE ? "
+                                         "ORDER BY id DESC LIMIT ?", (like, limit)):
+            out["titles"].append({"id": iid, "title": _clean(title, 120), "source": src})
+    ql = q.lower()
+    for s in sources.all_sources():
+        if ql in s.name.lower() or ql in (s.url or "").lower():
+            out["sources"].append({"name": s.name, "category": s.category})
+            if len(out["sources"]) >= limit:
+                break
+    cats = {}
+    for s in sources.all_sources():
+        cats[s.category] = cats.get(s.category, 0) + 1
+    out["categories"] = [{"name": k, "n": v} for k, v in sorted(cats.items()) if ql in k.lower()][:5]
+    return out
+
+
+def health():
+    st = store.stats()
+    fl = source_state.flags()
+    all_s = sources.all_sources()
+    return {"ok": True, "items": st["items"], "sources": len(all_s),
+            "alive": sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0),
+            "runs": st["runs"], "uptime": int(time.time() - _START),
+            "db_mb": round(config.DB.stat().st_size / 1048576, 1) if config.DB.exists() else 0}
+
+
+_START = time.time()
+
+
+# ---------- telegram ----------
+
+def tg_status():
+    from . import tgsess
+    out = {"enabled": bool(tgsess.enabled()), "limit": 100, "me": None, "error": None}
+    try:
+        cfg = config.tg_cfg()
+        out["limit"] = int(cfg.get("limit") or 100)
+        out["session"] = os.path.basename(str(cfg.get("session") or "")) or ""
+    except Exception:
+        pass
+    if out["enabled"]:
+        try:
+            out["me"] = tgsess.me()
+        except Exception as e:
+            out["error"] = "%s: %s" % (type(e).__name__, str(e)[:160])
+    return out
+
+
+def tg_search(q, limit=40, min_members=0):
+    from . import tgsess
+    if not tgsess.enabled():
+        return {"ok": False, "error": "сессия не настроена", "channels": []}
+    rows = tgsess.search(q, limit=limit)
+    rows = [r for r in rows if int(r.get("participants") or 0) >= int(min_members or 0)]
+    rows.sort(key=lambda r: -(r.get("participants") or 0))
+    return {"ok": True, "query": q, "channels": rows}
+
+
+def tg_pull(handles, limit=100):
+    """Тянет посты через сессию и складывает в общий склад. Возвращает счётчики."""
+    from . import tgsess
+    from .model import Item
+    if not tgsess.enabled():
+        return {"ok": False, "error": "сессия не настроена", "new": 0}
+    chans = tgsess.fetch([h for h in handles if h], limit=limit)
+    items, errors = [], []
+    for ch in chans:
+        if ch.get("error"):
+            errors.append({"handle": ch.get("handle"), "error": ch["error"]})
+        for p in ch.get("posts") or []:
+            items.append(Item("tg_sess", "post", url=p.get("url") or "",
+                              title=p.get("title") or "", text=p.get("text") or "",
+                              source="tg_sess_" + str(ch.get("handle") or ""),
+                              published=(p.get("date") or "")[:10],
+                              extra={"views": p.get("views"), "handle": ch.get("handle")}))
+    new, dup = store.add(items) if items else (0, 0)
+    return {"ok": True, "channels": len(chans), "fetched": len(items), "new": new, "dup": dup,
+            "errors": errors[:20]}
+
+
+# ---------- дайджест ----------
+
+def digest(days=1, limit=30):
+    """Markdown-дайджест: главное за период по score, уязвимости, сливы, здоровье реестра."""
+    days = max(1, min(30, int(days or 1)))
+    since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat()
+    rows, total = store.browse2(limit=limit, offset=0, sort="score", since=since)
+    cves, n_cve = store.browse2(limit=12, offset=0, sort="fresh", typ="cve", since=since)
+    leaks, n_leak = store.browse2(limit=10, offset=0, sort="fresh", typ="leak", since=since)
+    fl = source_state.flags()
+    all_s = sources.all_sources()
+    alive = sum(1 for s in all_s if (fl.get(s.name, {}).get("items") or 0) > 0)
+    ab = abuse_mod.scan(limit=8)
+    L = []
+    L.append("# omni · дайджест за %d дн. (%s)" % (days, _dt.date.today().isoformat()))
+    L.append("")
+    L.append("Записей за период: **%d**. Реестр: %d источников, живых %d." % (total, len(all_s), alive))
+    L.append("")
+    L.append("## Главное по score")
+    for r in rows[:15]:
+        t = _clean(r.get("title"), 160)
+        L.append("- [%s](%s) — `%s` · %s" % (t, r.get("url") or "#", r.get("source"), r.get("published") or ""))
+    L.append("")
+    L.append("## Уязвимости (%d)" % n_cve)
+    for r in cves:
+        L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
+    L.append("")
+    L.append("## Сливы и утечки (%d)" % n_leak)
+    for r in leaks:
+        L.append("- [%s](%s) — `%s`" % (_clean(r.get("title"), 160), r.get("url") or "#", r.get("source")))
+    L.append("")
+    L.append("## Абузы: промо, триалы, ключи")
+    for it in (ab.get("items") or [])[:8]:
+        L.append("- [%s](%s) — `%s` · score %s" % (_clean(it.get("title"), 140), it.get("url") or "#",
+                                                   it.get("source"), it.get("abuse_score")))
+    md = "\n".join(L)
+    return {"ok": True, "days": days, "md": md, "n": total, "path": str(_write_digest(md))}
+
+
+def _write_digest(md):
+    p = config.DATA / "digest_latest.md"
+    try:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(md, encoding="utf-8")
+    except Exception:
+        pass
+    return p
+
+
+# ---------- watchlist ----------
+
+WATCH = config.DATA / "watch.json"
+
+
+def _watch_load():
+    try:
+        d = json.loads(WATCH.read_text(encoding="utf-8"))
+        if isinstance(d, list):
+            return d
+    except Exception:
+        pass
+    return []
+
+
+def _watch_save(rows):
+    try:
+        WATCH.parent.mkdir(parents=True, exist_ok=True)
+        WATCH.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
+    except Exception:
+        pass
+
+
+def watch_list(hits=3):
+    """Правила + сколько записей подошло после добавления и 3 последних примера."""
+    rows = _watch_load()
+    out = []
+    for r in rows:
+        q = (r.get("q") or "").strip()
+        typ = r.get("type") or "all"
+        since = _dt.datetime.fromtimestamp(int(r.get("added") or 0), _dt.timezone.utc).date().isoformat()
+        items, n = store.browse2(q=q, limit=hits, offset=0, sort="fresh", typ=typ, since=since)
+        out.append({**r, "count": n,
+                    "latest": [{"id": x.get("id"), "title": _clean(x.get("title"), 120),
+                                "source": x.get("source"), "url": x.get("url")} for x in items]})
+    return {"ok": True, "rules": out}
+
+
+def watch_add(q, typ="all"):
+    q = (q or "").strip()
+    if len(q) < 2:
+        return {"error": "нужно минимум 2 символа"}
+    rows = _watch_load()
+    for r in rows:
+        if r.get("q", "").lower() == q.lower() and r.get("type", "all") == (typ or "all"):
+            return {"ok": True, "rule": r, "existed": True}
+    rule = {"id": "%x" % int(time.time() * 1000), "q": q, "type": typ or "all", "added": int(time.time())}
+    rows.append(rule)
+    _watch_save(rows)
+    return {"ok": True, "rule": rule}
+
+
+def watch_remove(rid):
+    rows = [r for r in _watch_load() if r.get("id") != rid]
+    _watch_save(rows)
+    return {"ok": True, "left": len(rows)}
+
+
+# ---------- источник подробно ----------
+
+def source_detail(name):
+    fl = source_state.flags()
+    src = next((s for s in sources.all_sources() if s.name == name), None)
+    if not src:
+        return {"error": "not found"}
+    with store.db() as c:
+        stored = c.execute("SELECT COUNT(*) FROM items WHERE source=?", (name,)).fetchone()[0]
+        rows = c.execute("SELECT id,title,url,types,published,score FROM items WHERE source=? "
+                         "ORDER BY id DESC LIMIT 12", (name,)).fetchall()
+        runs = c.execute("SELECT ts, got, kept, status, query FROM runs WHERE source=? "
+                         "ORDER BY id DESC LIMIT 15", (name,)).fetchall()
+    return {"ok": True,
+            "source": {"name": src.name, "category": src.category, "kind": src.kind, "url": src.url,
+                       "adapter": src.adapter, "tags": list(src.tags), "limit": src.limit},
+            "state": fl.get(name, {}), "stored": stored,
+            "latest": [{"id": r[0], "title": _clean(r[1], 160), "url": r[2], "types": r[3],
+                        "published": r[4], "score": r[5]} for r in rows],
+            "runs": [{"when": time.strftime("%Y-%m-%d %H:%M", time.localtime(r[0] or 0)),
+                      "got": r[1], "kept": r[2], "status": r[3], "query": r[4]} for r in runs]}
+
+
+# ---------- экспорт ----------
+
+def export_json(**kw):
+    rows, total = store.browse2(**kw)
+    return {"ok": True, "total": total, "count": len(rows), "items": rows}