464 lines
20 KiB
Python
464 lines
20 KiB
Python
"""Панель v2: API-слой panel_api + регистрация роутов в serve. Без сети."""
|
|
import inspect
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT))
|
|
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolated_panel_data(tmp_path, monkeypatch):
|
|
"""Never touch the operator's database, watchlist, digest or TG session."""
|
|
from omni import abuse, config, panel_api, store, tgsess
|
|
from omni.model import Item
|
|
|
|
monkeypatch.setattr(config, "DATA", tmp_path)
|
|
monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3")
|
|
monkeypatch.setattr(abuse, "DB", config.DB)
|
|
monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json")
|
|
monkeypatch.setattr(config, "tg_cfg", lambda: {"session": "", "limit": 100})
|
|
monkeypatch.setattr(tgsess, "enabled", lambda: False)
|
|
today = time.strftime("%Y-%m-%d")
|
|
store.add([
|
|
Item(provider="pytest", kind="post", source="pytest-panel",
|
|
title="Firecrawl security dataset leak announcement",
|
|
text="Public incident report for regression coverage.",
|
|
url="https://example.invalid/seed", published=today),
|
|
Item(provider="pytest", kind="post", source="pytest-other",
|
|
title="Guide to observability and performance metrics",
|
|
text="A completely different article about tracing.",
|
|
url="https://example.invalid/other", published=today),
|
|
])
|
|
|
|
|
|
def _tmp_item(tag=None):
|
|
from omni import store
|
|
from omni.model import Item
|
|
|
|
tag = tag or str(time.time_ns())
|
|
it = Item(provider="pytest", kind="post", source="pytest-panel",
|
|
title="pytest panel %s" % tag, text="текст <b>%s</b> & xss" % tag,
|
|
url="https://example.invalid/p/%s" % tag, published=time.strftime("%Y-%m-%d"))
|
|
store.add([it])
|
|
with store.db() as c:
|
|
row = c.execute("SELECT id FROM items WHERE url=?", (it.url,)).fetchone()
|
|
return int(row[0]), tag
|
|
|
|
|
|
# ---------- timeline ----------
|
|
def test_timeline_shape_and_modes():
|
|
from omni import panel_api
|
|
|
|
for mode in ("pub", "seen"):
|
|
d = panel_api.timeline(days=7, mode=mode)
|
|
assert d["ok"] and d["mode"] == mode
|
|
assert len(d["points"]) == 7, "ровно N дней, пустые заполнены нулями"
|
|
assert d["total"] == sum(p["n"] for p in d["points"])
|
|
for p in d["points"]:
|
|
assert set(p) == {"day", "n", "cve", "sev"}
|
|
assert p["cve"] <= p["n"] and p["sev"] <= p["n"]
|
|
assert "last24" in panel_api.timeline(days=2)
|
|
|
|
|
|
def test_timeline_days_clamped():
|
|
"""0/отрицательное → 1 день, None → 30, огромное → потолок 365."""
|
|
from omni import panel_api
|
|
|
|
assert len(panel_api.timeline(days=None)["points"]) == 30
|
|
assert len(panel_api.timeline(days=0)["points"]) == 1
|
|
assert len(panel_api.timeline(days=-5)["points"]) == 1
|
|
assert len(panel_api.timeline(days=10_000)["points"]) == 365
|
|
|
|
|
|
# ---------- детали записи ----------
|
|
def test_item_detail_cleans_html_and_finds_siblings():
|
|
from omni import panel_api
|
|
|
|
iid, tag = _tmp_item()
|
|
d = panel_api.item_detail(iid)
|
|
assert d["ok"]
|
|
it = d["item"]
|
|
text = it["text"]
|
|
assert "<b>" not in text and "<" not in text, "сырые теги остались"
|
|
assert "&" not in text, "двойное экранирование"
|
|
assert tag in text
|
|
assert isinstance(it["related_same_source"], list)
|
|
assert all(r["source"] == "pytest-panel" for r in it["related_same_source"])
|
|
assert panel_api.item_detail(10 ** 12) == {"error": "not found"}
|
|
|
|
|
|
# ---------- подсказки ----------
|
|
def test_suggest_needs_two_chars_and_finds_source():
|
|
from omni import panel_api
|
|
|
|
assert panel_api.suggest("a") == {"titles": [], "sources": [], "categories": []}
|
|
d = panel_api.suggest("firecrawl")
|
|
assert isinstance(d["titles"], list) and isinstance(d["sources"], list)
|
|
if d["sources"]:
|
|
assert {"name", "category"} <= set(d["sources"][0])
|
|
|
|
|
|
# ---------- watchlist ----------
|
|
def test_watch_add_list_remove():
|
|
from omni import panel_api
|
|
|
|
q = "pytest-watch-%d" % time.time_ns()
|
|
r = panel_api.watch_add(q)
|
|
assert r["ok"] and r["rule"]["q"] == q
|
|
rid = r["rule"]["id"]
|
|
try:
|
|
rules = panel_api.watch_list()["rules"]
|
|
assert any(x["id"] == rid for x in rules)
|
|
assert panel_api.watch_remove(rid)["ok"]
|
|
assert not any(x["id"] == rid for x in panel_api.watch_list()["rules"])
|
|
finally:
|
|
panel_api.watch_remove(rid)
|
|
assert "error" in panel_api.watch_add("x")
|
|
|
|
|
|
# ---------- детали источника ----------
|
|
def test_source_detail_shape():
|
|
from omni import panel_api, sources
|
|
|
|
s = sources.all_sources()[0]
|
|
d = panel_api.source_detail(s.name)
|
|
assert d["ok"] and d["source"]["name"] == s.name
|
|
for k in ("category", "adapter", "kind", "url", "tags", "limit"):
|
|
assert k in d["source"], "нет поля source." + k
|
|
for k in ("state", "stored", "latest", "runs"):
|
|
assert k in d, "нет поля " + k
|
|
assert isinstance(d["latest"], list) and isinstance(d["runs"], list)
|
|
assert d["stored"] >= 0
|
|
assert panel_api.source_detail("нет-такого-источника") == {"error": "not found"}
|
|
|
|
|
|
# ---------- health ----------
|
|
def test_health_counts():
|
|
from omni import panel_api
|
|
|
|
d = panel_api.health()
|
|
assert d["ok"]
|
|
assert d["items"] > 0 and d["sources"] > 0
|
|
assert {"alive", "runs", "uptime", "db_mb"} <= set(d)
|
|
assert 0 <= d["alive"] <= d["sources"], "живых не может быть больше, чем всех"
|
|
assert d["db_mb"] >= 0 and d["uptime"] >= 0
|
|
|
|
|
|
# ---------- tg ----------
|
|
@pytest.mark.parametrize("enabled", [False, True])
|
|
def test_tg_status_shape_and_flag(monkeypatch, enabled):
|
|
from omni import config, panel_api, tgsess
|
|
|
|
monkeypatch.setattr(tgsess, "enabled", lambda: enabled)
|
|
monkeypatch.setattr(tgsess, "me", lambda: {"id": 123, "first_name": "Test"})
|
|
monkeypatch.setattr(config, "tg_cfg",
|
|
lambda: {"session": "test-account", "limit": 100})
|
|
d = panel_api.tg_status()
|
|
assert {"enabled", "me", "error", "limit", "session"} <= set(d)
|
|
assert d["enabled"] is enabled
|
|
assert d["me"] == ({"id": 123, "first_name": "Test"} if enabled else None)
|
|
assert d["error"] is None
|
|
|
|
|
|
# ---------- digest ----------
|
|
def test_digest_shape_and_day_window():
|
|
from pathlib import Path
|
|
|
|
from omni import panel_api
|
|
|
|
d = panel_api.digest(days=3, limit=10)
|
|
assert d["ok"] and d["days"] == 3 and d["n"] >= 0
|
|
md = d["md"]
|
|
assert md.startswith("# omni · дайджест")
|
|
for head in ("## Главное по score", "## Уязвимости", "## Сливы и утечки", "## Абузы"):
|
|
assert head in md, "в дайджесте нет раздела " + head
|
|
p = Path(d["path"])
|
|
assert p.exists() and p.read_text(encoding="utf-8").startswith("# omni · дайджест")
|
|
# clamp: 0 → 1 день, 999 → 30
|
|
assert panel_api.digest(days=0)["days"] == 1
|
|
assert panel_api.digest(days=999)["days"] == 30
|
|
|
|
|
|
# ---------- export ----------
|
|
def test_export_respects_filters_and_cap():
|
|
from omni import panel_api
|
|
|
|
d = panel_api.export_json(q="firecrawl", limit=5)
|
|
assert d["ok"] and d["count"] == len(d["items"]) <= 5
|
|
assert d["total"] >= d["count"], "total — это всё по фильтру, count — срез"
|
|
for it in d["items"]:
|
|
assert {"id", "title", "url", "source", "published", "types",
|
|
"price", "currency", "score"} <= set(it)
|
|
d2 = panel_api.export_json(typ="leak", limit=3)
|
|
assert d2["ok"] and all("leak" in (it["types"] or "") for it in d2["items"])
|
|
|
|
|
|
# ---------- serve: роуты зарегистрированы ----------
|
|
def test_serve_registers_panel_routes():
|
|
from omni import serve
|
|
|
|
src = inspect.getsource(serve)
|
|
for path in ("/api/item/", "/api/timeline", "/api/suggest", "/api/health",
|
|
"/api/tg/status", "/api/tg/search", "/api/tg/pull", "/api/source",
|
|
"/api/watch", "/api/digest", "/api/export.json"):
|
|
assert path in src, "роут не зарегистрирован: " + path
|
|
assert "browse2" in src, "browse2 не используется в serve"
|
|
|
|
|
|
# ---------- web: панель v2 содержит новые разделы ----------
|
|
def test_web_panel_v2_has_new_sections():
|
|
html = (ROOT / "web" / "index.html").read_text(encoding="utf-8")
|
|
for frag in ('id:"overview"', 'id:"feed"', 'id:"abuse"', 'id:"sources"', 'id:"runs"',
|
|
'id:"tg"', 'id:"keys"', 'id:"digest"', 'id:"watch"',
|
|
"/api/timeline", "/api/tg/status", "/api/suggest", "watchlist",
|
|
"/api/export.json"):
|
|
assert frag in html, "в панели нет " + frag
|
|
|
|
|
|
# ---------- store.browse2: since/src ----------
|
|
def test_browse2_filters_since_and_src():
|
|
from omni import store
|
|
|
|
rows, total = store.browse2(q="", limit=5, sort="seen", since="2026-01-01")
|
|
assert isinstance(rows, list) and total >= len(rows)
|
|
rows2, total2 = store.browse2(q="", limit=5, sort="seen", since="2099-01-01")
|
|
assert total2 == 0 and rows2 == [], "since не применился"
|
|
rows3, _ = store.browse2(q="", limit=200, sort="seen")
|
|
assert rows3, "нет строк для проверки src-фильтра"
|
|
name = rows3[0]["source"]
|
|
rows4, total4 = store.browse2(q="", limit=200, src=name)
|
|
assert total4 > 0 and all(x["source"] == name for x in rows4), "src не применился"
|
|
# src + since вместе
|
|
rows5, total5 = store.browse2(q="", limit=5, src=name, since="2099-01-01")
|
|
assert total5 == 0 and rows5 == []
|
|
|
|
|
|
def test_registry_dedupes_url_and_name(monkeypatch):
|
|
from omni import sources, source_state
|
|
|
|
first = sources.S("first", "security", "rss", "https://example.invalid/feed")
|
|
same_url = sources.S("alias", "security", "rss", first.url + "/")
|
|
same_name = sources.S("FIRST", "security", "rss", "https://example.invalid/mirror")
|
|
other = sources.S("other", "security", "rss", "https://example.invalid/other")
|
|
monkeypatch.setattr(sources, "SOURCES", [first, same_url, same_name, other])
|
|
monkeypatch.setattr(sources, "_gen_sources", lambda: [])
|
|
monkeypatch.setattr(sources, "_fresh_sources", lambda factory: [])
|
|
monkeypatch.setattr(sources, "_quality_sources", lambda factory: [])
|
|
monkeypatch.setattr(sources, "radar_sources", lambda: [])
|
|
monkeypatch.setattr(sources, "panel_sources", lambda: [])
|
|
monkeypatch.setattr(source_state, "custom", lambda: [])
|
|
assert sources.all_sources() == [first, other]
|
|
|
|
|
|
def test_fresh_sources_have_valid_metadata():
|
|
from omni import feeds, sources
|
|
from omni.sources_fresh import fresh
|
|
|
|
rows = fresh(sources.S)
|
|
assert rows
|
|
assert all(isinstance(s.tags, tuple) for s in rows)
|
|
assert all(s.adapter in feeds.ADAPTERS for s in rows)
|
|
assert len({s.name for s in rows}) == len(rows)
|
|
|
|
|
|
def test_quality_wave_is_integrated_without_collisions():
|
|
from omni import feeds, sources
|
|
from omni.sources_quality import quality_sources
|
|
|
|
wave = quality_sources(sources.S)
|
|
registry = sources.all_sources()
|
|
assert len(wave) >= 30
|
|
assert all(isinstance(s.tags, tuple) and s.adapter in feeds.ADAPTERS for s in wave)
|
|
assert {s.name for s in wave} <= {s.name for s in registry}
|
|
assert len(registry) == len({s.name.strip().lower() for s in registry})
|
|
assert len(registry) == len({(s.url.rstrip("/").lower(), s.adapter) for s in registry})
|
|
|
|
|
|
@pytest.mark.parametrize("tags", [(), ("news",), ("news", "cve")])
|
|
def test_fresh_generator_preserves_tag_tuples(tmp_path, tags):
|
|
import runpy
|
|
|
|
from omni import sources
|
|
|
|
emit = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))["emit"]
|
|
row = {"name": "test_feed", "category": "security", "kind": "rss",
|
|
"adapter": "rss", "url": "https://example.invalid/feed", "tags": tags}
|
|
path = tmp_path / "generated_sources.py"
|
|
assert emit([row], [], path, dedupe=False) == 1
|
|
generated = runpy.run_path(str(path))["fresh"](sources.S)
|
|
assert generated[0].tags == tags
|
|
assert isinstance(generated[0].tags, tuple)
|
|
|
|
|
|
def test_fresh_generator_dedupes_registry_and_batch(tmp_path, monkeypatch):
|
|
import runpy
|
|
from omni import sources
|
|
|
|
calls = []
|
|
existing = sources.S("Existing_Feed", "security", "rss", "https://example.invalid/known")
|
|
|
|
def registry(include_fresh=True):
|
|
calls.append(include_fresh)
|
|
return [existing]
|
|
|
|
monkeypatch.setattr(sources, "all_sources", registry)
|
|
probe = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))
|
|
rows = [
|
|
{"name": "existing-feed", "url": "https://example.invalid/other"},
|
|
{"name": "url_collision", "url": "http://www.example.invalid/known/"},
|
|
{"name": "novel", "url": "https://example.invalid/new"},
|
|
{"name": "novel_mirror", "url": "http://www.example.invalid/new/"},
|
|
{"name": "NOVEL", "url": "https://example.invalid/newer"},
|
|
]
|
|
path = tmp_path / "sources.py"
|
|
assert probe["emit"](rows, [], path) == 1
|
|
assert calls == [False]
|
|
assert [s.name for s in runpy.run_path(str(path))["fresh"](sources.S)] == ["novel"]
|
|
|
|
|
|
def test_fresh_generator_preserves_dynamic_window_and_python_literals(tmp_path):
|
|
import runpy
|
|
from urllib.parse import parse_qs, urlsplit
|
|
from omni import sources
|
|
|
|
probe = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))
|
|
row = {"name": "nvd_recent", "category": "security", "kind": "json", "adapter": "nvd",
|
|
"url": "https://example.invalid/old", "dynamic": "NVD_LAST_WEEK",
|
|
"extra": {"enabled": True, "optional": None}}
|
|
candidate, marker = probe["candidate_source"](row)
|
|
assert marker == candidate.extra["_dynamic"] == "NVD_LAST_WEEK"
|
|
assert "lastModStartDate=" in candidate.url
|
|
path = tmp_path / "sources.py"
|
|
assert probe["emit"]([dict(row, extra=candidate.extra)], [], path, dedupe=False) == 1
|
|
generated = runpy.run_path(str(path))["fresh"](sources.S)[0]
|
|
query = parse_qs(urlsplit(generated.url).query)
|
|
assert query["lastModStartDate"] and query["lastModEndDate"]
|
|
assert generated.extra == {"enabled": True, "optional": None}
|
|
|
|
|
|
def test_fresh_generator_rejects_unknown_dynamic_marker(tmp_path):
|
|
import runpy
|
|
|
|
probe = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))
|
|
row = {"name": "future", "url": "https://example.invalid/feed", "dynamic": "UNKNOWN"}
|
|
with pytest.raises(ValueError, match="dynamic"):
|
|
probe["candidate_source"](row)
|
|
with pytest.raises(ValueError, match="dynamic"):
|
|
probe["emit"]([row], [], tmp_path / "bad.py", dedupe=False)
|
|
|
|
|
|
def test_timeline_local_day_and_rolling_24_hours(monkeypatch):
|
|
import datetime
|
|
from omni import panel_api, store
|
|
|
|
now = datetime.datetime(2026, 10, 20, 1, 0).timestamp()
|
|
class FrozenDate(datetime.date):
|
|
@classmethod
|
|
def today(cls):
|
|
return cls(2026, 10, 20)
|
|
|
|
monkeypatch.setattr(panel_api._dt, "date", FrozenDate)
|
|
monkeypatch.setattr(panel_api.time, "time", lambda: now)
|
|
with store.db() as c:
|
|
c.execute("UPDATE items SET seen=? WHERE source='pytest-panel'", (now - 1800,))
|
|
c.execute("UPDATE items SET seen=? WHERE source='pytest-other'", (now - 23 * 3600,))
|
|
data = panel_api.timeline(days=2, mode="seen")
|
|
assert data["last24"] == 2, "rolling 24 hours must include yesterday evening"
|
|
assert data["today"] == 1, "local 00:30 import belongs to today's bar"
|
|
assert [p["n"] for p in data["points"]] == [1, 1]
|
|
|
|
|
|
def test_run_stats_count_partial_successes():
|
|
from omni import store
|
|
|
|
for status in ("ok", "ok 5/10", "пусто", "Exception: timeout"):
|
|
store.log_run("pytest", "test", "", 0, 0, status)
|
|
stats = store.stats()
|
|
assert stats["runs"] == 4
|
|
assert stats["runs_ok"] == 2
|
|
|
|
|
|
def test_timeline_last24_respects_publication_mode(monkeypatch):
|
|
import datetime
|
|
from omni import panel_api, store
|
|
|
|
now = time.time()
|
|
monkeypatch.setattr(panel_api.time, "time", lambda: now)
|
|
old = datetime.datetime.fromtimestamp(now - 48 * 3600, datetime.timezone.utc).isoformat()
|
|
future = datetime.datetime.fromtimestamp(now + 48 * 3600, datetime.timezone.utc).isoformat()
|
|
with store.db() as c:
|
|
c.execute("UPDATE items SET seen=?", (now - 60,))
|
|
c.execute("UPDATE items SET published=? WHERE source='pytest-panel'", (old,))
|
|
c.execute("UPDATE items SET published=? WHERE source='pytest-other'", (future,))
|
|
assert panel_api.timeline(mode="seen")["last24"] == 2
|
|
assert panel_api.timeline(mode="pub")["last24"] == 0
|
|
|
|
|
|
def test_http_exports_match_feed_filters_and_item_find(monkeypatch):
|
|
import csv
|
|
import datetime
|
|
import io
|
|
import json
|
|
import threading
|
|
import urllib.request
|
|
from http.server import ThreadingHTTPServer
|
|
from omni import serve, store
|
|
from omni.model import Item
|
|
|
|
today = datetime.date.today().isoformat()
|
|
old = (datetime.date.today() - datetime.timedelta(days=10)).isoformat()
|
|
rows = [
|
|
Item("pytest", "post", source="quality-http", title="promo code: QUAL2026 $20 30 days",
|
|
text="Official promotional offer for testing.", url="https://example.invalid/quality-new", published=today),
|
|
Item("pytest", "post", source="quality-http", title="Legacy license costs $99",
|
|
text="An old pricing page.", url="https://example.invalid/quality-old", published=old),
|
|
Item("pytest", "post", source="quality-http", title="Latest debugging guide",
|
|
text="A free guide to tracing request latency.", url="https://example.invalid/quality-free", published=today),
|
|
]
|
|
store.add(rows)
|
|
for status in (" OK ", "ok 2/5", "пусто", "Exception: timeout", None):
|
|
store.log_run("pytest", "quality-http", "", 0, 0, status)
|
|
monkeypatch.setattr(serve, "UI_TOKEN", "")
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), serve.Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
base = "http://127.0.0.1:%s" % server.server_address[1]
|
|
query = "?src=quality-http&days=1&priced=1&limit=1000"
|
|
try:
|
|
def get(path):
|
|
with urllib.request.urlopen(base + path, timeout=10) as res:
|
|
assert res.status == 200
|
|
return res.read().decode("utf-8-sig")
|
|
|
|
feed = json.loads(get("/api/items" + query))
|
|
export = json.loads(get("/api/export.json" + query))
|
|
csv_rows = list(csv.DictReader(io.StringIO(get("/api/export.csv" + query)), delimiter=";"))
|
|
assert feed["total"] == export["total"] == 1
|
|
assert [r["url"] for r in feed["items"]] == [rows[0].url]
|
|
assert [r["url"] for r in export["items"]] == [rows[0].url]
|
|
assert [r["url"] for r in csv_rows] == [rows[0].url]
|
|
assert "QUAL2026" in feed["items"][0]["find"]["codes"]
|
|
runs = json.loads(get("/api/runs?limit=2"))
|
|
assert runs["total"] == 5 and runs["shown"] == 2
|
|
assert runs["by_status"] == {"ok": 2, "empty": 1, "err": 1, "none": 1}
|
|
states = {r["state"] for r in json.loads(get("/api/runs?limit=10"))["items"]}
|
|
assert states == {"ok", "empty", "err", "none"}
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join(timeout=5)
|
|
|
|
def test_web_has_local_dashboard_and_free_panel_block():
|
|
"""The ready-made UI stays local and retains the free-panel discovery feature."""
|
|
html = (ROOT / "web" / "index.html").read_text(encoding="utf-8")
|
|
assert "loadPanels" in html
|
|
assert "/api/panels" in html
|
|
assert "/assets/vendor/tabler/tabler.min.css" in html
|
|
assert "/assets/vendor/tabler/tabler.min.js" in html
|