абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты

tests/conftest.py

@@ -0,0 +1,35 @@
+"""All tests use disposable storage, including module-import side effects."""
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+# This runs before test collection: abuse builds source metadata at import time.
+_SESSION_DATA = tempfile.TemporaryDirectory(prefix="omni-pytest-")
+os.environ["OMNI_DATA"] = _SESSION_DATA.name
+os.environ["OMNI_DB"] = str(Path(_SESSION_DATA.name) / "collection.sqlite3")
+
+from omni import config
+
+config.DATA = Path(_SESSION_DATA.name)
+config.DB = config.DATA / "collection.sqlite3"
+
+
+@pytest.fixture(autouse=True)
+def sandbox_data(tmp_path, monkeypatch):
+    """Give every test its own database, not only the new panel tests."""
+    monkeypatch.setattr(config, "DATA", tmp_path)
+    monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3")
+    monkeypatch.setenv("OMNI_DATA", str(tmp_path))
+    monkeypatch.setenv("OMNI_DB", str(config.DB))
+    from omni import abuse, panel_api
+    monkeypatch.setattr(abuse, "DB", config.DB)
+    monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json")
+
+
+def pytest_unconfigure(config):
+    _SESSION_DATA.cleanup()

tests/test_ingest_quality.py

@@ -0,0 +1,103 @@
+"""Offline regressions for feed metadata, date ranking and challenge filtering."""
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from omni import feeds, rank
+from omni.dates import parse_published
+from omni.model import Item
+from omni.sources import S
+
+
+@pytest.mark.parametrize("value", [
+    "2026-10-20T12:00:00Z", "2026-10-20T15:00:00+03:00",
+    "Tue, 20 Oct 2026 12:00:00 GMT", "2026-10-20T12:00:00",
+    "1792497600", "1792497600000", 1792497600,
+])
+def test_publication_formats_have_equal_freshness(value, monkeypatch):
+    stamp = datetime(2026, 10, 20, 12, tzinfo=timezone.utc).timestamp()
+    parsed = parse_published(value)
+    assert parsed is not None
+    assert parsed.timestamp() == stamp
+    monkeypatch.setattr(rank.time, "time", lambda: stamp + 86400)
+    assert rank._fresh(value) == pytest.approx(1 - 1 / 30)
+
+
+def test_date_only_and_invalid_dates(monkeypatch):
+    stamp = datetime(2026, 10, 20, tzinfo=timezone.utc).timestamp()
+    monkeypatch.setattr(rank.time, "time", lambda: stamp)
+    assert rank._fresh("2026-10-20") == 1
+    for value in (None, "", "not a date", "99999999999999999999999999", "NaN"):
+        assert parse_published(value) is None
+        assert rank._fresh(value) == 0.3
+    assert rank._fresh("2020-01-01") == 0
+    assert parse_published("2026") is None
+    assert parse_published("20261020") == datetime(2026, 10, 20, tzinfo=timezone.utc)
+
+
+def test_atom_alternate_links_author_and_date(monkeypatch):
+    src = S("atom_test", "models", "rss", "https://example.invalid/feed.atom")
+    xml = '''<feed xmlns="http://www.w3.org/2005/Atom"><entry>
+      <title>Runtime release 2.0</title>
+      <link rel="self" href="https://example.invalid/api/123"/>
+      <link rel="alternate" href="/releases/2.0"/>
+      <author><name>Release team</name></author>
+      <published>2026-10-20T15:00:00+03:00</published>
+      <summary>Faster inference and improved caching.</summary>
+    </entry></feed>'''
+    monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml))
+    rows = feeds.fetch(src)
+    assert len(rows) == 1
+    assert rows[0].url == "https://example.invalid/releases/2.0"
+    assert rows[0].author == "Release team"
+    assert rows[0].published == "2026-10-20T12:00:00+00:00"
+
+
+def test_rss_namespaced_creator_and_no_author(monkeypatch):
+    src = S("rss_test", "security", "rss", "https://example.invalid/feed")
+    xml = '''<rss xmlns:dc="http://purl.org/dc/elements/1.1/"><channel>
+      <item><title>Security advisory</title><link>https://example.invalid/advisory</link>
+      <dc:creator>Security team</dc:creator><pubDate>Tue, 20 Oct 2026 12:00:00 GMT</pubDate></item>
+      <item><title>New patch available</title><link>https://example.invalid/patch</link></item>
+    </channel></rss>'''
+    monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml))
+    rows = feeds.fetch(src)
+    assert len(rows) == 2
+    assert rows[0].author == "Security team"
+    assert rows[0].published == "2026-10-20T12:00:00+00:00"
+    assert rows[1].author == ""
+
+
+def test_fetch_drops_invalid_pages_but_keeps_short_real_posts(monkeypatch):
+    src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter")
+    def item(path, title, text=""):
+        return Item("test", "entry", url=path, title=title, text=text)
+    good = item("https://example.invalid/release", "v1.2")
+    real = item("https://example.invalid/article", "Access denied", "Real analysis of authorization failure. " * 20)
+    rows = [
+        item("", "A missing URL"),
+        item("javascript:alert(1)", "A dangerous URL"),
+        item("https://example.invalid/blank", ""),
+        item("https://example.invalid/challenge", "Just a moment...", "Checking your browser"),
+        item("https://example.invalid/denied", "Attention Required! | Cloudflare", "captcha"),
+        item("https://example.invalid/long-challenge", "Just a moment...",
+             "Checking your browser before accessing this site. " * 20),
+        {"title": "Malformed adapter result"}, None, "not an Item",
+        good, item(good.url, "Duplicate record"), real,
+    ]
+    monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: rows)
+    assert feeds.fetch(src) == [good, real]
+
+
+def test_fetch_bad_record_does_not_discard_valid_siblings(monkeypatch):
+    src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter")
+    good = Item("test", "entry", url="https://example.invalid/release", title="Real release")
+    bad = Item("test", "entry", url="https://example.invalid/bad", title="Malformed")
+    bad.text = None
+    monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: [bad, good])
+    assert feeds.fetch(src) == [good]

tests/test_panel_api.py

@@ -5,11 +5,38 @@ import sys
 import time
 from pathlib import Path
 
+import pytest
+
 ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT))
 os.environ.setdefault("PYTHONIOENCODING", "utf-8")
 
 
+@pytest.fixture(autouse=True)
+def isolated_panel_data(tmp_path, monkeypatch):
+    """Never touch the operator's database, watchlist, digest or TG session."""
+    from omni import abuse, config, panel_api, store, tgsess
+    from omni.model import Item
+
+    monkeypatch.setattr(config, "DATA", tmp_path)
+    monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3")
+    monkeypatch.setattr(abuse, "DB", config.DB)
+    monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json")
+    monkeypatch.setattr(config, "tg_cfg", lambda: {"session": "", "limit": 100})
+    monkeypatch.setattr(tgsess, "enabled", lambda: False)
+    today = time.strftime("%Y-%m-%d")
+    store.add([
+        Item(provider="pytest", kind="post", source="pytest-panel",
+             title="Firecrawl security dataset leak announcement",
+             text="Public incident report for regression coverage.",
+             url="https://example.invalid/seed", published=today),
+        Item(provider="pytest", kind="post", source="pytest-other",
+             title="Guide to observability and performance metrics",
+             text="A completely different article about tracing.",
+             url="https://example.invalid/other", published=today),
+    ])
+
+
 def _tmp_item(tag=None):
     from omni import store
     from omni.model import Item
@@ -124,14 +151,19 @@ def test_health_counts():
 
 
 # ---------- tg ----------
-def test_tg_status_shape_and_flag():
-    from omni import panel_api
+@pytest.mark.parametrize("enabled", [False, True])
+def test_tg_status_shape_and_flag(monkeypatch, enabled):
+    from omni import config, panel_api, tgsess
 
+    monkeypatch.setattr(tgsess, "enabled", lambda: enabled)
+    monkeypatch.setattr(tgsess, "me", lambda: {"id": 123, "first_name": "Test"})
+    monkeypatch.setattr(config, "tg_cfg",
+                        lambda: {"session": "test-account", "limit": 100})
     d = panel_api.tg_status()
-    assert {"enabled", "me", "error", "limit"} <= set(d)
-    assert isinstance(d["enabled"], bool)
-    if d["enabled"]:
-        assert d["session"]
+    assert {"enabled", "me", "error", "limit", "session"} <= set(d)
+    assert d["enabled"] is enabled
+    assert d["me"] == ({"id": 123, "first_name": "Test"} if enabled else None)
+    assert d["error"] is None
 
 
 # ---------- digest ----------
@@ -205,3 +237,170 @@ def test_browse2_filters_since_and_src():
     # src + since вместе
     rows5, total5 = store.browse2(q="", limit=5, src=name, since="2099-01-01")
     assert total5 == 0 and rows5 == []
+
+
+def test_registry_dedupes_url_and_name(monkeypatch):
+    from omni import sources, source_state
+
+    first = sources.S("first", "security", "rss", "https://example.invalid/feed")
+    same_url = sources.S("alias", "security", "rss", first.url + "/")
+    same_name = sources.S("FIRST", "security", "rss", "https://example.invalid/mirror")
+    other = sources.S("other", "security", "rss", "https://example.invalid/other")
+    monkeypatch.setattr(sources, "SOURCES", [first, same_url, same_name, other])
+    monkeypatch.setattr(sources, "_gen_sources", lambda: [])
+    monkeypatch.setattr(sources, "_fresh_sources", lambda factory: [])
+    monkeypatch.setattr(sources, "_quality_sources", lambda factory: [])
+    monkeypatch.setattr(sources, "radar_sources", lambda: [])
+    monkeypatch.setattr(sources, "panel_sources", lambda: [])
+    monkeypatch.setattr(source_state, "custom", lambda: [])
+    assert sources.all_sources() == [first, other]
+
+
+def test_fresh_sources_have_valid_metadata():
+    from omni import feeds, sources
+    from omni.sources_fresh import fresh
+
+    rows = fresh(sources.S)
+    assert rows
+    assert all(isinstance(s.tags, tuple) for s in rows)
+    assert all(s.adapter in feeds.ADAPTERS for s in rows)
+    assert len({s.name for s in rows}) == len(rows)
+
+
+def test_quality_wave_is_integrated_without_collisions():
+    from omni import feeds, sources
+    from omni.sources_quality import quality_sources
+
+    wave = quality_sources(sources.S)
+    registry = sources.all_sources()
+    assert len(wave) >= 30
+    assert all(isinstance(s.tags, tuple) and s.adapter in feeds.ADAPTERS for s in wave)
+    assert {s.name for s in wave} <= {s.name for s in registry}
+    assert len(registry) == len({s.name.strip().lower() for s in registry})
+    assert len(registry) == len({s.url.rstrip("/").lower() for s in registry})
+
+
+@pytest.mark.parametrize("tags", [(), ("news",), ("news", "cve")])
+def test_fresh_generator_preserves_tag_tuples(tmp_path, tags):
+    import runpy
+
+    from omni import sources
+
+    emit = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))["emit"]
+    row = {"name": "test_feed", "category": "security", "kind": "rss",
+           "adapter": "rss", "url": "https://example.invalid/feed", "tags": tags}
+    path = tmp_path / "generated_sources.py"
+    assert emit([row], [], path, dedupe=False) == 1
+    generated = runpy.run_path(str(path))["fresh"](sources.S)
+    assert generated[0].tags == tags
+    assert isinstance(generated[0].tags, tuple)
+
+
+def test_timeline_local_day_and_rolling_24_hours(monkeypatch):
+    import datetime
+    from omni import panel_api, store
+
+    now = datetime.datetime(2026, 10, 20, 1, 0).timestamp()
+    class FrozenDate(datetime.date):
+        @classmethod
+        def today(cls):
+            return cls(2026, 10, 20)
+
+    monkeypatch.setattr(panel_api._dt, "date", FrozenDate)
+    monkeypatch.setattr(panel_api.time, "time", lambda: now)
+    with store.db() as c:
+        c.execute("UPDATE items SET seen=? WHERE source='pytest-panel'", (now - 1800,))
+        c.execute("UPDATE items SET seen=? WHERE source='pytest-other'", (now - 23 * 3600,))
+    data = panel_api.timeline(days=2, mode="seen")
+    assert data["last24"] == 2, "rolling 24 hours must include yesterday evening"
+    assert data["today"] == 1, "local 00:30 import belongs to today's bar"
+    assert [p["n"] for p in data["points"]] == [1, 1]
+
+
+def test_run_stats_count_partial_successes():
+    from omni import store
+
+    for status in ("ok", "ok 5/10", "пусто", "Exception: timeout"):
+        store.log_run("pytest", "test", "", 0, 0, status)
+    stats = store.stats()
+    assert stats["runs"] == 4
+    assert stats["runs_ok"] == 2
+
+
+def test_timeline_last24_respects_publication_mode(monkeypatch):
+    import datetime
+    from omni import panel_api, store
+
+    now = time.time()
+    monkeypatch.setattr(panel_api.time, "time", lambda: now)
+    old = datetime.datetime.fromtimestamp(now - 48 * 3600, datetime.timezone.utc).isoformat()
+    future = datetime.datetime.fromtimestamp(now + 48 * 3600, datetime.timezone.utc).isoformat()
+    with store.db() as c:
+        c.execute("UPDATE items SET seen=?", (now - 60,))
+        c.execute("UPDATE items SET published=? WHERE source='pytest-panel'", (old,))
+        c.execute("UPDATE items SET published=? WHERE source='pytest-other'", (future,))
+    assert panel_api.timeline(mode="seen")["last24"] == 2
+    assert panel_api.timeline(mode="pub")["last24"] == 0
+
+
+def test_http_exports_match_feed_filters_and_item_find(monkeypatch):
+    import csv
+    import datetime
+    import io
+    import json
+    import threading
+    import urllib.request
+    from http.server import ThreadingHTTPServer
+    from omni import serve, store
+    from omni.model import Item
+
+    today = datetime.date.today().isoformat()
+    old = (datetime.date.today() - datetime.timedelta(days=10)).isoformat()
+    rows = [
+        Item("pytest", "post", source="quality-http", title="promo code: QUAL2026 $20 30 days",
+             text="Official promotional offer for testing.", url="https://example.invalid/quality-new", published=today),
+        Item("pytest", "post", source="quality-http", title="Legacy license costs $99",
+             text="An old pricing page.", url="https://example.invalid/quality-old", published=old),
+        Item("pytest", "post", source="quality-http", title="Latest debugging guide",
+             text="A free guide to tracing request latency.", url="https://example.invalid/quality-free", published=today),
+    ]
+    store.add(rows)
+    for status in (" OK ", "ok 2/5", "пусто", "Exception: timeout", None):
+        store.log_run("pytest", "quality-http", "", 0, 0, status)
+    monkeypatch.setattr(serve, "UI_TOKEN", "")
+    server = ThreadingHTTPServer(("127.0.0.1", 0), serve.Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    base = "http://127.0.0.1:%s" % server.server_address[1]
+    query = "?src=quality-http&days=1&priced=1&limit=1000"
+    try:
+        def get(path):
+            with urllib.request.urlopen(base + path, timeout=10) as res:
+                assert res.status == 200
+                return res.read().decode("utf-8-sig")
+
+        feed = json.loads(get("/api/items" + query))
+        export = json.loads(get("/api/export.json" + query))
+        csv_rows = list(csv.DictReader(io.StringIO(get("/api/export.csv" + query)), delimiter=";"))
+        assert feed["total"] == export["total"] == 1
+        assert [r["url"] for r in feed["items"]] == [rows[0].url]
+        assert [r["url"] for r in export["items"]] == [rows[0].url]
+        assert [r["url"] for r in csv_rows] == [rows[0].url]
+        assert "QUAL2026" in feed["items"][0]["find"]["codes"]
+        runs = json.loads(get("/api/runs?limit=2"))
+        assert runs["total"] == 5 and runs["shown"] == 2
+        assert runs["by_status"] == {"ok": 2, "empty": 1, "err": 1, "none": 1}
+        states = {r["state"] for r in json.loads(get("/api/runs?limit=10"))["items"]}
+        assert states == {"ok", "empty", "err", "none"}
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+def test_web_has_free_panel_block():
+    """Вкладка абузов обязана показывать free-панели new-api."""
+    html = open(os.path.join(ROOT, "web", "index.html"), encoding="utf-8").read()
+    assert "loadPanels" in html
+    assert "/api/panels" in html
+    assert "pn-grid" in html
+    assert ".pn-grid{" in html