ui: локальный Tabler dashboard + 150 проверенных TG-источников и probe 595 каналов

tools/tg_pool_probe.py

@@ -0,0 +1,122 @@
+"""Probe public Telegram previews without writing posts to the store.
+
+Run from the repository root: python tools/tg_pool_probe.py
+Each of up to 12 coordinator threads runs one isolated adapter subprocess.
+The 20-second subprocess deadline includes imports, network fallbacks and parsing;
+Future.result(timeout=...) alone would leave hung network threads running.
+Results preserve pool order and username spelling; ok means items > 0.
+"""
+from __future__ import annotations
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+from types import SimpleNamespace
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKERS = 12
+TIMEOUT = 20
+LIMIT = 20
+
+
+def load_channels(path: Path) -> list[str]:
+    channels = json.loads(path.read_text(encoding="utf-8"))["channels"]
+    if not isinstance(channels, list):
+        raise ValueError("channels must be a list")
+    names = []
+    for entry in channels:
+        username = entry.get("username") if isinstance(entry, dict) else entry
+        if not isinstance(username, str) or not re.fullmatch(r"[A-Za-z0-9_]+", username):
+            raise ValueError(f"Invalid username in channels: {username!r}")
+        names.append(username)
+    return names
+
+
+def adapter_count(username: str) -> int:
+    sys.path.insert(0, str(ROOT))
+    from omni import feeds
+
+    src = SimpleNamespace(name=f"ab_tg_auto_{username}", category="abuse",
+                          kind="site", url=f"https://t.me/s/{username}",
+                          adapter="tg_web", tags=("abuse", "abuse", "tg"),
+                          limit=LIMIT, detail=False, extra={})
+    # Call the actual adapter and validation directly so unexpected exceptions
+    # reach the subprocess exit status instead of fetch() swallowing them.
+    return len(feeds._valid_items(feeds.ADAPTERS["tg_web"](src, LIMIT) or []))
+
+
+def probe(username: str, timeout: float = TIMEOUT) -> dict:
+    count = 0
+    try:
+        env = dict(os.environ, PYTHONPATH="", PYTHONIOENCODING="utf-8",
+                   PYTHONUTF8="1", PYTHONDONTWRITEBYTECODE="1")
+        result = subprocess.run([sys.executable, str(Path(__file__).resolve()),
+                                 "--probe", username],
+                                cwd=str(ROOT), env=env, capture_output=True,
+                                encoding="utf-8", timeout=timeout, check=False)
+        if result.returncode == 0:
+            value = json.loads(result.stdout)
+            if type(value) is int and 0 <= value <= LIMIT:
+                count = value
+        else:
+            print(f"probe error @{username}: exit={result.returncode} "
+                  f"{result.stderr[-300:].strip()}", file=sys.stderr, flush=True)
+    except subprocess.TimeoutExpired:
+        print(f"probe timeout @{username}: {timeout:g}s", file=sys.stderr, flush=True)
+    except (OSError, ValueError) as exc:
+        print(f"probe error @{username}: {type(exc).__name__}: {exc}",
+              file=sys.stderr, flush=True)
+    return {"username": username, "items": count, "ok": count > 0}
+
+
+def save_results(path: Path, rows: list[dict | None]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps([r for r in rows if r is not None],
+                               ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, default=ROOT / "data/radar_sources.json")
+    parser.add_argument("--output", type=Path, default=ROOT / "data/tg_pool_probe.json")
+    parser.add_argument("--workers", type=int, choices=range(1, WORKERS + 1), default=WORKERS)
+    parser.add_argument("--timeout", type=float, default=TIMEOUT)
+    parser.add_argument("--probe", help=argparse.SUPPRESS)
+    args = parser.parse_args()
+    if not 0 < args.timeout <= TIMEOUT:
+        parser.error("timeout must be greater than zero and at most 20 seconds")
+    if args.probe is not None:
+        if not re.fullmatch(r"[A-Za-z0-9_]+", args.probe):
+            parser.error("invalid username")
+        print(json.dumps(adapter_count(args.probe)))
+        return 0
+    if args.input.resolve() == args.output.resolve():
+        parser.error("output must not overwrite the input pool")
+    names = load_channels(args.input)
+    rows = [None] * len(names)
+    print(f"channels={len(names)} workers={args.workers} timeout={args.timeout:g}s limit={LIMIT}", flush=True)
+    with ThreadPoolExecutor(max_workers=args.workers) as executor:
+        pending = {executor.submit(probe, name, args.timeout): i
+                   for i, name in enumerate(names)}
+        for done, future in enumerate(as_completed(pending), 1):
+            rows[pending[future]] = future.result()
+            if done % 25 == 0 or done == len(names):
+                save_results(args.output, rows)
+                positive = sum(r is not None and r["ok"] for r in rows)
+                print(f"checked={done}/{len(names)} positive={positive}", flush=True)
+    save_results(args.output, rows)
+    print(json.dumps({"checked": len(rows),
+                      "positive": sum(r["ok"] for r in rows),
+                      "zero": sum(r["items"] == 0 for r in rows),
+                      "items_ge_10": sum(r["items"] >= 10 for r in rows),
+                      "output": str(args.output)}, ensure_ascii=False), flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/ui_smoke.py

@@ -1,96 +1,238 @@
-"""Смоук панели в реальном Chromium: скриншоты + ошибки консоли.
+"""Read-only Chromium acceptance checks for the black OMNI panel.
 
-python tools/ui_smoke.py [--url http://127.0.0.1:8078/] [--shots data/shots]
+python tools/ui_smoke.py [--url http://127.0.0.1:8078/] [--shots data/shots/black_shadcn]
 """
 import argparse
 import asyncio
 import json
 import pathlib
+import re
 import sys
+from urllib.parse import parse_qs, quote, urljoin, urlsplit
 
 from playwright.async_api import async_playwright
 
 
 async def run(url, out):
     out.mkdir(parents=True, exist_ok=True)
-    errors, report = [], {}
-    async with async_playwright() as p:
-        b = await p.chromium.launch()
-        pg = await b.new_page(viewport={"width": 1500, "height": 950})
-        pg.on("console", lambda m: errors.append([m.type, m.text]) if m.type in ("error",) else None)
-        pg.on("pageerror", lambda e: errors.append(["pageerror", str(e)]))
-        await pg.goto(url, wait_until="domcontentloaded", timeout=30000)
-        await pg.wait_for_selector(".kpi", timeout=20000)
-        await pg.wait_for_timeout(1200)
-        report["title"] = await pg.title()
-        report["nav"] = await pg.locator(".nav-i").count()
-        report["kpi"] = (await pg.locator(".kpi").all_inner_texts())[:4]
-        await pg.screenshot(path=str(out / "v2_overview.png"), full_page=False)
+    errors, assets, external = [], {}, set()
+    report = {}
+    origin = urlsplit(url).netloc
+    async with async_playwright() as playwright:
+        browser = await playwright.chromium.launch()
+        page = await browser.new_page(viewport={"width": 1440, "height": 960})
+        page.on("console", lambda m: errors.append(m.text) if m.type == "error" else None)
+        page.on("pageerror", lambda e: errors.append(str(e)))
 
-        await pg.click("[data-nav=feed]")
-        await pg.wait_for_selector("#list .item", timeout=20000)
-        report["feed_rows"] = await pg.locator(".item").count()
-        await pg.screenshot(path=str(out / "v2_feed.png"))
-        await pg.locator(".item").first.click()
-        await pg.wait_for_timeout(1200)
-        report["drawer"] = (await pg.locator("#d-title").inner_text())[:100]
-        await pg.screenshot(path=str(out / "v2_drawer.png"))
-        await pg.keyboard.press("Escape")
+        def response_seen(response):
+            parts = urlsplit(response.url)
+            if parts.path.startswith("/assets/"):
+                assets[parts.path] = response.status
+            if parts.scheme in ("http", "https") and parts.netloc != origin:
+                external.add(parts.netloc)
 
-        await pg.click("[data-nav=sources]")
-        await pg.wait_for_timeout(1500)
-        report["src_rows"] = await pg.locator("#src-tbl tbody tr").count()
-        await pg.screenshot(path=str(out / "v2_sources.png"))
+        page.on("response", response_seen)
 
-        await pg.click("[data-nav=abuse]")
-        await pg.wait_for_timeout(1500)
-        report["abuse_rows"] = await pg.locator(".item").count()
-        await pg.screenshot(path=str(out / "v2_abuse.png"))
+        async def settled():
+            await page.wait_for_load_state("networkidle", timeout=25000)
 
-        await pg.click("[data-nav=runs]")
-        await pg.wait_for_timeout(900)
-        report["run_rows"] = await pg.locator("#r-tbl tbody tr").count()
-        await pg.screenshot(path=str(out / "v2_runs.png"))
+        async def navigate(view):
+            nav = page.locator(f'[data-nav="{view}"]').first
+            if not await nav.is_visible():
+                summary = page.locator("#system-nav > summary")
+                if await summary.count() and await page.locator("#system-nav").get_attribute("open") is None:
+                    await summary.click()
+            await nav.click()
+            await page.wait_for_function("v => location.hash.startsWith('#/' + v)", arg=view)
+            await settled()
 
-        await pg.click("[data-nav=tg]")
-        await pg.wait_for_timeout(2000)
-        report["tg"] = (await pg.locator(".card").first.inner_text())[:160].replace("\n", " | ")
-        await pg.screenshot(path=str(out / "v2_tg.png"))
+        async def assert_no_overflow(label):
+            size = await page.evaluate("({scroll:document.documentElement.scrollWidth, width:innerWidth})")
+            assert size["scroll"] <= size["width"] + 1, f"Horizontal overflow at {label}: {size}"
 
-        await pg.click("[data-nav=keys]")
-        await pg.wait_for_timeout(800)
-        report["key_rows"] = await pg.locator("#k-tbl tbody tr").count()
-        await pg.screenshot(path=str(out / "v2_keys.png"))
+        async def assert_dark_theme(label):
+            colors = await page.evaluate("""() => {
+                const canvas = document.createElement('canvas');
+                canvas.width = canvas.height = 1;
+                const ctx = canvas.getContext('2d', {willReadFrequently: true});
+                const selectors = ['body', '.navbar', '.card', '.list-group-item',
+                    '.form-control', '.form-select', '.table', '.table thead',
+                    '.offcanvas.show', '.modal.show .modal-content', '.dropdown-menu.show'];
+                return selectors.flatMap(selector => [...document.querySelectorAll(selector)]
+                    .filter(el => el.getClientRects().length && getComputedStyle(el).visibility !== 'hidden')
+                    .slice(0, 12).map(el => {
+                        const bg = getComputedStyle(el).backgroundColor;
+                        ctx.clearRect(0, 0, 1, 1);
+                        ctx.fillStyle = bg;
+                        ctx.fillRect(0, 0, 1, 1);
+                        return {selector, bg, rgba: [...ctx.getImageData(0, 0, 1, 1).data]};
+                    }));
+            }""")
+            assert await page.evaluate("matchMedia('(prefers-color-scheme: dark)').matches || getComputedStyle(document.documentElement).colorScheme.includes('dark')")
+            assert any(c["selector"] == "body" and c["rgba"][3] == 255 for c in colors), colors
+            light = [c for c in colors if c["rgba"][3] > 127 and max(c["rgba"][:3]) > 64]
+            assert not light, f"Light surfaces in {label}: {light}"
+            report.setdefault("dark_routes", []).append(label)
 
-        await pg.click("[data-nav=digest]")
-        await pg.wait_for_timeout(500)
-        await pg.click("[data-act=digest]")
-        await pg.wait_for_timeout(6000)
-        report["digest_chars"] = len(await pg.locator("#d-out").inner_text())
-        await pg.screenshot(path=str(out / "v2_digest.png"))
+        try:
+            await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            await page.wait_for_selector("#list .item", timeout=25000)
+            await settled()
+            assert await page.locator('[data-nav="feed"]').count() == 1
+            report["default_feed"] = await page.locator("#list .item").count()
+            report["navigation_routes"] = await page.locator("[data-nav]").count()
+            assert report["navigation_routes"] == 9
+            await assert_no_overflow("desktop")
+            await assert_dark_theme("feed")
+            await page.screenshot(path=str(out / "black_feed.png"))
 
-        await pg.click("[data-nav=watch]")
-        await pg.wait_for_timeout(600)
-        await pg.screenshot(path=str(out / "v2_watch.png"))
+            # A real search must update both results and the total, not just the input.
+            missing = "zz_omni_smoke_no_match_7f91"
+            async with page.expect_response(lambda r: "/api/items?" in r.url and missing in r.url) as pending:
+                await page.locator("#q").fill(missing)
+                await page.locator("#q").press("Enter")
+            searched = await (await pending.value).json()
+            await settled()
+            assert searched["total"] == 0
+            assert await page.locator("#list .item").count() == 0
+            report["search_empty_state"] = True
+            await page.locator("#q").fill("")
+            await page.locator("#q").press("Enter")
+            await page.wait_for_selector("#list .item", timeout=20000)
+            await settled()
 
-        # мобильный вид
-        await pg.set_viewport_size({"width": 420, "height": 900})
-        await pg.click("[data-nav=feed]")
-        await pg.wait_for_timeout(1200)
-        await pg.screenshot(path=str(out / "v2_mobile.png"))
-        await b.close()
-    report["console"] = errors[:12]
-    print(json.dumps(report, ensure_ascii=False, indent=1))
-    return 0 if not errors else 1
+            # Reading panel: real detail response, keyboard focus containment and Escape.
+            row = page.locator("#list .item").first
+            opener = row.locator("[data-id]").first if await row.locator("[data-id]").count() else row
+            async with page.expect_response(re.compile(r"/api/item/\d+")) as detail_pending:
+                await opener.click()
+            detail = await (await detail_pending.value).json()
+            assert detail.get("ok") and detail.get("item", {}).get("title")
+            await page.wait_for_selector("#drawer.show", timeout=15000)
+            await settled()
+            assert (await page.locator("#d-title").inner_text()).strip()
+            await assert_dark_theme("reader")
+            await page.screenshot(path=str(out / "black_reader.png"))
+            focusables = page.locator('#drawer a[href], #drawer button:not([disabled]), #drawer input:not([disabled])')
+            if await focusables.count():
+                await focusables.last.focus()
+                await page.keyboard.press("Tab")
+                assert await page.evaluate("document.querySelector('#drawer').contains(document.activeElement)")
+                await focusables.first.focus()
+                await page.keyboard.press("Shift+Tab")
+                assert await page.evaluate("document.querySelector('#drawer').contains(document.activeElement)")
+            await page.keyboard.press("Escape")
+            await page.wait_for_function("!document.querySelector('#drawer').classList.contains('show')")
+            await page.locator("#drawer").wait_for(state="hidden")
+            assert await opener.evaluate("el => document.activeElement === el"), "Reader did not restore focus"
+            await page.locator('[data-act="help"]').first.click()
+            await page.wait_for_selector("#modal.show")
+            await settled()
+            await assert_dark_theme("help modal")
+            await page.keyboard.press("Escape")
+            await page.locator("#modal").wait_for(state="hidden")
+            report["reader_keyboard"] = True
+
+            # Source filtering is a real API query, not a dead text field.
+            await navigate("sources")
+            await page.wait_for_selector("#src-tbl")
+            async with page.expect_response(lambda r: "/api/sources?" in r.url and missing in r.url) as src_pending:
+                await page.locator("#s-q").fill(missing)
+            source_search = await (await src_pending.value).json()
+            await settled()
+            assert not source_search["items"]
+            report["source_search"] = True
+            await page.locator("#s-q").fill("")
+            await settled()
+            await assert_dark_theme("sources")
+            await page.screenshot(path=str(out / "black_sources.png"))
+
+            await navigate("abuse")
+            report["find_rows"] = await page.locator("#ab-list .item").count()
+            await assert_dark_theme("abuse")
+            await page.screenshot(path=str(out / "black_finds.png"))
+            async with page.expect_response(lambda r: urlsplit(r.url).path == "/api/panels") as panels_pending:
+                await page.locator('[data-ab-tab="panels"]').click()
+            panels = await (await panels_pending.value).json()
+            await settled()
+            assert isinstance(panels.get("panels"), list), panels.keys()
+            assert await page.locator("#pn-box").is_visible()
+            await assert_dark_theme("api panels")
+            report["api_panels"] = len(panels["panels"])
+            for view, selector in (("runs", "#r-tbl"), ("tg", ".card"), ("keys", "#k-tbl"), ("watch", ".card"), ("overview", ".card")):
+                await navigate(view)
+                await page.wait_for_selector(selector)
+                await assert_dark_theme(view)
+                await page.screenshot(path=str(out / f"black_{view}.png"))
+                report[view] = "rendered"
+            await navigate("digest")
+            async with page.expect_response(lambda r: "/api/digest?" in r.url) as digest_pending:
+                await page.locator('[data-act="digest"]').click()
+            digest = await (await digest_pending.value).json()
+            await settled()
+            assert digest.get("ok") and len(digest.get("md", "")) > 80
+            assert len((await page.locator("#d-out").inner_text()).strip()) > 80
+            report["digest"] = "rendered"
+            await assert_dark_theme("digest")
+            await page.screenshot(path=str(out / "black_digest.png"))
+
+            # Deep-link source filters and both exported formats must stay in sync.
+            sample = await (await page.request.get(url.rstrip("/") + "/api/items?limit=1")).json()
+            source = sample["items"][0]["source"]
+            deep_link = url.rstrip("/") + "/#/feed?sort=fresh&days=3650&src=" + quote(source, safe="")
+            await page.goto(deep_link, wait_until="domcontentloaded")
+            await page.wait_for_selector("#list .item")
+            await settled()
+            for extension in ("json", "csv"):
+                link = page.locator(f'a[href*="/api/export.{extension}"]').first
+                href = await link.get_attribute("href")
+                assert href is not None
+                query = parse_qs(urlsplit(href).query)
+                assert query.get("src") == [source] and query.get("days") == ["3650"]
+                exported = await page.request.get(urljoin(url, href))
+                assert exported.status == 200
+                if extension == "json":
+                    data = await exported.json()
+                    assert data["count"] and all(item["source"] == source for item in data["items"])
+            report["filtered_exports"] = True
+
+            # Mobile navigation has labels, a real toggle, and no page overflow.
+            await page.set_viewport_size({"width": 390, "height": 844})
+            await page.locator("#sidebar-toggle").click()
+            await page.locator("#sidebar-menu").wait_for(state="visible")
+            assert await page.locator('[data-nav="sources"]').is_visible()
+            await assert_no_overflow("mobile navigation 390")
+            await assert_dark_theme("mobile navigation")
+            await page.screenshot(path=str(out / "black_mobile_navigation.png"))
+            await page.locator('[data-nav="feed"]').click()
+            await settled()
+            await assert_no_overflow("mobile 390")
+            await page.screenshot(path=str(out / "black_mobile.png"))
+            await page.set_viewport_size({"width": 768, "height": 1024})
+            await assert_no_overflow("tablet 768")
+            report["responsive"] = [390, 768, 1440]
+
+            assert assets.get("/assets/vendor/tabler/tabler.min.css") == 200
+            assert assets.get("/assets/vendor/tabler/tabler.min.js") == 200
+            assert not external, f"External asset/API dependencies: {sorted(external)}"
+            assert not errors, errors
+        finally:
+            report["assets"] = assets
+            report["external_hosts"] = sorted(external)
+            report["console_errors"] = errors
+            (out / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+            print(json.dumps(report, ensure_ascii=False, indent=2))
+            await browser.close()
+    return 0
 
 
 def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--url", default="http://127.0.0.1:8078/")
-    ap.add_argument("--shots", default="data/shots")
-    a = ap.parse_args()
-    sys.exit(asyncio.run(run(a.url, pathlib.Path(a.shots))))
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", default="http://127.0.0.1:8078/")
+    parser.add_argument("--shots", default="data/shots/black_shadcn")
+    args = parser.parse_args()
+    sys.exit(asyncio.run(run(args.url, pathlib.Path(args.shots))))
 
 
 if __name__ == "__main__":
-    main()
+    main()