fix: 10 пунктов ревью — /run (кортеж store.add), валидация limit/offset в панели, close() соединений SQLite, троттлинг под локом, read-only бот без OMNI_BOT_OWNERS, eviction JOBS, push_new отдаёт свежие записи, ai.py не спит после последней попытки, дедуп реестра, подключён abuse-реестр (+191 источник); tests/test_regress.py 16 проверок + CI py3.10-3.12

tools/_port_fixes.py

@@ -0,0 +1,199 @@
+"""Портирование проверенных фиксов из ревью (без снятия /abuse) + свои правки.
+
+Каждая замена ассертится: если шаблон не найден — падаем, а не молча пропускаем.
+"""
+import io
+import sys
+from pathlib import Path
+
+ROOT = Path(r"C:/Users/User/tmp/omni-parse")
+sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
+report = []
+
+
+def patch(rel, pairs, count_check=True):
+    p = ROOT / rel
+    s = p.read_text(encoding="utf-8")
+    orig = s
+    for old, new in pairs:
+        n = s.count(old)
+        if n == 0:
+            raise SystemExit(f"НЕ НАЙДЕНО в {rel}: {old[:70]!r}")
+        s = s.replace(old, new)
+        report.append(f"{rel}: заменено {n}x {old.strip().splitlines()[0][:60]}")
+    p.write_text(s, encoding="utf-8", newline="\n")
+    return orig != s
+
+
+DB_CTX = '''@contextlib.contextmanager
+def db():
+    """Соединение с гарантированным close(): `with conn()` коммитит, но не закрывает,
+    и хендлы копятся (замер: 200 вызовов == +76 handles)."""
+    c = conn()
+    try:
+        yield c
+        c.commit()
+    except Exception:
+        c.rollback()
+        raise
+    finally:
+        c.close()
+
+
+'''
+
+# ---------- store.py ----------
+p = ROOT / "omni/store.py"
+s = p.read_text(encoding="utf-8")
+s = s.replace('"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам."""\n',
+              '"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам."""\nimport contextlib\n', 1)
+anchor = "def _signed(h):"
+assert anchor in s
+s = s.replace(anchor, DB_CTX + anchor, 1)
+n = s.count("with conn() as c:")
+s = s.replace("with conn() as c:", "with db() as c:")
+report.append(f"store.py: db() добавлен, {n} вызовов переведены на close()")
+
+s = s.replace('''def _popcount(x):
+    return bin(x).count("1")''',
+'''def _popcount(x):
+    try:
+        return x.bit_count()      # py3.10+, быстрее bin().count("1")
+    except AttributeError:
+        return bin(x).count("1")''')
+
+old_near = '''    h = _unsigned(h)
+    for b in range(BANDS):
+        band = (h >> (16 * b)) & MASK
+        rows = c.execute(
+            "SELECT i.h FROM bands b JOIN items i ON i.id=b.item_id "
+            "WHERE b.band=? LIMIT 200", (b,)).fetchall()
+        for (other,) in rows:
+            if _popcount(_unsigned(other) ^ h) <= max_dist:
+                return True
+    return False'''
+new_near = '''    h = _unsigned(h)
+    seen = set()
+    for b in range(BANDS):
+        band = (h >> (16 * b)) & MASK
+        rows = c.execute(
+            "SELECT i.id, i.h FROM bands b JOIN items i ON i.id=b.item_id "
+            "WHERE b.band=?", (b,)).fetchall()
+        for iid, other in rows:
+            if iid in seen:
+                continue
+            seen.add(iid)
+            if _popcount(_unsigned(other) ^ h) <= max_dist:
+                return True
+    return False'''
+assert old_near in s, "near_dup шаблон не найден"
+s = s.replace(old_near, new_near)
+report.append("store.py: _near_dup без LIMIT 200 (полный проход по полосе)")
+p.write_text(s, encoding="utf-8", newline="\n")
+
+# ---------- source_state.py ----------
+patch("omni/source_state.py", [
+    ('  source_custom (name, category, kind, url, adapter, tags, limit, detail, added)\n"""\n',
+     '  source_custom (name, category, kind, url, adapter, tags, limit, detail, added)\n"""\nimport contextlib\n'),
+    ("def flags():", DB_CTX + "def flags():"),
+    ("with conn() as c:", "with db() as c:"),
+])
+
+# ---------- net.py: троттлинг ----------
+patch("omni/net.py", [
+    ('''def _throttle(host):
+    delay = float(_cfg.get("per_host_delay", 0.6))
+    with _lock:
+        prev = _last.get(host, 0)
+        wait = delay - (time.time() - prev)
+    if wait > 0:
+        time.sleep(wait)
+    with _lock:
+        _last[host] = time.time()''',
+     '''def _throttle(host):
+    """Слот резервируется под локом ДО сна: иначе N потоков проходят одновременно.
+    Замер до фикса: 24 вызова на хост при 0.3с == 0.9с вместо 6.9с."""
+    delay = float(_cfg.get("per_host_delay", 0.6))
+    with _lock:
+        now = time.time()
+        prev = _last.get(host, 0)
+        wait = max(0.0, delay - (now - prev))
+        _last[host] = now + wait
+    if wait > 0:
+        time.sleep(wait)'''),
+])
+
+# ---------- ai.py: не спать в пустоту ----------
+patch("omni/ai.py", [
+    ('''            except Exception:
+                time.sleep(1.5 + attempt)''',
+     '''            except Exception:
+                if attempt == 0:      # после последней попытки спим в пустоту
+                    time.sleep(1.5)'''),
+])
+
+# ---------- bot.py ----------
+patch("omni/bot.py", [
+    ("            new += store.add(items)",
+     "            n_new, _dup = store.add(items)\n            new += n_new"),
+    ("        chunk = fresh[-per_chat:]",
+     "        chunk = fresh[:per_chat]        # раньше уходили САМЫЕ СТАРЫЕ из окна"),
+    ('''CMDS = {"/start": cmd_start,''',
+     '''READONLY_CMDS = {"/start", "/help", "/new", "/abuse", "/find", "/src", "/stats"}
+
+CMDS = {"/start": cmd_start,'''),
+    ('''    if OWNERS and uid not in OWNERS:
+        return
+    if not OWNERS:
+        OWNERS.add(uid)  # первый написавший становится владельцем до рестарта
+    parts = text.split(None, 1)
+    cmd = parts[0].lower().split("@")[0]
+    arg = parts[1] if len(parts) > 1 else ""''',
+     '''    parts = text.split(None, 1)
+    cmd = parts[0].lower().split("@")[0]
+    arg = parts[1] if len(parts) > 1 else ""
+    if OWNERS and uid not in OWNERS:
+        return
+    if not OWNERS and cmd not in READONLY_CMDS:
+        # раньше первый написавший становился владельцем: любой, кто нашёл бота
+        return _send(chat_id, "read-only: задай OMNI_BOT_OWNERS и перезапусти бота")'''),
+    ("Доступ:  OMNI_BOT_OWNERS=7448683285,123   (если пусто — только первый написавший)",
+     "Доступ:  OMNI_BOT_OWNERS=<id>,<id>   (пусто = read-only: /new /abuse /find /src /stats)"),
+])
+
+# ---------- sources.py: дедуп по url ----------
+p = ROOT / "omni/sources.py"
+s = p.read_text(encoding="utf-8")
+if "def all_sources():" in s:
+    s2 = s.replace("def all_sources():", '''def all_sources():
+    """Реестр без дублей по url: одинаковые площадки приходили из разных наборов."""''', 1)
+    # добавляем дедуп в возврат: ищем return внутри all_sources
+    idx = s2.index("def all_sources():")
+    tail = s2[idx:]
+    if "return _dedup(" in tail:
+        report.append("sources.py: дедуп уже есть")
+    else:
+        # оборачиваем последний return списка
+        import re
+        m = list(re.finditer(r"\n    return (\[\]|\w+|\w+ \+|.*?)\n(?=\n|$)", tail))
+        report.append(f"sources.py: return-кандидатов {len(m)}")
+        # используем _dedup через хелпер, определённый перед all_sources
+        s2 = s2.replace("def all_sources():", '''def _dedup(items):
+    seen, out = set(), []
+    for it in items:
+        key = (it.url or "").rstrip("/")
+        if key and key in seen:
+            continue
+        seen.add(key)
+        out.append(it)
+    return out
+
+
+def all_sources():''', 1)
+        s2 = re.sub(r"(\n    return )(?!_dedup)(\[[^\n]*\]|\w+(?: \+ \w+)*)\n", r"\1_dedup(\2)\n", s2, count=1)
+        report.append("sources.py: _dedup + обёртка return")
+    s = s2
+p.write_text(s, encoding="utf-8", newline="\n")
+
+print("\n".join(report))

tools/_port_serve.py

@@ -0,0 +1,85 @@
+"""serve.py: валидация int-параметров, eviction JOBS, токен на POST, лимит тела."""
+from pathlib import Path
+
+p = Path(r"C:/Users/User/tmp/omni-parse/omni/serve.py")
+s = p.read_text(encoding="utf-8")
+rep = []
+
+if "\nimport os\n" not in s:
+    s = s.replace("import json\n", "import json\nimport os\n", 1)
+    rep.append("+ import os")
+
+helper = '''
+
+UI_TOKEN = os.environ.get("OMNI_UI_TOKEN", "")
+
+
+def _ui_auth(req):
+    """Панель наружу: без токена мутации открыты любому, кто дошёл до порта."""
+    if not UI_TOKEN:
+        return True
+    return req.headers.get("X-Omni-Token") == UI_TOKEN
+
+
+def _intv(q, key, default, lo, hi):
+    """int из query без падений: ?limit=abc раньше давал трейс и обрыв соединения,
+    ?limit=-1 отдавал всю базу (SQLite LIMIT -1 = без лимита)."""
+    try:
+        v = int((q.get(key) or [str(default)])[0])
+    except (TypeError, ValueError):
+        return default
+    return max(lo, min(hi, v))
+'''
+
+assert "def _job(jid, **kw):" in s
+s = s.replace("def _job(jid, **kw):", helper.strip("\n") + "\n\n\ndef _job(jid, **kw):", 1)
+
+s = s.replace('''    with _LOCK:
+        cur = JOBS.get(jid) or {}
+        cur.update(kw)
+        JOBS[jid] = cur''',
+'''    with _LOCK:
+        cur = JOBS.get(jid) or {}
+        cur.update(kw)
+        cur["_ts"] = time.time()
+        JOBS[jid] = cur
+        if len(JOBS) > 400:          # раньше росло без границ (500 запусков -> 500 записей)
+            dead = sorted(JOBS.items(), key=lambda kv: kv[1].get("_ts", 0))[:len(JOBS) - 300]
+            for k, _v in dead:
+                JOBS.pop(k, None)''')
+rep.append("JOBS eviction")
+
+pairs = [
+    ('limit = max(1, min(500, int((q.get("limit") or ["80"])[0])))', 'limit = _intv(q, "limit", 80, 1, 500)'),
+    ('lim = max(1, min(200, int((q.get("limit") or ["40"])[0])))', 'lim = _intv(q, "limit", 40, 1, 200)'),
+    ('lim = max(1, min(40, int((q.get("limit") or ["14"])[0])))', 'lim = _intv(q, "limit", 14, 1, 40)'),
+    ('limit = min(int((q.get("limit") or ["40"])[0]), 200)', 'limit = _intv(q, "limit", 40, 1, 200)'),
+    ('offset = int((q.get("offset") or ["0"])[0])', 'offset = _intv(q, "offset", 0, 0, 1_000_000)'),
+    ('hosts = max(5, min(120, int((q.get("hosts") or ["40"])[0])))', 'hosts = _intv(q, "hosts", 40, 5, 120)'),
+    ('add = max(1, min(60, int((q.get("add") or ["25"])[0])))', 'add = _intv(q, "add", 25, 1, 60)'),
+]
+for old, new in pairs:
+    if old in s:
+        s = s.replace(old, new)
+        rep.append("int -> " + new.split(" = ")[0] + " " + new.split("_intv")[1][:26])
+
+old_post = '''    def do_POST(self):
+        u = urllib.parse.urlsplit(self.path)
+        n = int(self.headers.get("Content-Length") or 0)'''
+new_post = '''    def do_POST(self):
+        u = urllib.parse.urlsplit(self.path)
+        if not _ui_auth(self):
+            return self._json({"error": "unauthorized"}, 401)
+        n = min(int(self.headers.get("Content-Length") or 0), 2_000_000)'''
+assert old_post in s, "do_POST шаблон не найден"
+s = s.replace(old_post, new_post)
+rep.append("POST: токен + лимит тела")
+
+s = s.replace('''    print(f"omni ui: http://127.0.0.1:{port}  (источников {len(sources.all_sources())})")''',
+'''    print(f"omni ui: http://127.0.0.1:{port}  (источников {len(sources.all_sources())})")
+    if not UI_TOKEN:
+        print("внимание: OMNI_UI_TOKEN не задан — POST /api/* без авторизации; наружу только за nginx с basic-auth")''')
+
+p.write_text(s, encoding="utf-8", newline="\n")
+print("\n".join(rep))
+print("остались сырые int((q.get:", s.count("int((q.get"))

tools/_verify2.py

@@ -0,0 +1,34 @@
+"""Точные замеры: живые sqlite-соединения, логика владельца бота, реальность rank.py."""
+import gc
+import os
+import re
+import sqlite3
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+os.environ.setdefault("PYTHONIOENCODING", "utf-8")
+
+
+def count_conns():
+    gc.collect()
+    return sum(1 for o in gc.get_objects() if isinstance(o, sqlite3.Connection))
+
+
+from omni import store  # noqa: E402
+
+base = count_conns()
+for _ in range(50):
+    store.top(limit=1)
+print("sqlite conn живых: до", base, "после 50 вызовов", count_conns())
+
+src = (ROOT / "omni" / "bot.py").read_text(encoding="utf-8")
+m = re.search(r"if not OWNERS:.*?(?=\n\s*fn = |\n\s*cmd = )", src, re.S)
+print("owner-блок:\n", (m.group(0).strip()[:400] if m else "не найден"))
+print("OMNI_BOT_OWNERS в коде:", "OMNI_BOT_OWNERS" in src)
+
+cli = (ROOT / "omni" / "cli.py").read_text(encoding="utf-8")
+print("rank в cli.py:", [ln.strip()[:80] for ln in cli.splitlines() if "rank" in ln][:4])
+rank = (ROOT / "omni" / "rank.py").read_text(encoding="utf-8")
+print("rank.py функций:", re.findall(r"def (\w+)", rank))

tools/_verify_claims.py

@@ -0,0 +1,142 @@
+"""Проверка пунктов прожарки на текущем коде (до патчей). Только чтение, ничего не пишет в базу."""
+import gc
+import json
+import os
+import re
+import sys
+import threading
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+os.environ.setdefault("PYTHONIOENCODING", "utf-8")
+
+import psutil  # noqa: E402
+
+ROOT = Path(__file__).resolve().parent.parent
+OUT = {}
+
+
+def p(k, v):
+    OUT[k] = v
+    print(f"{k}: {v}")
+
+
+# 2. валидация limit: эмулируем то, что делает serve для /api/items
+def check_limit():
+    def parse(raw):
+        try:
+            return min(int(raw), 200)
+        except ValueError as e:
+            return f"ValueError: {e}"
+
+    p("limit[-1]", parse("-1"))
+    p("limit[abc]", parse("abc"))
+    p("limit[999999]", parse("999999"))
+
+
+# 3. утечка соединений
+def check_conn_leak():
+    from omni import store
+    gc.collect()
+    before = len(psutil.Process().open_files())
+    for _ in range(50):
+        store.top(limit=1)
+    gc.collect()
+    after = len(psutil.Process().open_files())
+    p("fd_before", before)
+    p("fd_after_50_calls", after)
+    p("fd_leak", after - before)
+
+
+# 4. троттлинг
+def check_throttle():
+    from omni import net
+    net._cfg["per_host_delay"] = 0.3
+    net._last.clear()
+    t0 = time.time()
+    threads = [threading.Thread(target=lambda: [net._throttle("example.org") for _ in range(3)]) for _ in range(8)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+    el = time.time() - t0
+    expected = 0.3 * (8 * 3 - 1)
+    p("throttle_elapsed", round(el, 2))
+    p("throttle_expected_min", round(expected, 2))
+    p("throttle_broken", el < expected * 0.5)
+
+
+# 6. мёртвые файлы
+def check_dead():
+    names = ["abuse_sources", "rank", "sources_abuse"]
+    for n in names:
+        hits = []
+        for f in (ROOT / "omni").glob("*.py"):
+            if f.stem == n:
+                continue
+            txt = f.read_text(encoding="utf-8", errors="replace")
+            if re.search(rf"(import|from)\s+.*\b{n}\b", txt):
+                hits.append(f.name)
+        p(f"imports_{n}", hits)
+
+
+# 7. дубли источников
+def check_dupe_sources():
+    from omni import sources
+    urls = [s.url for s in sources.all_sources()]
+    p("sources_total", len(urls))
+    p("urls_unique", len(set(urls)))
+    p("dupe_groups", len(urls) - len(set(urls)))
+
+
+# 8. except Exception
+def check_except():
+    total = 0
+    for f in (ROOT / "omni").rglob("*.py"):
+        total += len(re.findall(r"except\s+Exception", f.read_text(encoding="utf-8", errors="replace")))
+    p("except_exception_total", total)
+
+
+# 9. JOBS без eviction
+def check_jobs():
+    from omni import serve
+    src = (ROOT / "omni" / "serve.py").read_text(encoding="utf-8")
+    p("jobs_has_eviction", "JOBS.pop" in src or "del JOBS" in src)
+    for i in range(500):
+        serve._job(f"j{i}", status="done")
+    p("jobs_len_after_500", len(serve.JOBS))
+
+
+# 10. sleep в ai.py
+def check_ai_sleep():
+    src = (ROOT / "omni" / "ai.py").read_text(encoding="utf-8")
+    m = re.search(r"except Exception:\s*\n\s*time\.sleep\(([^)]*)\)", src)
+    p("ai_sleep_line", m.group(0).replace("\n", " ") if m else None)
+
+
+# 1. /run в боте
+def check_bot_run():
+    src = (ROOT / "omni" / "bot.py").read_text(encoding="utf-8")
+    p("bot_new_plus_equals_add", "new += store.add(items)" in src)
+    p("bot_cmd_abuse_present", "def cmd_abuse" in src)
+    p("bot_cmds_abuse_present", "cmd_abuse" in src)
+
+
+if __name__ == "__main__":
+    import contextlib
+    import io
+
+    check_limit()
+    with contextlib.redirect_stdout(io.StringIO()):
+        check_bot_run()
+    check_bot_run()
+    check_throttle()
+    check_dead()
+    check_dupe_sources()
+    check_except()
+    with contextlib.redirect_stderr(io.StringIO()):
+        check_jobs()
+    check_ai_sleep()
+    check_conn_leak()
+    print("\nJSON:", json.dumps(OUT, ensure_ascii=False, default=str))

tools/selfcheck.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 import json
 import os
 import py_compile
+import subprocess
 import sqlite3
 import sys
 import time
@@ -35,6 +36,17 @@ def one(con, sql, args=(), default=0):
     return r[0][0] if r and r[0][0] is not None else default
 
 
+def run_tests():
+    """Тесты — часть самоаудита: без них «работает» держится на памяти."""
+    try:
+        r = subprocess.run([sys.executable, "-m", "pytest", "tests", "-q"],
+                           cwd=str(ROOT), capture_output=True, text=True, timeout=300)
+    except Exception as e:
+        return {"ok": False, "note": type(e).__name__}
+    tail = (r.stdout or "").strip().splitlines()
+    return {"ok": r.returncode == 0, "note": tail[-1][:120] if tail else ""}
+
+
 def compile_all():
     bad = []
     for p in sorted((ROOT / "omni").glob("*.py")):
@@ -48,6 +60,7 @@ def compile_all():
 def collect():
     st: dict = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds")}
     st["compile_errors"] = compile_all()
+    st["tests"] = run_tests()
 
     from omni import sources, taxonomy  # noqa: E402
 
@@ -93,6 +106,8 @@ def collect():
     problems = []
     if st["compile_errors"]:
         problems.append(f"compile: {len(st['compile_errors'])} файлов")
+    if not (st.get("tests") or {}).get("ok"):
+        problems.append(f"тесты: {(st.get('tests') or {}).get('note', 'fail')}")
     if not st["has_types_col"]:
         problems.append("нет колонки items.types")
     if st["items"] and st["typed"] < st["items"] * 0.95:
@@ -131,6 +146,7 @@ _обновлено {st['ts']}_
 |---|---|
 {rows}
 
+тесты: {(st.get('tests') or {}).get('note') or '-'}
 самоаудит: {'OK' if not problems else '; '.join(problems)}
 <!-- STATS:END -->"""
     return block
@@ -165,7 +181,8 @@ def main():
         print(json.dumps(st, ensure_ascii=False, indent=1))
     else:
         print(f"items={st.get('items')} typed={st.get('typed')} sources={st.get('sources')} "
-              f"priced={st.get('priced')} runs={st.get('runs')} problems={st.get('problems') or 'OK'}")
+              f"priced={st.get('priced')} runs={st.get('runs')} "
+              f"tests={(st.get('tests') or {}).get('ok')} problems={st.get('problems') or 'OK'}")
     return 0