fix: 10 пунктов ревью — /run (кортеж store.add), валидация limit/offset в панели, close() соединений SQLite, троттлинг под локом, read-only бот без OMNI_BOT_OWNERS, eviction JOBS, push_new отдаёт свежие записи, ai.py не спит после последней попытки, дедуп реестра, подключён abuse-реестр (+191 источник); tests/test_regress.py 16 проверок + CI py3.10-3.12
Some checks failed
ci / test (3.10) (push) Has been cancelled
ci / test (3.11) (push) Has been cancelled
ci / test (3.12) (push) Has been cancelled

This commit is contained in:
ENI
2026-10-26 20:15:50 +03:00
parent 136aa88dd6
commit c9c0a11ef7
16 changed files with 806 additions and 54 deletions

View File

@@ -1,4 +1,5 @@
"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам."""
import contextlib
import json
import sqlite3
import time
@@ -50,6 +51,21 @@ def conn():
return c
@contextlib.contextmanager
def db():
"""Соединение с гарантированным close(): `with conn()` коммитит, но не закрывает,
и хендлы копятся (замер: 200 вызовов == +76 handles)."""
c = conn()
try:
yield c
c.commit()
except Exception:
c.rollback()
raise
finally:
c.close()
def _signed(h):
return h - (1 << 64) if h >= (1 << 63) else h
@@ -59,18 +75,25 @@ def _unsigned(v):
def _popcount(x):
return bin(x).count("1")
try:
return x.bit_count() # py3.10+, быстрее bin().count("1")
except AttributeError:
return bin(x).count("1")
def _near_dup(c, h, max_dist=3):
"""Кандидат ищем по полосе, решение принимаем по расстоянию Хэмминга."""
h = _unsigned(h)
seen = set()
for b in range(BANDS):
band = (h >> (16 * b)) & MASK
rows = c.execute(
"SELECT i.h FROM bands b JOIN items i ON i.id=b.item_id "
"WHERE b.band=? LIMIT 200", (b,)).fetchall()
for (other,) in rows:
"SELECT i.id, i.h FROM bands b JOIN items i ON i.id=b.item_id "
"WHERE b.band=?", (b,)).fetchall()
for iid, other in rows:
if iid in seen:
continue
seen.add(iid)
if _popcount(_unsigned(other) ^ h) <= max_dist:
return True
return False
@@ -80,7 +103,7 @@ def add(items):
"""Возвращает (новых, дублей)."""
new = dup = 0
ts = int(time.time())
with conn() as c:
with db() as c:
for it in items:
h = _unsigned(it.h)
if _near_dup(c, h):
@@ -111,7 +134,7 @@ def add(items):
def log_run(provider, source, query, got, kept, status, nbytes=0, mode=""):
with conn() as c:
with db() as c:
c.execute("INSERT INTO runs(ts,provider,source,query,got,kept,status,bytes,mode) "
"VALUES(?,?,?,?,?,?,?,?,?)",
(int(time.time()), provider, source, (query or "")[:200], got, kept, status, nbytes, mode))
@@ -122,12 +145,12 @@ def top(limit=20, provider=None):
+ ("WHERE provider=? " if provider else "") +
"ORDER BY score DESC, id DESC LIMIT ?")
args = (provider, limit) if provider else (limit,)
with conn() as c:
with db() as c:
return c.execute(q, args).fetchall()
def search(q, limit=20):
with conn() as c:
with db() as c:
rows = c.execute(
"SELECT i.provider,i.kind,i.source,i.title,i.score,i.url FROM items_fts f "
"JOIN items i ON i.id=f.rowid WHERE items_fts MATCH ? ORDER BY i.score DESC LIMIT ?",
@@ -153,7 +176,7 @@ def browse(category=None, q="", limit=60, offset=0):
args += [f"%{q}%", f"%{q}%"]
sql += "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"
args += [limit, offset]
with conn() as c:
with db() as c:
rows = c.execute(sql, args).fetchall()
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
return [dict(zip(keys, r)) for r in rows]
@@ -177,7 +200,7 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty
order = {"fresh": "ORDER BY published DESC, id DESC",
"price": "ORDER BY CAST(price AS REAL) DESC, score DESC",
"old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
with conn() as c:
with db() as c:
total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
"FROM items " + where + order + " LIMIT ? OFFSET ?",
@@ -189,7 +212,7 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty
def type_counts():
"""Сколько записей на каждый тип."""
out = {}
with conn() as c:
with db() as c:
for row in c.execute("SELECT COALESCE(types,'') FROM items"):
for t in (row[0] or "").split(","):
t = t.strip()
@@ -201,7 +224,7 @@ def type_counts():
def retype(limit=None):
"""Пересчитать типы у уже сохранённых записей."""
n = 0
with conn() as c:
with db() as c:
q = "SELECT id,title,text FROM items"
if limit:
q += " LIMIT %d" % int(limit)
@@ -213,11 +236,11 @@ def retype(limit=None):
def count_by_source():
with conn() as c:
with db() as c:
return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall())
def stats():
with conn() as c:
with db() as c:
total = c.execute("SELECT COUNT(*) FROM items").fetchone()[0]
by_prov = c.execute("SELECT provider, COUNT(*) FROM items GROUP BY provider ORDER BY 2 DESC").fetchall()
by_kind = c.execute("SELECT kind, COUNT(*) FROM items GROUP BY kind ORDER BY 2 DESC").fetchall()
@@ -230,7 +253,7 @@ def stats():
def export(path, fmt="jsonl", limit=1000):
path = Path(path)
with conn() as c:
with db() as c:
rows = c.execute("SELECT provider,kind,source,url,title,text,author,published,lang,tags,"
"price,currency,score,extra FROM items ORDER BY score DESC LIMIT ?",
(limit,)).fetchall()
@@ -255,7 +278,7 @@ def dedupe(apply: bool = False):
Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено).
"""
key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END"
with conn() as c:
with db() as c:
groups = c.execute(
"SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0]
extra = c.execute(