199 lines
7.9 KiB
Python
199 lines
7.9 KiB
Python
"""Портирование проверенных фиксов из ревью (без снятия /abuse) + свои правки.
|
||
|
||
Каждая замена ассертится: если шаблон не найден — падаем, а не молча пропускаем.
|
||
"""
|
||
import io
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(r"C:/Users/User/tmp/omni-parse")
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
|
||
report = []
|
||
|
||
|
||
def patch(rel, pairs, count_check=True):
|
||
p = ROOT / rel
|
||
s = p.read_text(encoding="utf-8")
|
||
orig = s
|
||
for old, new in pairs:
|
||
n = s.count(old)
|
||
if n == 0:
|
||
raise SystemExit(f"НЕ НАЙДЕНО в {rel}: {old[:70]!r}")
|
||
s = s.replace(old, new)
|
||
report.append(f"{rel}: заменено {n}x {old.strip().splitlines()[0][:60]}")
|
||
p.write_text(s, encoding="utf-8", newline="\n")
|
||
return orig != s
|
||
|
||
|
||
DB_CTX = '''@contextlib.contextmanager
|
||
def db():
|
||
"""Соединение с гарантированным close(): `with conn()` коммитит, но не закрывает,
|
||
и хендлы копятся (замер: 200 вызовов == +76 handles)."""
|
||
c = conn()
|
||
try:
|
||
yield c
|
||
c.commit()
|
||
except Exception:
|
||
c.rollback()
|
||
raise
|
||
finally:
|
||
c.close()
|
||
|
||
|
||
'''
|
||
|
||
# ---------- store.py ----------
|
||
p = ROOT / "omni/store.py"
|
||
s = p.read_text(encoding="utf-8")
|
||
s = s.replace('"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам."""\n',
|
||
'"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам."""\nimport contextlib\n', 1)
|
||
anchor = "def _signed(h):"
|
||
assert anchor in s
|
||
s = s.replace(anchor, DB_CTX + anchor, 1)
|
||
n = s.count("with conn() as c:")
|
||
s = s.replace("with conn() as c:", "with db() as c:")
|
||
report.append(f"store.py: db() добавлен, {n} вызовов переведены на close()")
|
||
|
||
s = s.replace('''def _popcount(x):
|
||
return bin(x).count("1")''',
|
||
'''def _popcount(x):
|
||
try:
|
||
return x.bit_count() # py3.10+, быстрее bin().count("1")
|
||
except AttributeError:
|
||
return bin(x).count("1")''')
|
||
|
||
old_near = ''' h = _unsigned(h)
|
||
for b in range(BANDS):
|
||
band = (h >> (16 * b)) & MASK
|
||
rows = c.execute(
|
||
"SELECT i.h FROM bands b JOIN items i ON i.id=b.item_id "
|
||
"WHERE b.band=? LIMIT 200", (b,)).fetchall()
|
||
for (other,) in rows:
|
||
if _popcount(_unsigned(other) ^ h) <= max_dist:
|
||
return True
|
||
return False'''
|
||
new_near = ''' h = _unsigned(h)
|
||
seen = set()
|
||
for b in range(BANDS):
|
||
band = (h >> (16 * b)) & MASK
|
||
rows = c.execute(
|
||
"SELECT i.id, i.h FROM bands b JOIN items i ON i.id=b.item_id "
|
||
"WHERE b.band=?", (b,)).fetchall()
|
||
for iid, other in rows:
|
||
if iid in seen:
|
||
continue
|
||
seen.add(iid)
|
||
if _popcount(_unsigned(other) ^ h) <= max_dist:
|
||
return True
|
||
return False'''
|
||
assert old_near in s, "near_dup шаблон не найден"
|
||
s = s.replace(old_near, new_near)
|
||
report.append("store.py: _near_dup без LIMIT 200 (полный проход по полосе)")
|
||
p.write_text(s, encoding="utf-8", newline="\n")
|
||
|
||
# ---------- source_state.py ----------
|
||
patch("omni/source_state.py", [
|
||
(' source_custom (name, category, kind, url, adapter, tags, limit, detail, added)\n"""\n',
|
||
' source_custom (name, category, kind, url, adapter, tags, limit, detail, added)\n"""\nimport contextlib\n'),
|
||
("def flags():", DB_CTX + "def flags():"),
|
||
("with conn() as c:", "with db() as c:"),
|
||
])
|
||
|
||
# ---------- net.py: троттлинг ----------
|
||
patch("omni/net.py", [
|
||
('''def _throttle(host):
|
||
delay = float(_cfg.get("per_host_delay", 0.6))
|
||
with _lock:
|
||
prev = _last.get(host, 0)
|
||
wait = delay - (time.time() - prev)
|
||
if wait > 0:
|
||
time.sleep(wait)
|
||
with _lock:
|
||
_last[host] = time.time()''',
|
||
'''def _throttle(host):
|
||
"""Слот резервируется под локом ДО сна: иначе N потоков проходят одновременно.
|
||
Замер до фикса: 24 вызова на хост при 0.3с == 0.9с вместо 6.9с."""
|
||
delay = float(_cfg.get("per_host_delay", 0.6))
|
||
with _lock:
|
||
now = time.time()
|
||
prev = _last.get(host, 0)
|
||
wait = max(0.0, delay - (now - prev))
|
||
_last[host] = now + wait
|
||
if wait > 0:
|
||
time.sleep(wait)'''),
|
||
])
|
||
|
||
# ---------- ai.py: не спать в пустоту ----------
|
||
patch("omni/ai.py", [
|
||
(''' except Exception:
|
||
time.sleep(1.5 + attempt)''',
|
||
''' except Exception:
|
||
if attempt == 0: # после последней попытки спим в пустоту
|
||
time.sleep(1.5)'''),
|
||
])
|
||
|
||
# ---------- bot.py ----------
|
||
patch("omni/bot.py", [
|
||
(" new += store.add(items)",
|
||
" n_new, _dup = store.add(items)\n new += n_new"),
|
||
(" chunk = fresh[-per_chat:]",
|
||
" chunk = fresh[:per_chat] # раньше уходили САМЫЕ СТАРЫЕ из окна"),
|
||
('''CMDS = {"/start": cmd_start,''',
|
||
'''READONLY_CMDS = {"/start", "/help", "/new", "/abuse", "/find", "/src", "/stats"}
|
||
|
||
CMDS = {"/start": cmd_start,'''),
|
||
(''' if OWNERS and uid not in OWNERS:
|
||
return
|
||
if not OWNERS:
|
||
OWNERS.add(uid) # первый написавший становится владельцем до рестарта
|
||
parts = text.split(None, 1)
|
||
cmd = parts[0].lower().split("@")[0]
|
||
arg = parts[1] if len(parts) > 1 else ""''',
|
||
''' parts = text.split(None, 1)
|
||
cmd = parts[0].lower().split("@")[0]
|
||
arg = parts[1] if len(parts) > 1 else ""
|
||
if OWNERS and uid not in OWNERS:
|
||
return
|
||
if not OWNERS and cmd not in READONLY_CMDS:
|
||
# раньше первый написавший становился владельцем: любой, кто нашёл бота
|
||
return _send(chat_id, "read-only: задай OMNI_BOT_OWNERS и перезапусти бота")'''),
|
||
("Доступ: OMNI_BOT_OWNERS=7448683285,123 (если пусто — только первый написавший)",
|
||
"Доступ: OMNI_BOT_OWNERS=<id>,<id> (пусто = read-only: /new /abuse /find /src /stats)"),
|
||
])
|
||
|
||
# ---------- sources.py: дедуп по url ----------
|
||
p = ROOT / "omni/sources.py"
|
||
s = p.read_text(encoding="utf-8")
|
||
if "def all_sources():" in s:
|
||
s2 = s.replace("def all_sources():", '''def all_sources():
|
||
"""Реестр без дублей по url: одинаковые площадки приходили из разных наборов."""''', 1)
|
||
# добавляем дедуп в возврат: ищем return внутри all_sources
|
||
idx = s2.index("def all_sources():")
|
||
tail = s2[idx:]
|
||
if "return _dedup(" in tail:
|
||
report.append("sources.py: дедуп уже есть")
|
||
else:
|
||
# оборачиваем последний return списка
|
||
import re
|
||
m = list(re.finditer(r"\n return (\[\]|\w+|\w+ \+|.*?)\n(?=\n|$)", tail))
|
||
report.append(f"sources.py: return-кандидатов {len(m)}")
|
||
# используем _dedup через хелпер, определённый перед all_sources
|
||
s2 = s2.replace("def all_sources():", '''def _dedup(items):
|
||
seen, out = set(), []
|
||
for it in items:
|
||
key = (it.url or "").rstrip("/")
|
||
if key and key in seen:
|
||
continue
|
||
seen.add(key)
|
||
out.append(it)
|
||
return out
|
||
|
||
|
||
def all_sources():''', 1)
|
||
s2 = re.sub(r"(\n return )(?!_dedup)(\[[^\n]*\]|\w+(?: \+ \w+)*)\n", r"\1_dedup(\2)\n", s2, count=1)
|
||
report.append("sources.py: _dedup + обёртка return")
|
||
s = s2
|
||
p.write_text(s, encoding="utf-8", newline="\n")
|
||
|
||
print("\n".join(report)) |