feat(tg): реальные абузы из Telegram без сессии — адаптер tg_web переписан (служебное/пустыши отсекаются, заголовок из тела, порог 12/8 для leak-каналов), 19 живых каналов из 87 проверенных, сигнал абуза в score (+до 1.8), tools/tg_discover.py, tools/fixtg.py, tools/purge_tg_junk.py, tools/rescore.py; реестр 869, база 7295 (ab_ 3135, tg_ 374); тесты 19
Some checks failed
ci / test (3.10) (push) Has been cancelled
ci / test (3.11) (push) Has been cancelled
ci / test (3.12) (push) Has been cancelled

This commit is contained in:
Vlad Gothbreach
2026-10-26 21:06:50 +03:00
parent f9719b9149
commit eaedd37bcd
11 changed files with 414 additions and 22 deletions

58
tools/purge_tg_junk.py Normal file
View File

@@ -0,0 +1,58 @@
"""Чистка мусора из tg_веб-источников: служебные сообщения и пустыши.
Причина: старый адаптер _tg_web брал любой div.tgme_widget_message_text, включая
"Channel created", "Channel photo updated" и посты из одной ссылки.
python tools/purge_tg_junk.py # показать
python tools/purge_tg_junk.py --apply # удалить
"""
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from omni import config, store # noqa: E402
SERVICE_RE = re.compile(
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
def junk(title):
t = (title or "").strip()
if len(t) < 25:
return True
if SERVICE_RE.match(t):
return True
if sum(ch.isalpha() for ch in t) < 25:
return True
return False
def main():
apply = "--apply" in sys.argv
c = store.conn()
rows = c.execute("SELECT id, source, title FROM items WHERE source LIKE 'tg_%'").fetchall()
bad = [(i, s, t) for i, s, t in rows if junk(t)]
print(f"tg-записей {len(rows)}, мусора {len(bad)}")
for i, s, t in bad[:10]:
print(" -", s, "|", (t or "")[:60])
if apply and bad:
ids = [i for i, _s, _t in bad]
c.executemany("DELETE FROM items WHERE id=?", [(i,) for i in ids])
try:
c.executemany("DELETE FROM items_fts WHERE rowid=?", [(i,) for i in ids])
except Exception:
pass
try:
c.executemany("DELETE FROM bands WHERE item_id=?", [(i,) for i in ids])
except Exception:
pass
c.commit()
print("удалено", len(ids))
c.close()
if __name__ == "__main__":
main()