58 lines
1.9 KiB
Python
58 lines
1.9 KiB
Python
"""Чистка мусора из tg_веб-источников: служебные сообщения и пустыши.
|
||
|
||
Причина: старый адаптер _tg_web брал любой div.tgme_widget_message_text, включая
|
||
"Channel created", "Channel photo updated" и посты из одной ссылки.
|
||
|
||
python tools/purge_tg_junk.py # показать
|
||
python tools/purge_tg_junk.py --apply # удалить
|
||
"""
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||
|
||
from omni import config, store # noqa: E402
|
||
|
||
SERVICE_RE = re.compile(
|
||
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
|
||
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
|
||
|
||
|
||
def junk(title):
|
||
t = (title or "").strip()
|
||
if len(t) < 25:
|
||
return True
|
||
if SERVICE_RE.match(t):
|
||
return True
|
||
if sum(ch.isalpha() for ch in t) < 25:
|
||
return True
|
||
return False
|
||
|
||
|
||
def main():
|
||
apply = "--apply" in sys.argv
|
||
c = store.conn()
|
||
rows = c.execute("SELECT id, source, title FROM items WHERE source LIKE 'tg_%'").fetchall()
|
||
bad = [(i, s, t) for i, s, t in rows if junk(t)]
|
||
print(f"tg-записей {len(rows)}, мусора {len(bad)}")
|
||
for i, s, t in bad[:10]:
|
||
print(" -", s, "|", (t or "")[:60])
|
||
if apply and bad:
|
||
ids = [i for i, _s, _t in bad]
|
||
c.executemany("DELETE FROM items WHERE id=?", [(i,) for i in ids])
|
||
try:
|
||
c.executemany("DELETE FROM items_fts WHERE rowid=?", [(i,) for i in ids])
|
||
except Exception:
|
||
pass
|
||
try:
|
||
c.executemany("DELETE FROM bands WHERE item_id=?", [(i,) for i in ids])
|
||
except Exception:
|
||
pass
|
||
c.commit()
|
||
print("удалено", len(ids))
|
||
c.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |