feat(tg): реальные абузы из Telegram без сессии — адаптер tg_web переписан (служебное/пустыши отсекаются, заголовок из тела, порог 12/8 для leak-каналов), 19 живых каналов из 87 проверенных, сигнал абуза в score (+до 1.8), tools/tg_discover.py, tools/fixtg.py, tools/purge_tg_junk.py, tools/rescore.py; реестр 869, база 7295 (ab_ 3135, tg_ 374); тесты 19
This commit is contained in:
@@ -27,7 +27,13 @@ def _txt(x):
|
||||
def _item(src, kind, url, title, text="", author="", published="", extra=None):
|
||||
ex = dict(extra or {})
|
||||
ex["tags"] = list(src.tags)
|
||||
return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
|
||||
t = _txt(title)
|
||||
body = _txt(text)
|
||||
if sum(ch.isalpha() for ch in t) < 12 and body:
|
||||
cand = [ln for ln in body.splitlines() if sum(ch.isalpha() for ch in ln) >= 12]
|
||||
if cand:
|
||||
t = max(cand, key=len)[:140]
|
||||
return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
|
||||
author=_txt(author)[:80], published=str(published or ""), source=src.name,
|
||||
extra=ex)
|
||||
|
||||
@@ -344,20 +350,63 @@ def _xenforo_latest(src, limit):
|
||||
return out
|
||||
|
||||
|
||||
SERVICE_RE = re.compile(
|
||||
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
|
||||
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
|
||||
|
||||
|
||||
def _tg_web(src, limit):
|
||||
"""Публичное веб-превью t.me/s/<канал>.
|
||||
|
||||
Отсекаем служебное (Channel created / photo updated / pinned) и пустыши:
|
||||
раньше в базу уезжали заголовки "@upenbullet" и "Channel photo removed"."""
|
||||
r = net.get(src.url, headers=UA)
|
||||
if r.status != 200:
|
||||
return []
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
out = []
|
||||
for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]:
|
||||
text_el = msg.select_one("div.tgme_widget_message_text")
|
||||
link_el = msg.select_one("a.tgme_widget_message_date")
|
||||
text = text_el.get_text("\n", strip=True) if text_el else ""
|
||||
link = str(link_el.get("href") or src.url) if link_el else src.url
|
||||
if not text:
|
||||
for msg in soup.select("div.tgme_widget_message_wrap")[-max(limit * 2, 20):]:
|
||||
if msg.select_one("div.tgme_widget_message_service"):
|
||||
continue
|
||||
out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {}))
|
||||
text_el = msg.select_one("div.tgme_widget_message_text")
|
||||
if not text_el:
|
||||
continue
|
||||
text = text_el.get_text("\n", strip=True)
|
||||
text = re.sub(r"\n{2,}", "\n", text).strip()
|
||||
# лив-каналы пишут коротко («<домен> 2M users»), им порог ниже
|
||||
min_len = 12 if "leak" in src.tags else 25
|
||||
min_alpha = 8 if "leak" in src.tags else 15
|
||||
if len(text) < min_len:
|
||||
continue
|
||||
if SERVICE_RE.match(text):
|
||||
continue
|
||||
if sum(ch.isalpha() for ch in text) < min_alpha: # только ссылки/эмодзи/цифры
|
||||
continue
|
||||
link_el = msg.select_one("a.tgme_widget_message_date")
|
||||
link = str(link_el.get("href") or src.url) if link_el else src.url
|
||||
dt = ""
|
||||
t_el = msg.select_one("time[datetime]")
|
||||
if t_el:
|
||||
dt = str(t_el.get("datetime") or "")
|
||||
views = ""
|
||||
v_el = msg.select_one("span.tgme_widget_message_views")
|
||||
if v_el:
|
||||
views = v_el.get_text(" ", strip=True)
|
||||
via = ""
|
||||
vv = msg.select_one("span.tgme_widget_message_via")
|
||||
if vv:
|
||||
via = vv.get_text(" ", strip=True)
|
||||
lines = [ln.strip() for ln in text.split("\n")
|
||||
if ln.strip() and not ln.strip().startswith(("http", "t.co/"))]
|
||||
lines = [ln for ln in lines if sum(ch.isalpha() for ch in ln) >= 12]
|
||||
if not lines:
|
||||
continue
|
||||
first = [ln for ln in lines[:6] if len(ln) >= 20]
|
||||
head = (first[0] if first else max(lines[:6], key=len)).strip(' ,.;:-——')
|
||||
out.append(_item(src, "post", link, head[:140], text, via, dt[:10],
|
||||
{"views": views, "handle": src.url.rsplit("/", 1)[-1], "lang": ""}))
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user