reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(tg): реальные абузы из Telegram без сессии — адаптер tg_web переписан (служебное/пустыши отсекаются, заголовок из тела, порог 12/8 для leak-каналов), 19 живых каналов из 87 проверенных, сигнал абуза в score (+до 1.8), tools/tg_discover.py, tools/fixtg.py, tools/purge_tg_junk.py, tools/rescore.py; реестр 869, база 7295 (ab_ 3135, tg_ 374); тесты 19
Some checks failed
ci / test (3.10) (push) Has been cancelled
Details
ci / test (3.11) (push) Has been cancelled
Details
ci / test (3.12) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Vlad Gothbreach
2026-10-26 21:06:50 +03:00
parent f9719b9149
commit eaedd37bcd
11 changed files with 414 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
omni/feeds.py
View File

@@ -27,7 +27,13 @@ def _txt(x):
def _item(src, kind, url, title, text="", author="", published="", extra=None):
ex = dict(extra or {})
ex["tags"] = list(src.tags)
return Item(src.name, kind, url=str(url or ""), title=_txt(title)[:300], text=_txt(text)[:6000],
t = _txt(title)
body = _txt(text)
if sum(ch.isalpha() for ch in t) < 12 and body:
cand = [ln for ln in body.splitlines() if sum(ch.isalpha() for ch in ln) >= 12]
if cand:
t = max(cand, key=len)[:140]
return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
author=_txt(author)[:80], published=str(published or ""), source=src.name,
extra=ex)
@@ -344,20 +350,63 @@ def _xenforo_latest(src, limit):
return out
SERVICE_RE = re.compile(
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
def _tg_web(src, limit):
"""Публичное веб-превью t.me/s/<канал>.
Отсекаем служебное (Channel created / photo updated / pinned) и пустыши:
раньше в базу уезжали заголовки "@upenbullet" и "Channel photo removed"."""
r = net.get(src.url, headers=UA)
if r.status != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
out = []
for msg in soup.select("div.tgme_widget_message_wrap")[-limit:]:
text_el = msg.select_one("div.tgme_widget_message_text")
link_el = msg.select_one("a.tgme_widget_message_date")
text = text_el.get_text("\n", strip=True) if text_el else ""
link = str(link_el.get("href") or src.url) if link_el else src.url
if not text:
for msg in soup.select("div.tgme_widget_message_wrap")[-max(limit * 2, 20):]:
if msg.select_one("div.tgme_widget_message_service"):
continue
out.append(_item(src, "post", link, text.split("\n")[0][:120], text, "", "", {}))
text_el = msg.select_one("div.tgme_widget_message_text")
if not text_el:
continue
text = text_el.get_text("\n", strip=True)
text = re.sub(r"\n{2,}", "\n", text).strip()
# лив-каналы пишут коротко («<домен> 2M users»), им порог ниже
min_len = 12 if "leak" in src.tags else 25
min_alpha = 8 if "leak" in src.tags else 15
if len(text) < min_len:
continue
if SERVICE_RE.match(text):
continue
if sum(ch.isalpha() for ch in text) < min_alpha: # только ссылки/эмодзи/цифры
continue
link_el = msg.select_one("a.tgme_widget_message_date")
link = str(link_el.get("href") or src.url) if link_el else src.url
dt = ""
t_el = msg.select_one("time[datetime]")
if t_el:
dt = str(t_el.get("datetime") or "")
views = ""
v_el = msg.select_one("span.tgme_widget_message_views")
if v_el:
views = v_el.get_text(" ", strip=True)
via = ""
vv = msg.select_one("span.tgme_widget_message_via")
if vv:
via = vv.get_text(" ", strip=True)
lines = [ln.strip() for ln in text.split("\n")
if ln.strip() and not ln.strip().startswith(("http", "t.co/"))]
lines = [ln for ln in lines if sum(ch.isalpha() for ch in ln) >= 12]
if not lines:
continue
first = [ln for ln in lines[:6] if len(ln) >= 20]
head = (first[0] if first else max(lines[:6], key=len)).strip(' ,.;:-——')
out.append(_item(src, "post", link, head[:140], text, via, dt[:10],
{"views": views, "handle": src.url.rsplit("/", 1)[-1], "lang": ""}))
if len(out) >= limit:
break
return out

20
omni/rank.py
View File

@@ -1,10 +1,25 @@
"""Скор записи: вес провайдера + вовлечённость + свежесть + ценник + объём текста."""
"""Скор записи: вес провайдера + вовлечённость + свежесть + ценник + объём текста + сигнал абуза."""
import re
import time
WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0,
"firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8}
ABUSE_TERMS = re.compile(
r"(cve-\d{4}-\d+|0day|0-day|zero[- ]day|exploit|rce|sqli|xss|ssrf|rce|"
r"leak|leaked|dump|breach|combo[- ]?list|stealer|ransomware|botnet|credentials|"
r"password[s]?|api[- ]?key[s]?|token[s]?|аккаунт\w*|база|слив\w*|утечк\w*|комбо|"
r"взлом|эксплойт|уязвим\w*|парол\w*|ключи)",
re.I)
def _abuse(it):
"""Реальные абузы должны всплывать выше новостей: считаем сигнальные термины."""
blob = f"{it.title} {it.text}"[:2000]
hits = len({m.group(0).lower() for m in ABUSE_TERMS.finditer(blob)})
return min(4, hits)
KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "result": 0.1, "post": 0.1}
@@ -32,7 +47,8 @@ def score(it):
priced = 0.35 if it.price else 0.0
body = min(1.0, len(it.text) / 1200.0)
eng = min(3.0, base ** 0.5) if base > 1 else base
total = (w * (1.0 + eng) + fresh * 1.2 + priced + body + KIND_BONUS.get(it.kind, 0.0))
total = (w * (1.0 + eng) + fresh * 1.2 + priced + body + KIND_BONUS.get(it.kind, 0.0)
+ 0.45 * _abuse(it))
it.score = round(total, 4)
return it

3
omni/sources.py
View File

@@ -15,6 +15,7 @@ from .sources_ext2 import EXTRA2
from .sources_ext3 import EXTRA3
from .sources_abuse import ABUSE_LIVE
from .abuse_sources import ABUSE as ABUSE_DEEP
from .sources_tg_abuse import TG_ABUSE
from .sources_ext4 import EXTRA5
from .sources_ext5 import EXTRA6
from .sources_x import X_SOURCES
@@ -201,7 +202,7 @@ def _dedup(rows):
return out
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in X_SOURCES])
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES])
BY_NAME = {s.name: s for s in SOURCES}
CATEGORIES = sorted({s.category for s in SOURCES})

49
omni/sources_tg_abuse.py Normal file
View File

@@ -0,0 +1,49 @@
"""Telegram-каналы с абузами/сливами/утечками: публичное веб-превью t.me/s, без сессии и ключей.
Отобраны живыми: парсер даёт 1+ содержательный пост.
Выкинуты (превью пустое: только Channel created + инвайт-ссылки):
leakbase, abuse_ch, cracked_io, ai_zona, gpt_bot_news, codeby.
"""
TG_ABUSE = [
{"name": "tg_vxunderground", "category": "abuse", "kind": "html", "url": "https://t.me/s/vxunderground",
"adapter": "tg_web", "tags": ["malware", "abuse"], "limit": 25},
{"name": "tg_infosec_ru", "category": "abuse", "kind": "html", "url": "https://t.me/s/infosec_ru",
"adapter": "tg_web", "tags": ["security", "abuse"], "limit": 25},
{"name": "tg_true_secator", "category": "abuse", "kind": "html", "url": "https://t.me/s/true_secator",
"adapter": "tg_web", "tags": ["security", "abuse"], "limit": 25},
{"name": "tg_ai_newz", "category": "abuse", "kind": "html", "url": "https://t.me/s/ai_newz",
"adapter": "tg_web", "tags": ["ai", "news"], "limit": 25},
{"name": "tg_neurohive", "category": "abuse", "kind": "html", "url": "https://t.me/s/neurohive",
"adapter": "tg_web", "tags": ["abuse"], "limit": 25},
{"name": "tg_ai_free_ru", "category": "abuse", "kind": "html", "url": "https://t.me/s/ai_free_ru",
"adapter": "tg_web", "tags": ["abuse"], "limit": 25},
{"name": "tg_alstack", "category": "abuse", "kind": "html", "url": "https://t.me/s/alstack",
"adapter": "tg_web", "tags": ["own", "ai", "deals"], "limit": 25},
{"name": "tg_abuz_ai_soft", "category": "abuse", "kind": "html", "url": "https://t.me/s/Abuz_ai_soft",
"adapter": "tg_web", "tags": ["own", "abuse", "soft"], "limit": 25},
{"name": "tg_openai_news", "category": "abuse", "kind": "html", "url": "https://t.me/s/openai_news",
"adapter": "tg_web", "tags": ["ai", "news"], "limit": 25},
{"name": "tg_xakep_ru", "category": "abuse", "kind": "html", "url": "https://t.me/s/xakep_ru",
"adapter": "tg_web", "tags": ["security", "abuse"], "limit": 25},
{"name": "tg_ai_products", "category": "abuse", "kind": "html", "url": "https://t.me/s/ai_products",
"adapter": "tg_web", "tags": ["ai", "promo"], "limit": 25},
{"name": "tg_breachradar", "category": "abuse", "kind": "html", "url": "https://t.me/s/breachradar",
"adapter": "tg_web", "tags": ["leak", "abuse"], "limit": 25},
{"name": "tg_thehackernews", "category": "abuse", "kind": "html", "url": "https://t.me/s/thehackernews",
"adapter": "tg_web", "tags": ["security", "abuse"], "limit": 25},
{"name": "tg_ransomware_news", "category": "abuse", "kind": "html", "url": "https://t.me/s/ransomware_news",
"adapter": "tg_web", "tags": ["malware", "abuse"], "limit": 25},
{"name": "tg_vulnwatch", "category": "abuse", "kind": "html", "url": "https://t.me/s/vulnwatch",
"adapter": "tg_web", "tags": ["cve", "abuse"], "limit": 25},
{"name": "tg_poc_in_github", "category": "abuse", "kind": "html", "url": "https://t.me/s/poc_in_github",
"adapter": "tg_web", "tags": ["exploit", "abuse"], "limit": 25},
{"name": "tg_prompt_injection", "category": "abuse", "kind": "html", "url": "https://t.me/s/prompt_injection",
"adapter": "tg_web", "tags": ["ai", "abuse"], "limit": 25},
{"name": "tg_bleepingcomputer", "category": "abuse", "kind": "html", "url": "https://t.me/s/bleepingcomputer",
"adapter": "tg_web", "tags": ["security", "abuse"], "limit": 25},
{"name": "tg_cvefeed", "category": "abuse", "kind": "html", "url": "https://t.me/s/cvefeed",
"adapter": "tg_web", "tags": ["cve", "abuse"], "limit": 25},
{"name": "tg_hunt_news", "category": "abuse", "kind": "html", "url": "https://t.me/s/hunt_news",
"adapter": "tg_web", "tags": ["security", "news"], "limit": 25},
]