"""Telegram-бот поверх парсера: лента, абузы, источники, прогон, автодобавление. Запуск: OMNI_BOT_TOKEN=... python -m omni.bot Доступ: OMNI_BOT_OWNERS=, (пусто = read-only: /new /abuse /find /src /stats /tag) Бот не постит никуда сам: только отвечает в личный чат тем, кто в списке. Команды: /new [n] последние записи /abuse [тег] абузы: promo trial keys leak tool resale /find <текст> поиск по базе /src <текст> поиск источника /add добавить источник (сначала реальная проверка фида) /run <категория> прогнать категорию /auto [n] автодобавление новых фидов /sub [@канал] подписать чат/канал на новые абузы /unsub [@канал] отписаться /post [@канал] опубликовать абузы сейчас\n /sum [тег] ИИ-саммари подборки\n /tr перевод записи /stats цифры """ from __future__ import annotations import html import json import os import re import time import urllib.parse import urllib.request from . import abuse as abuse_mod from . import ai as ai_mod from . import discover as discover_mod from . import feeds, net, source_state, sources, store API = "https://api.telegram.org/bot%s/" % os.environ.get("OMNI_BOT_TOKEN", "") TOKEN = os.environ.get("OMNI_BOT_TOKEN", "") OWNERS = {int(x) for x in re.findall(r"\d{5,}", os.environ.get("OMNI_BOT_OWNERS") or os.environ.get("OMNI_BOT_ALLOW") or "")} MAX_LEN = 3600 def _api(method, **params): if not TOKEN: raise SystemExit("OMNI_BOT_TOKEN не задан") url = API + method data = urllib.parse.urlencode(params).encode() req = urllib.request.Request(url, data=data) with urllib.request.urlopen(req, timeout=40) as resp: return json.loads(resp.read().decode("utf-8")) def _send(chat_id, text, preview=True, reply_markup=None): text = text[:MAX_LEN] if os.environ.get("OMNI_BOT_DRY"): print("[DRY] " + text.replace("", "").replace("", "")[:1200]) return {"ok": True} kw = {} if reply_markup: kw["reply_markup"] = json.dumps(reply_markup, ensure_ascii=False) return _api("sendMessage", chat_id=chat_id, text=text, parse_mode="HTML", disable_web_page_preview="false" if preview else "true", **kw) def _esc(s): return html.escape(str(s or ""), quote=False) def fmt_item(it): title = _esc((it.get("title") or "")[:150]) src = _esc(it.get("source") or "") url = it.get("url") or "" line = "%s\n%s" % (title, src) if url: line += ' · открыть' % _esc(url) return line def cmd_start(chat_id, arg): _send(chat_id, "omni — парсер-радар\n\n" "/new 5 — последние записи\n" "/abuse promo — абузы по тегу\n" "/find claude — поиск\n" "/src reddit — источник\n" "/add <url> — подключить фид\n" "/run abuse — прогнать категорию\n" "/auto 10 — автодобавление\n" "/stats — цифры\n/sub @канал — постить новые абузы в канал\n/post @канал — опубликовать сейчас", reply_markup=KB_MAIN) def cmd_new(chat_id, arg): n = int(arg) if arg.strip().isdigit() else 8 items, _total = store.browse2(limit=min(n, 20), sort="fresh") if not items: return _send(chat_id, "пусто") _send(chat_id, "последние %d:\n\n" % len(items) + "\n\n".join(fmt_item(i) for i in items)) def cmd_abuse(chat_id, arg): tag = arg.strip().split()[0] if arg.strip() else None d = abuse_mod.scan(limit=10, tag=tag) if not d["items"]: return _send(chat_id, "абузов не видно") head = "абузы · %s · %d\n\n" % ("все" if not tag else tag, d["total"]) body = [] for it in d["items"]: body.append("%s\n%s · %s" % ( _esc(it["title"][:120]), _esc(it["source"]), _esc(",".join(it["tags"])))) _send(chat_id, head + "\n\n".join(body)) def cmd_find(chat_id, arg): q = arg.strip() if not q: return _send(chat_id, "что искать?") rows = store.search(q, limit=8) keys = ("provider", "kind", "source", "title", "score", "url") items = [dict(zip(keys, r)) if not isinstance(r, dict) else r for r in rows] if not items: return _send(chat_id, "нет совпадений") _send(chat_id, "поиск: %s\n\n" % _esc(q) + "\n\n".join(fmt_item(i) for i in items)) def cmd_src(chat_id, arg): q = arg.strip().lower() all_s = sources.all_sources() hits = [s for s in all_s if q and (q in s.name.lower() or q in s.url.lower() or q in s.category.lower())] if not hits: return _send(chat_id, "нет такого источника") out = ["%s · %s · %s" % (_esc(s.name), _esc(s.category), _esc(s.adapter)) for s in hits[:12]] _send(chat_id, "источники (%d):\n\n" % len(hits) + "\n".join(out)) def cmd_add(chat_id, arg): url = arg.strip() if not url.startswith("http"): return _send(chat_id, "нужен url фида") ok, n, ad = discover_mod.probe(url, "community", limit=6) if not ok: return _send(chat_id, "фид пустой или недоступен: %s" % _esc(ad)) name = "manual_" + re.sub(r"[^a-z0-9]+", "_", urllib.parse.urlsplit(url).netloc.lower())[:32] source_state.add_custom(name, "community", url, kind="rss" if ad == "rss" else "html", adapter=ad, tags=["manual"], limit=15) _send(chat_id, "добавлен %s · %s · %d записей" % (_esc(name), _esc(ad), n)) def cmd_run(chat_id, arg): cat = arg.strip() or "abuse" srcs = [s for s in sources.all_sources() if s.category == cat] if not srcs: return _send(chat_id, "нет категории %s" % _esc(cat)) _send(chat_id, "прогон %s (%d источников)…" % (_esc(cat), len(srcs))) got = new = 0 for s in srcs[:80]: try: items = feeds.fetch(s, 10) except Exception: items = [] got += len(items) if items: n_new, _dup = store.add(items) new += n_new _send(chat_id, "готово: собрано %d, новых %d" % (got, new)) def cmd_auto(chat_id, arg): n = int(arg) if arg.strip().isdigit() else 15 res = discover_mod.discover(limit_hosts=30, workers=8, max_add=min(n, 30)) _send(chat_id, "автодобавление: кандидатов %d, живых %d\n%s" % ( res["found"], res["alive"], _esc(", ".join(res["added"][:10])))) def cmd_stats(chat_id, arg): s = store.stats() a = sources.all_sources() fl = source_state.flags() alive = sum(1 for x in a if (fl.get(x.name, {}).get("items") or 0) > 0) _send(chat_id, "omni\nзаписей %s · источников %d · живых %s\nкатегорий %d · прогонов %s · с ценой %s" % ( s.get("items"), len(a), alive, len(sources.CATEGORIES), s.get("runs"), s.get("priced"))) # --- подписка, формат, канал ------------------------------------------------- HIDE_TAGS = {"keys"} # про ключи не постим и не показываем SUBS_SQL = ("CREATE TABLE IF NOT EXISTS abuse_subs (chat_id INTEGER PRIMARY KEY, " "ts INTEGER DEFAULT 0, last_id INTEGER DEFAULT 0)") KB_MAIN = {"inline_keyboard": [ [{"text": "Лента", "callback_data": "m:new"}, {"text": "Абузы", "callback_data": "m:abuse"}], [{"text": "Источники", "callback_data": "m:src"}, {"text": "Цифры", "callback_data": "m:stats"}], [{"text": "Промо", "callback_data": "t:promo"}, {"text": "Триалы", "callback_data": "t:trial"}, {"text": "Сливы", "callback_data": "t:leak"}], [{"text": "Подписка: вкл", "callback_data": "sub:on"}, {"text": "выкл", "callback_data": "sub:off"}], [{"text": "Формат: кратко", "callback_data": "fmt:short"}, {"text": "полно", "callback_data": "fmt:full"}], [{"text": "ИИ-саммари", "callback_data": "ai:sum"}, {"text": "Перевод: вкл/выкл", "callback_data": "ai:tr"}], [{"text": "Саммари в подписке: вкл/выкл", "callback_data": "ai:sm"}], [{"text": "Пост в канал", "callback_data": "post:now"}], ]} PREFS_SQL = "CREATE TABLE IF NOT EXISTS bot_prefs (chat_id INTEGER PRIMARY KEY, fmt TEXT DEFAULT 'short')" def _conn(): import sqlite3 from .config import DB c = sqlite3.connect(str(DB), timeout=30) c.execute(SUBS_SQL) c.execute(PREFS_SQL) c.commit() return c def prefs_get(chat_id): c = _conn() r = c.execute("SELECT fmt FROM bot_prefs WHERE chat_id=?", (chat_id,)).fetchone() c.close() return (r[0] if r else "short") def prefs_set(chat_id, fmt): c = _conn() c.execute("INSERT INTO bot_prefs(chat_id, fmt) VALUES(?,?) " "ON CONFLICT(chat_id) DO UPDATE SET fmt=excluded.fmt", (chat_id, fmt)) c.commit() c.close() def subs_add(target, last_id=0): c = _conn() if not last_id: last_id = c.execute("SELECT COALESCE(MAX(id),0) FROM items").fetchone()[0] c.execute("INSERT INTO abuse_subs(chat_id, ts, last_id) VALUES(?,?,?) " "ON CONFLICT(chat_id) DO UPDATE SET last_id=excluded.last_id", (target, int(time.time()), last_id)) c.commit() c.close() return last_id def subs_del(target): c = _conn() n = c.execute("DELETE FROM abuse_subs WHERE chat_id=?", (target,)).rowcount c.commit() c.close() return n def subs_all(): c = _conn() rows = c.execute("SELECT chat_id, last_id FROM abuse_subs ORDER BY ts DESC").fetchall() c.close() return rows def _resolve_chat(arg, chat_id): a = (arg or "").strip() if not a: return chat_id if a.lstrip("-").isdigit(): return int(a) try: r = _api("getChat", chat_id=a) return (r.get("result") or {}).get("id") except Exception: return None def _scan_safe(limit=10, tag=None, window=6000): d = abuse_mod.scan(limit=limit * 3, tag=tag, window=window) out = [] for it in d.get("items", []): tags = [t for t in (it.get("tags") or []) if t not in HIDE_TAGS] if not tags: continue it = dict(it) it["tags"] = tags out.append(it) d["items"] = out[:limit] d["hidden_keys"] = len(d.get("items", [])) - len(out) return d def _line(it, fmt="short", tr=False): raw = (it.get("title") or "")[:150] if tr: raw = ai_mod.translate(raw) or raw title = _esc(raw) src = _esc(it.get("source") or "") tags = _esc(",".join(it.get("tags") or [])) head = "%s\n%s · %s" % (title, src, tags) if fmt == "full": url = it.get("url") or "" if url: head += ' · открыть' % _esc(url) return head AI_SQL = "CREATE TABLE IF NOT EXISTS bot_ai (chat_id INTEGER PRIMARY KEY, tr INTEGER DEFAULT 0, sm INTEGER DEFAULT 0)" def ai_prefs(chat_id): c = _conn() c.execute(AI_SQL) c.commit() r = c.execute("SELECT tr, sm FROM bot_ai WHERE chat_id=?", (chat_id,)).fetchone() c.close() return (r[0], r[1]) if r else (0, 0) def ai_set(chat_id, field, val): c = _conn() c.execute(AI_SQL) c.execute("INSERT INTO bot_ai(chat_id, tr, sm) VALUES(?,0,0) ON CONFLICT(chat_id) DO NOTHING", (chat_id,)) c.execute("UPDATE bot_ai SET %s=? WHERE chat_id=?" % ("tr" if field == "tr" else "sm"), (val, chat_id)) c.commit() c.close() def ai_scan_summary(limit=12, tag=None): """ИИ-саммари свежих абузов. None если ИИ недоступен.""" d = _scan_safe(limit=limit, tag=tag) items = d.get("items") or [] if not items: return None, 0 posts = ["%s — %s" % (i.get("title") or "", i.get("source") or "") for i in items] return ai_mod.summarize(posts, tag=tag), len(items) def cmd_sum(chat_id, arg): tag = arg.strip().split()[0] if arg.strip() else None if not ai_mod.available(): return _send(chat_id, "ИИ не подключён: задай OMNI_AI_KEY (или ключ reformboss в конфиге Hermes)", reply_markup=KB_MAIN) txt, n = ai_scan_summary(limit=14, tag=tag) if not txt: return _send(chat_id, "саммари не собралось", reply_markup=KB_MAIN) _send(chat_id, "саммари · %s · %d записей\n\n%s" % (tag or "все", n, _esc(txt)), reply_markup=KB_MAIN) def cmd_tr(chat_id, arg): """Перевод записи: /tr .""" q = arg.strip() if not q: return _send(chat_id, "что перевести? /tr 4933 или /tr muse", reply_markup=KB_MAIN) if not ai_mod.available(): return _send(chat_id, "ИИ не подключён: задай OMNI_AI_KEY", reply_markup=KB_MAIN) title = text = src = url = None if q.isdigit(): import sqlite3 from .config import DB c = sqlite3.connect(str(DB), timeout=20) r = c.execute("SELECT title, text, source, url FROM items WHERE id=?", (int(q),)).fetchone() c.close() if r: title, text, src, url = r[0], r[1] or "", r[2], r[3] if title is None: import sqlite3 from .config import DB c = sqlite3.connect(str(DB), timeout=20) r = c.execute("SELECT title, text, source, url FROM items WHERE title LIKE ? " "ORDER BY id DESC LIMIT 1", ("%" + q + "%",)).fetchone() c.close() if r: title, text, src, url = r[0], r[1] or "", r[2], r[3] if title is None: rows = store.search(q, limit=1) if rows: r = rows[0] if isinstance(r, dict): title, src, url = r.get("title"), r.get("source"), r.get("url") text = r.get("text") or "" else: src, title, url = r[2], r[3], r[5] text = "" if not title: return _send(chat_id, "не нашёл запись", reply_markup=KB_MAIN) payload = title if len(title) > 120 else (title + "\n" + (text or "")[:600]).strip() ru = ai_mod.translate(payload) if not ru: return _send(chat_id, "перевод не получился, попробуй ещё раз", reply_markup=KB_MAIN) out = "%s\n%s\n\n%s" % (_esc(title[:150]), _esc(src), _esc(ru)) if url: out += '\nисточник' % _esc(url) _send(chat_id, out, reply_markup=KB_MAIN) def abuse_text(tag=None, limit=6, fmt="short", tr=False): d = _scan_safe(limit=limit, tag=tag) if not d["items"]: return None head = "абузы · %s · %d" % (tag or "все", d["total"]) if tr: head += " · перевод" lines = [] for n, i in enumerate(d["items"]): lines.append(_line(i, fmt, tr and n < 5)) return head + "\n\n" + "\n\n".join(lines) def cmd_abuse(chat_id, arg): tag = arg.strip().split()[0] if arg.strip() else None tr = bool(ai_prefs(chat_id)[0]) and ai_mod.available() txt = abuse_text(tag, fmt=prefs_get(chat_id), tr=tr) _send(chat_id, txt or "абузов не видно", reply_markup=KB_MAIN) def cmd_sub(chat_id, arg): target = _resolve_chat(arg, chat_id) if not target: return _send(chat_id, "не вижу такой чат: добавь бота админом и повтори /sub @канал", reply_markup=KB_MAIN) last = subs_add(target) where = "сюда" if target == chat_id else "%s" % target _send(chat_id, "подписан: %s, шлю только новые абузы (после id %d). /unsub — отписаться" % (where, last), reply_markup=KB_MAIN) def cmd_unsub(chat_id, arg): target = _resolve_chat(arg, chat_id) or chat_id n = subs_del(target) _send(chat_id, "отписал" if n else "подписки не было", reply_markup=KB_MAIN) def cmd_subs(chat_id, arg): rows = subs_all() if not rows: return _send(chat_id, "подписок нет", reply_markup=KB_MAIN) body = "\n".join("%s · id %s" % (c, i) for c, i in rows[:20]) _send(chat_id, "подписки (%d)\n%s" % (len(rows), body), reply_markup=KB_MAIN) def publish(target, tag=None, limit=5): """Один пост с абузами в указанный чат/канал.""" fmt = prefs_get(target) if isinstance(target, int) else "short" txt = abuse_text(tag, limit=limit, fmt=fmt) if not txt: return False _send(target, txt) return True def cmd_test(chat_id, arg): target = _resolve_chat(arg, chat_id) if not target: return _send(chat_id, "куда постить? /post @канал", reply_markup=KB_MAIN) ok = publish(target) _send(chat_id, "опубликовал" if ok else "нечего публиковать", reply_markup=KB_MAIN) def push_new(per_chat=5, window=2000): subs = subs_all() if not subs: return 0 d = _scan_safe(limit=400, window=window) items = sorted(d["items"], key=lambda x: x.get("id") or 0) sent = 0 for target, last in subs: fresh = [i for i in items if (i.get("id") or 0) > (last or 0)] if not fresh: continue chunk = fresh[:per_chat] # раньше уходили САМЫЕ СТАРЫЕ из окна fmt = prefs_get(target) if isinstance(target, int) else "short" tr = sm = 0 if isinstance(target, int): tr, sm = ai_prefs(target) tr = bool(tr) and ai_mod.available() pre = "" if sm and ai_mod.available(): posts = ["%s — %s" % (i.get("title") or "", i.get("source") or "") for i in chunk] summary = ai_mod.summarize(posts) if summary: pre = "саммари\n%s\n\n" % _esc(summary) text = pre + ("новые абузы · %d\n\n" % len(fresh)) + "\n\n".join(_line(i, fmt, tr) for i in chunk) try: _send(target, text[:MAX_LEN]) sent += 1 except Exception as e: print("push fail", target, e, flush=True) continue subs_add(target, last_id=chunk[-1].get("id") or last) return sent def pusher(every=300): while True: time.sleep(every) try: n = push_new() if n: print("pushed", n, flush=True) except Exception as e: print("pusher fail", e, flush=True) READONLY_CMDS = {"/start", "/help", "/new", "/abuse", "/find", "/src", "/stats", "/tag", "/типы", "/sum", "/tr"} def cmd_tag(chat_id, arg): """Типы записей: /tag — счётчики, /tag leak — топ записей этого типа.""" from . import taxonomy t = arg.strip().lower() if not t: counts = store.type_counts() rows = sorted(counts.items(), key=lambda x: -x[1])[:14] body = "\n".join("%-12s %d" % (taxonomy.LABELS.get(k, k), v) for k, v in rows) return _send(chat_id, "типы записей\n
%s
\n/tag <тип> — топ" % _esc(body), reply_markup=KB_MAIN) if t not in taxonomy.LABELS: return _send(chat_id, "нет такого типа: %s" % _esc(t), reply_markup=KB_MAIN) items, total = store.browse2(limit=8, sort="score", typ=t) if not items: return _send(chat_id, "по типу %s пусто" % _esc(t), reply_markup=KB_MAIN) head = "%s · всего %d\n\n" % (_esc(taxonomy.LABELS.get(t, t)), total) _send(chat_id, head + "\n\n".join(fmt_item(i) for i in items), reply_markup=KB_MAIN) CMDS = {"/start": cmd_start, "/help": cmd_start, "/new": cmd_new, "/abuse": cmd_abuse, "/find": cmd_find, "/src": cmd_src, "/add": cmd_add, "/run": cmd_run, "/auto": cmd_auto, "/stats": cmd_stats, "/sub": cmd_sub, "/unsub": cmd_unsub, "/subs": cmd_subs, "/post": cmd_test, "/sum": cmd_sum, "/tr": cmd_tr, "/tag": cmd_tag, "/типы": cmd_tag} def handle(update): msg = update.get("message") or update.get("edited_message") or {} chat = msg.get("chat") or {} chat_id = chat.get("id") text = (msg.get("text") or "").strip() uid = (msg.get("from") or {}).get("id") if not chat_id or not text: return parts = text.split(None, 1) cmd = parts[0].lower().split("@")[0] arg = parts[1] if len(parts) > 1 else "" if OWNERS and uid not in OWNERS: return if not OWNERS and cmd not in READONLY_CMDS: # раньше первый написавший становился владельцем: любой, кто нашёл бота return _send(chat_id, "read-only: задай OMNI_BOT_OWNERS и перезапусти бота") fn = CMDS.get(cmd) if not fn: return _send(chat_id, "не знаю такую команду. /help") try: fn(chat_id, arg) except Exception as e: _send(chat_id, "ошибка: %s" % _esc(type(e).__name__ + ": " + str(e)[:160])) def handle_cb(cb): cb_id = cb.get("id") data = cb.get("data") or "" msg = cb.get("message") or {} chat_id = (msg.get("chat") or {}).get("id") try: _api("answerCallbackQuery", callback_query_id=cb_id) except Exception: pass if not chat_id or (OWNERS and (cb.get("from") or {}).get("id") not in OWNERS): return fmt = prefs_get(chat_id) if data == "m:new": return cmd_new(chat_id, "5") if data == "m:abuse": return cmd_abuse(chat_id, "") if data == "m:src": return cmd_src(chat_id, "") if data == "m:stats": return cmd_stats(chat_id, "") if data.startswith("t:"): return cmd_abuse(chat_id, data[2:]) if data == "sub:on": return cmd_sub(chat_id, "") if data == "sub:off": return cmd_unsub(chat_id, "") if data.startswith("fmt:"): prefs_set(chat_id, data[4:]) return _send(chat_id, "формат: %s" % ("кратко" if data[4:] == "short" else "полно"), reply_markup=KB_MAIN) if data == "post:now": return cmd_test(chat_id, "") if data == "ai:sum": return cmd_sum(chat_id, "") if data == "ai:tr": tr, _sm = ai_prefs(chat_id) ai_set(chat_id, "tr", 0 if tr else 1) return _send(chat_id, "перевод абузов: %s" % ("выкл" if tr else "вкл"), reply_markup=KB_MAIN) if data == "ai:sm": _tr, sm = ai_prefs(chat_id) ai_set(chat_id, "sm", 0 if sm else 1) return _send(chat_id, "саммари в подписке: %s" % ("выкл" if sm else "вкл"), reply_markup=KB_MAIN) def run_forever(): if not TOKEN: raise SystemExit("нужен OMNI_BOT_TOKEN") offset = 0 import threading threading.Thread(target=pusher, daemon=True).start() print("bot up", flush=True) while True: try: r = _api("getUpdates", offset=offset, timeout=25, allowed_updates='["message","callback_query"]') except Exception as e: print("poll fail", e, flush=True) time.sleep(5) continue for up in r.get("result", []): offset = up["update_id"] + 1 if up.get("callback_query"): try: handle_cb(up["callback_query"]) except Exception as e: print("cb fail", e, flush=True) else: handle(up) time.sleep(0.3) def _single_instance(): """Один процесс на токен: второй инстанс выходит, иначе TelegramConflictError и бот молчит.""" import os, sys, time, tempfile path = os.path.join(tempfile.gettempdir(), "omni_bot.lock") try: if os.path.exists(path): age = time.time() - os.path.getmtime(path) try: pid = int((open(path, encoding="utf-8").read() or "0").strip() or 0) except ValueError: pid = 0 if pid and age < 6 * 3600: alive = False try: import psutil alive = psutil.pid_exists(pid) and "omni" in " ".join(psutil.Process(pid).cmdline() or []) except Exception: alive = False if alive: print("уже запущен pid %s - выхожу" % pid, flush=True) sys.exit(0) open(path, "w", encoding="utf-8").write(str(os.getpid())) os.utime(path, None) except Exception as e: print("lock skip", e, flush=True) if __name__ == "__main__": _single_instance() run_forever()