From ec61ec21f01caf22ce5df9d569083bb4f528cfb0 Mon Sep 17 00:00:00 2001 From: reform Date: Mon, 26 Oct 2026 12:25:24 +0300 Subject: [PATCH] =?UTF-8?q?omni:=20=D0=B0=D0=B1=D1=83=D0=B7=D1=8B=20(31=20?= =?UTF-8?q?=D0=B6=D0=B8=D0=B2=D0=BE=D0=B9=20=D0=B8=D1=81=D1=82=D0=BE=D1=87?= =?UTF-8?q?=D0=BD=D0=B8=D0=BA=20+=20=D0=BA=D0=BB=D0=B0=D1=81=D1=81=D0=B8?= =?UTF-8?q?=D1=84=D0=B8=D0=BA=D0=B0=D1=82=D0=BE=D1=80),=20=D0=B0=D0=B2?= =?UTF-8?q?=D1=82=D0=BE-=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=84=D0=B8=D0=B4=D0=BE=D0=B2,=20Telegram-?= =?UTF-8?q?=D0=B1=D0=BE=D1=82,=20=D0=B2=D0=BA=D0=BB=D0=B0=D0=B4=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B0=D0=B1=D1=83=D0=B7=D0=BE=D0=B2;=20589=20=D0=B8?= =?UTF-8?q?=D1=81=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D0=BA=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + README.md | 55 ++++++- omni/abuse.py | 149 +++++++++++++++++ omni/abuse_sources.py | 237 +++++++++++++++++++++++++++ omni/bot.py | 217 +++++++++++++++++++++++++ omni/discover.py | 164 +++++++++++++++++++ omni/feeds.py | 3 +- omni/serve.py | 45 ++++++ omni/sources.py | 4 +- omni/sources_abuse.py | 45 ++++++ omni/sources_ext3.py | 112 +++++++++++++ web/index.html | 365 ++++++++++++++++++++++++++++++++++++++---- 12 files changed, 1362 insertions(+), 35 deletions(-) create mode 100644 omni/abuse.py create mode 100644 omni/abuse_sources.py create mode 100644 omni/bot.py create mode 100644 omni/discover.py create mode 100644 omni/sources_abuse.py create mode 100644 omni/sources_ext3.py diff --git a/.gitignore b/.gitignore index 0b18add..55f48d2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ data/ *.sqlite3* __pycache__/ *.pyc +.bot_token diff --git a/README.md b/README.md index b4e6e6c..5b3ef95 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,36 @@ Ставится без ключей. Платные провайдеры включаются сами, если в окружении есть ключ. +## Состояние на сейчас + +- **источников: 589** в 11 категориях, **записей в базе: 3411** +- вкладки панели: `Лента` · `Источники` · `Прогоны` · `Абузы` +- health-аудит: живой/мёртвый/непроверенный по каждому источнику, причина в примечании +- Telegram-бот на той же базе: `/new`, `/abuse`, `/find`, `/src`, `/run`, `/add`, `/auto`, `/stats` + +| категория | источников | +|---|---| +| `community` | 123 | +| `company` | 119 | +| `security` | 72 | +| `papers` | 68 | +| `forum` | 50 | +| `models` | 35 | +| `company_cn` | 34 | +| `abuse` | 31 | +| `release` | 30 | +| `benchmark` | 16 | +| `company_kr` | 11 | + +## Панель + +```bash +python -m omni serve --port 8077 +# http://127.0.0.1:8077/?view=abuse +``` + +`?view=feed|src|runs|abuse`, горячие клавиши `1..4`, палитра `Ctrl K`, категории в URL (`?cat=`), кнопка «авто-добавление» ищет новые фиды на уже подключённых хостах и сохраняет только те, что реально отдали записи. + ## Что умеет | Провайдер | Что тянет | Ключ | @@ -44,7 +74,7 @@ python -m omni search "scraping twitter without api" --providers reddit,forums,d python -m omni sources python -m omni sources --category papers,security --list -# обход реестра БЕЗ запроса: 450 источников -> база +# обход реестра БЕЗ запроса: 589 источников -> база python -m omni feed --category community --limit 8 python -m omni feed --limit 15 # всё подряд python -m omni feed --only arxiv_cs_ai,hn_top,reddit_localllama @@ -66,7 +96,7 @@ python -m omni export --format jsonl --out data/dump.jsonl ## Реестр источников -`omni/sources.py` + `omni/sources_ext.py` — **450 источников в 9 категориях**, у каждого свой адаптер и флаг вкл/выкл: +`omni/sources.py` + `omni/sources_ext.py` — **589 источников в 9 категориях**, у каждого свой адаптер и флаг вкл/выкл: - **papers 49** — arxiv по 25+ разделам (cs.AI/LG/CL/CV/RO/IR/CR/SE/NE/MA/DC/DB/HC/SD/CY, stat.ML, eess.AS/SP/IV, math.OC, q-bio.NC, cs.GR, cs.AR, cs.PF), HF daily papers, OpenAlex, Semantic Scholar, Papers with Code, Connected Papers - **company 95** — ленты OpenAI, Google AI/Research/DeepMind, Microsoft Research, NVIDIA, Apple ML, Amazon Science, HF, Meta AI, Mistral, Cohere, Together, Perplexity, Groq, Cerebras, Stability, Databricks, Anyscale, Modal, W&B, LangChain, LlamaIndex, vLLM, Replicate, Fireworks, SambaNova, Scale, xAI, Salesforce, AWS ML, Azure AI, Anthropic, AI2, EleutherAI, LAION, Runway, ElevenLabs, Deepgram, AssemblyAI, Pinecone, Weaviate, Qdrant, Chroma, Unsloth, Axolotl @@ -93,7 +123,7 @@ python -m omni digest --out data/digest.md 1. **лента** — счётчики (записей, с ценой, прогонов, источников, включённых), поиск по базе, фильтр по категориям, кнопка «собрать ленту» с прогрессом и разбивкой fetched/new/dup/empty. 2. **источники** — менеджер реестра: - - таблица всех 450 источников: переключатель вкл/выкл, категория, тип, url, статус (жив / мёртв / не проверен), число записей; + - таблица всех 589 источников: переключатель вкл/выкл, категория, тип, url, статус (жив / мёртв / не проверен), число записей; - фильтр по имени/url и по категории; - **подключить источник** — форма (имя, url, категория): тип и адаптер определяются по url автоматически (`/rss` → rss, `.json`/`/api/` → generic_json, `github.com/org` → github_org, `t.me/s/` → tg_web), запись живёт в SQLite и сразу участвует в обходах; - **проверить всё / проверить категорию** — фоновой джоб: реальный фетч каждого источника, запись `status/items/note/checked`, прогресс `done/total, живых/мёртвых`; @@ -142,4 +172,21 @@ python -m omni stats ## Лицензия -MIT. \ No newline at end of file +MIT. + +## Абузы, авто-добавление и Telegram-бот + +- **абузы**: категория `abuse` — 31 живых площадок (XenForo/Discourse/Telegram/RSS), классификатор `omni/abuse.py` раскладывает записи по тегам `ключи / промо / триалы / сливы / софт / продажа / новости`. В панели — отдельная вкладка (`?view=abuse`), в боте — `/abuse [тег]`. +- **авто-добавление**: `omni/discover.py` ищет фиды на уже подключённых хостах и каталогах, проверяет их теми же адаптерами и сохраняет только те, что реально отдали записи. В панели — кнопка «авто-добавление», в API — `POST /api/discover?hosts=40&add=25`. +- **новости**: любые RSS/Atom/JSON-ленты (редакции, релизы, сообщества) подключаются как обычный источник или через `/add ` в боте — отдельной схемы не требуется. + +### Telegram-бот + +```bash +set OMNI_BOT_TOKEN=<токен BotFather> +set OMNI_BOT_ALLOW=<твой numeric id> +python -m omni.bot +``` + +Команды: `/new [N]`, `/abuse [тег]`, `/find <слово>`, `/src <слово>`, `/run <категория>`, `/add `, `/stats`. Бот работает на той же базе и том же реестре, что панель; отвечает только id из `OMNI_BOT_ALLOW`. + diff --git a/omni/abuse.py b/omni/abuse.py new file mode 100644 index 0000000..4a9dc37 --- /dev/null +++ b/omni/abuse.py @@ -0,0 +1,149 @@ +"""Разбор абузов: промо, триалы, ключи, сливы, софт, реселл. + +Работает поверх уже собранной базы (items) и поверх новых абуз-источников: +каждая запись получает набор тегов и вес, панель показывает только то, +что реально пахнет абузом, а не весь поток. +""" +from __future__ import annotations + +import re +import sqlite3 + +from .config import DB + +# тег -> (человеческое имя, вес, регулярки) +RULES = [ + ("keys", "ключи", 40, [ + r"sk-[A-Za-z0-9_\-]{20,}", r"gh[pousr]_[A-Za-z0-9]{20,}", r"xox[baprs]-[A-Za-z0-9\-]{10,}", + r"(разда|выда|дарю|бесплатн|free|giveaway)[^.]{0,40}(ключ|key|токен|token)", + r"(ключ|key|токен)[^.]{0,25}(бесплатн|free|дарю|раздач|giveaway)", + ]), + ("promo", "промо", 26, [ + r"промокод", r"promo\s?code", r"промо[\s-]?код", r"скидк[аи]\s+\d{1,3}", r"\d{1,3}\s?%\s?(off|скидк)", + r"redeem\s?(code|код)", r"invite\s?code", r"реф(еральн|код)", r"бонус\s+на\s+сч[её]т", r"кэшб[еэ]к|cashback", + ]), + ("trial", "триалы", 24, [ + r"trial", r"триал", r"free\s+credits?", r"бесплатн\w*\s+(кредит|токен|баланс|доступ|период)", + r"free\s+tier", r"\$\d+\s+free", r"试用", r"免费", r"пробн\w+\s+период", r"без\s+депозита", r"no\s+deposit", + ]), + ("leak", "сливы", 26, [ + r"слив(?!\s*(сок|ки|очн|тест|трафик))", r"leaked\s+(database|db|dump|combo)", r"db\s?dump", + r"combo[\s-]?list", r"stealer\s?log", r"утечк\w*\s+баз", r"база\s+(пассов|почт|данных)", + r"пароли\s+от\s+\w+\.(com|ru|net)", + ]), + ("tool", "софт", 18, [ + r"софт", r"кряк", r"crack(ed)?", r"bypass", r"обход\s+(блокиров|бан|защит)", + r"авторег", r"autoreg", r"panel", r"панел[ьи]\s", r"рассылк", r"sender", r"скрипт", + ]), + ("resale", "продажа", 14, [ + r"продаю", r"продам", r"куплю\s+аккаунт", r"for\s+sale", r"цена\s*[:$]", r"\$\d+\s?/\s?(мес|mo|1k|1000)", + r"реселлер", r"resell", + ]), + ("news", "новости", 6, [ + r"анонс", r"выпустил", r"released?", r"launch", r"обновлени", r"release notes", + r"новая модель", r"new model", + ]), +] + +COMPILED = [(tag, label, w, [re.compile(p, re.I) for p in pats]) for tag, label, w, pats in RULES] +TAG_LABEL = {t: l for t, l, _, _ in RULES} +STRONG = {"keys", "promo", "trial", "resale"} +# категории-источники, которые вообще имеют смысл для абузов +ABUSE_CATS = {"abuse", "forum", "community", "company_cn"} +# «статьи»: слова про ключи и утечки там — текст, а не раздача +NOISE = re.compile(r"(blog|devto|medium|substack|newsletter|docs?|wiki)", re.I) + + +def _source_meta(): + try: + from . import sources as _s + return {x.name: x.category for x in _s.all_sources()} + except Exception: + return {} + + +SRC_CAT = _source_meta() + + +def classify(title: str, text: str, tags: str = ""): + blob = (" ".join([title or "", text or "", tags or ""]))[:4000] + hits = [] + score = 0 + for tag, label, w, pats in COMPILED: + n = 0 + for p in pats: + m = p.search(blob) + if m: + n += 1 + if n: + hits.append(tag) + score += w * min(n, 3) + return score, hits + + +def _qualifies(source, title, text, tags): + sc, hits = classify(title, text, tags) + if not hits: + return 0, [] + strong = bool(set(hits) & STRONG) + cat = SRC_CAT.get(source) + blob = (title or "") + " " + (text or "") + raw_key = bool(re.search(r"(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob)) + if NOISE.search(source or "") and not raw_key: + return 0, [] + if cat is None: + # старые записи хранят категорию вместо имени источника + if not raw_key and not str(source or "").startswith(("ab_", "tg_", "reddit_", "discourse_", "xenforo")): + return 0, [] + elif cat not in ABUSE_CATS and not raw_key: + return 0, [] + if strong and sc >= 24: + return sc, hits + if len(hits) >= 2 and sc >= 40: + return sc, hits + return 0, [] + + +def _rows(limit=400, tag=None, q=None, since_days=None): + sql = "SELECT id, source, url, title, text, published, tags, score FROM items" + where, args = [], [] + if q: + where.append("(title LIKE ? OR text LIKE ? OR source LIKE ?)") + args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q] + if since_days: + where.append("published >= date('now', ?)") + args.append("-%d day" % int(since_days)) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY id DESC LIMIT ?" + args.append(int(limit) * 6) + con = sqlite3.connect(str(DB), timeout=30) + try: + cur = con.execute(sql, args) + out = [] + for r in cur: + sc, hits = _qualifies(r[1], r[3], r[4], r[6] or "") + if not sc: + continue + if tag and tag not in hits: + continue + out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3], + "text": (r[4] or "")[:600], "published": r[5], + "tags": hits, "abuse_score": sc, "rank": r[7]}) + if len(out) >= limit: + break + return out + finally: + con.close() + + +def scan(limit=120, tag=None, q=None, since_days=None, window=6000): + """limit — сколько отдать, window — сколько последних записей просканировать.""" + items = _rows(limit=window, tag=tag, q=q, since_days=since_days) + items.sort(key=lambda x: (-x["abuse_score"], -(x["id"] or 0))) + counts = {} + for it in items: + for t in it["tags"]: + counts[t] = counts.get(t, 0) + 1 + return {"items": items[:limit], "total": len(items), "tags": counts, + "labels": TAG_LABEL} \ No newline at end of file diff --git a/omni/abuse_sources.py b/omni/abuse_sources.py new file mode 100644 index 0000000..df30bc9 --- /dev/null +++ b/omni/abuse_sources.py @@ -0,0 +1,237 @@ +"""Абузы: форумы, каналы и площадки с промо/сливами/триалами/ключами. + +Сгенерировано из реестров парсера абузов; адаптер выведен из движка сайта, +каждая строка проходит реальную проверку fetch перед включением. +""" + + +def A(name, category, kind, url, adapter, tags=(), limit=20): + return {"name": name, "category": category, "kind": kind, "url": url, + "adapter": adapter, "tags": tuple(tags), "limit": limit, "detail": False, "extra": {}} + + +ABUSE = [ + # ---- abuse (222) ---- + A("ab_lolz_live", "abuse", "html", "https://lolz.live/tags/rassylka/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_crackingx_com", "abuse", "html", "https://crackingx.com/forums/5/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_xreactor_org", "abuse", "html", "https://xreactor.org/forums/combolists.152/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_crdpro_cc", "abuse", "html", "https://crdpro.cc/tags/tools/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_trafficultras_com", "abuse", "html", "https://trafficultras.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_blackhatworld_com", "abuse", "html", "https://www.blackhatworld.com/seo/up-to-date-list-of-specialized-software-for-telegram.1642970/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_addset_ru", "abuse", "html", "https://addset.ru/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_fb_killa_pro", "abuse", "html", "https://fb-killa.pro/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forum_x_com", "abuse", "html", "https://forum-x.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_afflift_com", "abuse", "html", "https://afflift.com/f/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cracked_io", "abuse", "html", "https://cracked.io/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_altenens_is", "abuse", "html", "https://altenens.is/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cardvilla_cc", "abuse", "html", "https://cardvilla.cc/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cybercarders_eu", "abuse", "html", "https://cybercarders.eu/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_shadowcarders_com", "abuse", "html", "https://shadowcarders.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_carders_biz", "abuse", "html", "https://carders.biz/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_ascarding_net", "abuse", "html", "https://ascarding.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_xakerplus_com", "abuse", "html", "https://xakerplus.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_lozerix_com", "abuse", "html", "https://lozerix.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_zhyk_org", "abuse", "html", "https://zhyk.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_zenno_club", "abuse", "html", "https://zenno.club/discussion/tags/arbitrazh-trafika/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_partnerkin_com", "abuse", "html", "https://partnerkin.com/blog/articles", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_afftimes_com", "abuse", "html", "https://afftimes.com/articles/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpa_rip", "abuse", "html", "https://cpa.rip/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpa_live", "abuse", "html", "https://cpa.live/articles/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_gdetraffic_com", "abuse", "html", "https://gdetraffic.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_affy_group", "abuse", "html", "https://affy.group/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_conversion_im", "abuse", "html", "https://conversion.im/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_warriorforum_com", "abuse", "html", "https://www.warriorforum.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forums_digitalpoint_com", "abuse", "html", "https://forums.digitalpoint.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpaelites_com", "abuse", "html", "https://www.cpaelites.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_hackforums_net", "abuse", "html", "https://hackforums.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cracking_org", "abuse", "html", "https://cracking.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_crackingsoul_com", "abuse", "html", "https://crackingsoul.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tele_sender_com", "abuse", "html", "https://tele-sender.com/blog/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_crmchat_ai", "abuse", "html", "https://crmchat.ai/blog/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_teleboost_app", "abuse", "html", "https://teleboost.app/blog/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_reddit_com", "abuse", "html", "https://www.reddit.com/r/TelegramBots/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_lzt_market", "abuse", "html", "https://lzt.market/telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_sourceforge_net", "abuse", "html", "https://sourceforge.net/directory/telegram-bots/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_t_me", "abuse", "html", "https://t.me/s/Abuz_ai_soft", "tg_web", ('abuse', 'abuse'), limit=20), + A("ab_habr_com", "abuse", "html", "https://habr.com/ru/search/?q=telegram+парсинг+софт&target_type=posts&order=date", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_teletype_in", "abuse", "html", "https://teletype.in/search?query=telegram+софт", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_youtube_com", "abuse", "html", "https://www.youtube.com/results?search_query=telegram+marketing+software+2026&sp=CAISBAgCEAE%253D", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_medium_com", "abuse", "html", "https://medium.com/search?q=telegram+bot+automation+2026", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tgstat_ru", "abuse", "html", "https://tgstat.ru/ratings/chats/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_carder_su", "abuse", "html", "https://carder.su/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_procarders_com", "abuse", "html", "https://procarders.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_darkpro_net", "abuse", "html", "https://darkpro.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tuts4you_com", "abuse", "html", "https://tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_forum_tuts4you_com", "abuse", "html", "https://forum.tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_forum_exetools_com", "abuse", "html", "https://forum.exetools.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tlgrm_eu", "abuse", "html", "https://tlgrm.eu/channels", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telemetr_me", "abuse", "html", "https://telemetr.me/channels", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpamafia_pro", "abuse", "html", "https://cpamafia.pro/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpamafia_top", "abuse", "html", "https://cpamafia.top/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_m_vk_com", "abuse", "html", "https://m.vk.com/search?c%5Bq%5D=продвижение+telegram+софт&c%5Bsection%5D=auto", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forobeta_com", "abuse", "html", "https://forobeta.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forexfactory_com", "abuse", "html", "https://www.forexfactory.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_gpwa_org", "abuse", "html", "https://www.gpwa.org/forum/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_casinomeister_com", "abuse", "html", "https://www.casinomeister.com/forums/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forums_babypips_com", "abuse", "html", "https://forums.babypips.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_github_com", "abuse", "html", "https://github.com/ofmtools/awesome-ofm-tools", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_searchengines_guru", "abuse", "html", "https://searchengines.guru/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_hardforum_com", "abuse", "html", "https://hardforum.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_bitcointalk_org", "abuse", "html", "https://bitcointalk.org/index.php?board=238.0", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_sendpulse_com", "abuse", "html", "https://sendpulse.com/blog/telegram-marketing-tools", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_softwaresuggest_com", "abuse", "html", "https://www.softwaresuggest.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tacticaldatamanager_com", "abuse", "html", "https://tacticaldatamanager.com/blog", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_statiko_io", "abuse", "html", "https://statiko.io/blog/telegram-anti-spam-bot", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tghero_pro", "abuse", "html", "https://tghero.pro/telegram-marketing-tools/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_enreach_ai", "abuse", "html", "https://www.enreach.ai/insights/best-telegram-outreach-tools", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_nulledbb_com", "abuse", "html", "https://nulledbb.com/syndication.php?limit=50", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cracking_forums_com", "abuse", "html", "http://www.cracking-forums.com/syndication.php?limit=50", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_eleaks_to", "abuse", "html", "https://eleaks.to/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_crackiansleaks_com", "abuse", "html", "https://crackiansleaks.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_niflheim_world", "abuse", "html", "https://niflheim.world/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_nodeloc_com", "abuse", "json", "https://www.nodeloc.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20), + A("ab_v2ex_com", "abuse", "json", "https://www.v2ex.com/api/topics/hot.json", "generic_json", ('abuse', 'abuse'), limit=20), + A("ab_hostloc_com", "abuse", "rss", "https://hostloc.com/forum.php?mod=rss", "rss", ('abuse', 'abuse'), limit=20), + A("ab_findaicredits_com", "abuse", "html", "https://www.findaicredits.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_bhf_io", "abuse", "html", "https://bhf.io/en-gb", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_leak_sx", "abuse", "html", "https://leak.sx/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forum_hashkiller_io", "abuse", "html", "https://forum.hashkiller.io/index.php", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_linux_do", "abuse", "json", "https://linux.do/latest.json", "discourse", ('abuse', 'abuse'), limit=20), + A("ab_xss_is", "abuse", "html", "https://xss.is/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_qqshill_com", "abuse", "html", "https://qqshill.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_mavibot_ai", "abuse", "html", "https://mavibot.ai/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_kenzabyte_com", "abuse", "html", "https://www.kenzabyte.com/telegram-auto-account-creator-version-history/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_ltesocks_io", "abuse", "html", "https://ltesocks.io/blog/specific-examples-of-mobile-proxies-en-en/top-programs-for-automatic-telegram-account-registration/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_dicloak_com", "abuse", "html", "https://dicloak.com/blog-detail/the-ultimate-guide-to-using-a-telegram-bulk-account-creator-in-2026", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_multilogin_com", "abuse", "html", "https://multilogin.com/multiple-accounting/create-multiple-telegram-accounts/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_quackr_io", "abuse", "html", "https://quackr.io/blog/create-unlimited-telegram-accounts", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_genfarmer_com", "abuse", "html", "https://genfarmer.com/tool-for-mass-telegram-registration-and-automatic-telegram-account-nurturing-en/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_like_tg", "abuse", "html", "https://www.like.tg/blog/telegram-bulk-registration-tool-best-two-thousand-twenty-six-stable-solution-captcha-guide", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_dexpose_io", "abuse", "html", "https://www.dexpose.io/dark-web-forums/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cybelangel_com", "abuse", "html", "https://cybelangel.com/blog/dark-web-telegram-channels-attackers-use-most-2026/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxyscrape_com", "abuse", "html", "https://proxyscrape.com/online-proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_localtonet_com", "abuse", "html", "https://localtonet.com/tools/proxy-check", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxy_seller_com", "abuse", "html", "https://proxy-seller.com/tools/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_gologin_com", "abuse", "html", "https://gologin.com/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxyway_com", "abuse", "html", "https://proxyway.com/tools/free-online-proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_chromewebstore_google_com", "abuse", "html", "https://chromewebstore.google.com/detail/tg-member-extractor/gphncajfmfojhafagcemjjacbekahdoh", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_apify_com", "abuse", "html", "https://apify.com/agentx/telegram-member-scraper", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_energent_ai", "abuse", "html", "https://www.energent.ai/use-cases/en/telegram-scraper", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_thunderbit_com", "abuse", "html", "https://thunderbit.com/template/telegram-scraper", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_axiom_ai", "abuse", "html", "https://axiom.ai/scrape/telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_flozic_ai", "abuse", "html", "https://www.flozic.ai/agents/conversational/telegram-bot", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_manybot_io", "abuse", "html", "https://manybot.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_botmother_com", "abuse", "html", "https://botmother.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_chatloom_app", "abuse", "html", "https://chatloom.app/en/channels/telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_chatbotbuilder_net", "abuse", "html", "https://chatbotbuilder.net/telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_saascounter_com", "abuse", "html", "https://www.saascounter.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_schedulala_com", "abuse", "html", "https://schedulala.com/blog/telegram-marketing-guide", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_technologycounter_com", "abuse", "html", "https://technologycounter.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_omisoft_net", "abuse", "html", "https://omisoft.net/blog/telegram-mini-apps-for-business/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_adsly_pro", "abuse", "html", "https://adsly.pro/guides/managed-service-vs-saas-cost/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_optimum_web_com", "abuse", "html", "https://www.optimum-web.com/blog/telegram-ai-bot-for-business-2026/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegramsender_com", "abuse", "html", "https://telegramsender.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_xreacher_com", "abuse", "html", "https://xreacher.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_zapier_com", "abuse", "html", "https://zapier.com/apps/outreach/integrations/telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_n8n_io", "abuse", "html", "https://n8n.io/workflows/13732-run-email-outreach-campaigns-from-telegram-with-gmail-and-google-sheets/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_albato_com", "abuse", "html", "https://albato.com/connect/outreach-with-telegramprivate", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_entergram_com", "abuse", "html", "https://www.entergram.com/blog/telegram-lead-management-complete-guide", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_breakcold_com", "abuse", "html", "https://www.breakcold.com/blog/crm-integration-with-telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_kommo_com", "abuse", "html", "https://www.kommo.com/integrations/telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_gramsales_com", "abuse", "html", "https://gramsales.com/blog/best-crm-tools-for-telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_synchubai_com", "abuse", "html", "https://synchubai.com/blog/telegram-messaging-crm/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_messaggio_com", "abuse", "html", "https://messaggio.com/messaging-in-telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_vc_ru", "abuse", "html", "https://vc.ru/id5905098/2879981-deskgram-2-moschnyj-soft-dlja-telegram-spama", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_unisender_com", "abuse", "html", "https://www.unisender.com/ru/blog/servisy-dlya-rassylki-v-telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_socradar_io", "abuse", "html", "https://socradar.io/blog/top-10-deep-web-and-dark-web-forums/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_nordlayer_com", "abuse", "html", "https://nordlayer.com/blog/top-dark-web-telegram-groups-and-channels/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_play_google_com", "abuse", "html", "https://play.google.com/store/apps/details?id=com.boardgames.bd.mtproto", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxyempire_io", "abuse", "html", "https://proxyempire.io/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxybros_com", "abuse", "html", "https://proxybros.com/proxies/proxy-for-telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_adfasten_com", "abuse", "html", "https://www.adfasten.com/telegram-marketing", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_chatimize_com", "abuse", "html", "https://chatimize.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tg_on_com", "abuse", "html", "https://tg-on.com/articles/top-programm-rassylka-telegram.html", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_botract_com", "abuse", "html", "https://www.botract.com/blog/how-to-choose-telegram-bot-builder", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_respond_io", "abuse", "html", "https://respond.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_sendgecko_io", "abuse", "html", "https://sendgecko.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_community_thermaltake_com", "abuse", "html", "https://community.thermaltake.com/index.php?/topic/692572-the-ultimate-2026-guide-to-mass-telegram-account-creation-zero-bans/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_discuss_flarum_org", "abuse", "html", "https://discuss.flarum.org/d/39670-what-forum-spam-looks-like-in-2026-summer-edition", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_texapi_net", "abuse", "html", "https://texapi.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pixelscan_net", "abuse", "html", "https://pixelscan.net/blog/best-telegram-batch-registration-software-in-2026/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pipedream_com", "abuse", "html", "https://pipedream.com/docs", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_p1sms_kz", "abuse", "html", "https://p1sms.kz/telegram-rassylki", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_nodemaven_com", "abuse", "html", "https://nodemaven.com/tools/proxy-bandwidth-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_infatica_io", "abuse", "html", "https://infatica.io/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_flowxo_com", "abuse", "html", "https://flowxo.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pas7_com_ua", "abuse", "html", "https://pas7.com.ua/blog/en/telegram-bot-development-cost-2026", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telelog_org", "abuse", "html", "https://telelog.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_help_nuelink_com", "abuse", "html", "https://help.nuelink.com/article/228-best-telegram-scheduling-tools", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_wooxy_com", "abuse", "html", "https://wooxy.com/telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_chatbots_studio", "abuse", "html", "https://chatbots.studio/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_app_flow_ai", "abuse", "html", "https://app.flow.ai", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_kelacyber_com", "abuse", "html", "https://www.kelacyber.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_infobip_com", "abuse", "html", "https://www.infobip.com/blog/best-marketing-automation-platforms", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_atompark_com", "abuse", "html", "https://www.atompark.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_foresiet_com", "abuse", "html", "https://foresiet.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_vovsoft_com", "abuse", "html", "https://vovsoft.com/software/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pocodot_ai", "abuse", "html", "https://pocodot.ai/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_richads_com", "abuse", "html", "https://richads.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxyline_net", "abuse", "html", "https://proxyline.net/proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pypi_org", "abuse", "html", "https://pypi.org/project/TGSessionsConverter/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_opentele_readthedocs_io", "abuse", "html", "https://opentele.readthedocs.io/en/latest/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_growity_ai", "abuse", "html", "https://growity.ai/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_pantheon_run", "abuse", "html", "https://pantheon.run/learn/best-telegram-crm", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_platform_openai_com", "abuse", "html", "https://platform.openai.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_dev_to", "abuse", "html", "https://dev.to/scofieldidehen/if-you-use-telegram-in-2026-read-this-now-4770", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_codester_com", "abuse", "html", "https://www.codester.com/items/54880/telegram-marketing-tools-bot", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_greasyfork_org", "abuse", "html", "https://greasyfork.org/en/scripts/538176-telegram-scraper-menu-commands-v2-3-3-human-like-defaults", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_medevel_com", "abuse", "html", "https://medevel.com/telegram-scraper-20/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_blog_checkpoint_com", "abuse", "html", "https://blog.checkpoint.com/research/telegrams-crackdown-in-2026-and-why-cyber-criminals-are-still-winning/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_make_com", "abuse", "html", "https://www.make.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_gopulsar_io", "abuse", "html", "https://gopulsar.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tgproxy_io", "abuse", "html", "https://tgproxy.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_texsender_net", "abuse", "html", "https://texsender.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_communicat_o_com", "abuse", "html", "https://www.communicat-o.com/crm-integration-for-telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegramscraper_shop", "abuse", "html", "https://telegramscraper.shop/blog/best-telegram-scraper-2026", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxyhorizon_com", "abuse", "html", "https://www.proxyhorizon.com/blog/proxies-for-telegram", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_ctiacademy_io", "abuse", "html", "https://ctiacademy.io/blog/telegram-cybercrime-marketplace", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tgsender_wadesk_io", "abuse", "html", "https://tgsender.wadesk.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_coronium_io", "abuse", "html", "https://www.coronium.io/partners/software/telegram-prime-review", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_stackoverflow_com", "abuse", "html", "https://stackoverflow.com/questions/61657596/telethon-first-steps-in-python", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tgninja_com", "abuse", "html", "https://tgninja.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_malware_news", "abuse", "html", "https://malware.news/t/top-dark-web-telegram-groups-channels-2026/104664", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telega_io", "abuse", "html", "https://telega.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_fiverr_com", "abuse", "html", "https://www.fiverr.com/gigs/telegram-invite", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_quora_com", "abuse", "html", "https://www.quora.com/What-bulk-messaging-software-is-best-for-Telegram-Line-and-WhatsApp", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_bolster_ai", "abuse", "html", "https://bolster.ai/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_capterra_com", "abuse", "html", "https://www.capterra.com/p/180347/Telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_manychat_com", "abuse", "html", "https://manychat.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_outpost24_com", "abuse", "html", "https://outpost24.com/blog/carding-ecosystem-fall-financial-cybercrime/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegramvpn_org", "abuse", "html", "https://telegramvpn.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_iricrm_com", "abuse", "html", "https://iricrm.com/telegram-crm/?lang=en", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_deepstrike_io", "abuse", "html", "https://deepstrike.io/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_stealthmole_com", "abuse", "html", "https://www.stealthmole.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_f6s_com", "abuse", "html", "https://www.f6s.com/software/category/no-code-telegram-bot-builder", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_exploit_in", "abuse", "html", "https://exploit.in/challenge/?return=/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_checkpoint_com", "abuse", "html", "https://www.checkpoint.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_blog_invitemember_com", "abuse", "html", "https://blog.invitemember.com/5-essential-telegram-bots-for-creators/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_tracexlabs_com", "abuse", "html", "https://tracexlabs.com/knowledge-base/top-10-deep-web-and-dark-web-forums.html", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_proxies_sx", "abuse", "html", "https://www.proxies.sx/partners/automation-platforms/telegram-expert", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_checkproxy_org", "abuse", "html", "https://checkproxy.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_automatedclientacquisition_com", "abuse", "html", "https://www.automatedclientacquisition.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_stormycloud_org", "abuse", "html", "https://stormycloud.org/mtproto/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_bot_market_net", "abuse", "html", "https://bot-market.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telebotbuilder_com", "abuse", "html", "https://telebotbuilder.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cavyro_com", "abuse", "html", "https://cavyro.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_positechglobal_com", "abuse", "html", "https://positechglobal.com/en/research/analytics/cybercriminal-market-in-telegram/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_trypromptlab_com", "abuse", "html", "https://trypromptlab.com/blog/best-crm-with-telegram-integration", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_flowcastle_ai", "abuse", "html", "https://flowcastle.ai/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegramsoftware_pro", "abuse", "html", "https://telegramsoftware.pro/blog/telegram-session-to-tdata-converter/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_darkbase_cc", "abuse", "html", "https://darkbase.cc/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_nocturne_cards", "abuse", "html", "https://nocturne.cards/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_ads_telegram_org", "abuse", "html", "https://ads.telegram.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_teleform_io", "abuse", "html", "https://www.teleform.io/", "html_listing", ('abuse', 'abuse'), limit=20), +] diff --git a/omni/bot.py b/omni/bot.py new file mode 100644 index 0000000..4cdf2b3 --- /dev/null +++ b/omni/bot.py @@ -0,0 +1,217 @@ +"""Telegram-бот поверх парсера: лента, абузы, источники, прогон, автодобавление. + +Запуск: OMNI_BOT_TOKEN=... python -m omni.bot +Доступ: OMNI_BOT_OWNERS=7448683285,123 (если пусто — только первый написавший) + +Бот не постит никуда сам: только отвечает в личный чат тем, кто в списке. +Команды: + /new [n] последние записи + /abuse [тег] абузы: promo trial keys leak tool resale + /find <текст> поиск по базе + /src <текст> поиск источника + /add добавить источник (сначала реальная проверка фида) + /run <категория> прогнать категорию + /auto [n] автодобавление новых фидов + /stats цифры +""" +from __future__ import annotations + +import html +import json +import os +import re +import time +import urllib.parse +import urllib.request + +from . import abuse as abuse_mod +from . import discover as discover_mod +from . import feeds, net, source_state, sources, store + +API = "https://api.telegram.org/bot%s/" % os.environ.get("OMNI_BOT_TOKEN", "") +TOKEN = os.environ.get("OMNI_BOT_TOKEN", "") +OWNERS = {int(x) for x in re.findall(r"\d{5,}", os.environ.get("OMNI_BOT_OWNERS", ""))} +MAX_LEN = 3600 + + +def _api(method, **params): + if not TOKEN: + raise SystemExit("OMNI_BOT_TOKEN не задан") + url = API + method + data = urllib.parse.urlencode(params).encode() + r = net.get(url, headers={"Content-Type": "application/x-www-form-urlencoded"}) if False else None + req = urllib.request.Request(url, data=data) + with urllib.request.urlopen(req, timeout=40) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def _send(chat_id, text, preview=True): + text = text[:MAX_LEN] + return _api("sendMessage", chat_id=chat_id, text=text, parse_mode="HTML", + disable_web_page_preview="false" if preview else "true") + + +def _esc(s): + return html.escape(str(s or ""), quote=False) + + +def fmt_item(it): + title = _esc((it.get("title") or "")[:150]) + src = _esc(it.get("source") or "") + url = it.get("url") or "" + line = "%s\n%s" % (title, src) + if url: + line += ' · открыть' % _esc(url) + return line + + +def cmd_start(chat_id, arg): + _send(chat_id, "omni — парсер-радар\n\n" + "/new 5 — последние записи\n" + "/abuse promo — абузы по тегу\n" + "/find claude — поиск\n" + "/src reddit — источник\n" + "/add <url> — подключить фид\n" + "/run abuse — прогнать категорию\n" + "/auto 10 — автодобавление\n" + "/stats — цифры") + + +def cmd_new(chat_id, arg): + n = int(arg) if arg.strip().isdigit() else 8 + rows = store.browse2(limit=min(n, 20), sort="fresh") + items = rows.get("items", rows) if isinstance(rows, dict) else rows + if not items: + return _send(chat_id, "пусто") + _send(chat_id, "последние %d:\n\n" % len(items) + "\n\n".join(fmt_item(i) for i in items)) + + +def cmd_abuse(chat_id, arg): + tag = arg.strip().split()[0] if arg.strip() else None + d = abuse_mod.scan(limit=10, tag=tag) + if not d["items"]: + return _send(chat_id, "абузов не видно") + head = "абузы · %s · %d\n\n" % ("все" if not tag else tag, d["total"]) + body = [] + for it in d["items"]: + body.append("%s\n%s · %s" % ( + _esc(it["title"][:120]), _esc(it["source"]), _esc(",".join(it["tags"])))) + _send(chat_id, head + "\n\n".join(body)) + + +def cmd_find(chat_id, arg): + q = arg.strip() + if not q: + return _send(chat_id, "что искать?") + rows = store.search(q, limit=8) + items = rows.get("items", rows) if isinstance(rows, dict) else rows + if not items: + return _send(chat_id, "нет совпадений") + _send(chat_id, "поиск: %s\n\n" % _esc(q) + "\n\n".join(fmt_item(i) for i in items)) + + +def cmd_src(chat_id, arg): + q = arg.strip().lower() + all_s = sources.all_sources() + hits = [s for s in all_s if q and (q in s.name.lower() or q in s.url.lower() or q in s.category.lower())] + if not hits: + return _send(chat_id, "нет такого источника") + out = ["%s · %s · %s" % (_esc(s.name), _esc(s.category), _esc(s.adapter)) for s in hits[:12]] + _send(chat_id, "источники (%d):\n\n" % len(hits) + "\n".join(out)) + + +def cmd_add(chat_id, arg): + url = arg.strip() + if not url.startswith("http"): + return _send(chat_id, "нужен url фида") + ok, n, ad = discover_mod.probe(url, "community", limit=6) + if not ok: + return _send(chat_id, "фид пустой или недоступен: %s" % _esc(ad)) + name = "manual_" + re.sub(r"[^a-z0-9]+", "_", urllib.parse.urlsplit(url).netloc.lower())[:32] + source_state.add_custom(name, "community", url, kind="rss" if ad == "rss" else "html", + adapter=ad, tags=["manual"], limit=15) + _send(chat_id, "добавлен %s · %s · %d записей" % (_esc(name), _esc(ad), n)) + + +def cmd_run(chat_id, arg): + cat = arg.strip() or "abuse" + srcs = [s for s in sources.all_sources() if s.category == cat] + if not srcs: + return _send(chat_id, "нет категории %s" % _esc(cat)) + _send(chat_id, "прогон %s (%d источников)…" % (_esc(cat), len(srcs))) + got = new = 0 + for s in srcs[:80]: + try: + items = feeds.fetch(s, 10) + except Exception: + items = [] + got += len(items) + if items: + new += store.add(items) + _send(chat_id, "готово: собрано %d, новых %d" % (got, new)) + + +def cmd_auto(chat_id, arg): + n = int(arg) if arg.strip().isdigit() else 15 + res = discover_mod.discover(limit_hosts=30, workers=8, max_add=min(n, 30)) + _send(chat_id, "автодобавление: кандидатов %d, живых %d\n%s" % ( + res["found"], res["alive"], _esc(", ".join(res["added"][:10])))) + + +def cmd_stats(chat_id, arg): + s = store.stats() + a = sources.all_sources() + alive = source_state.alive_count() if hasattr(source_state, "alive_count") else "?" + _send(chat_id, "omni\nзаписей %s · источников %d · живых %s\nкатегорий %d · прогонов %s · с ценой %s" % ( + s.get("items"), len(a), alive, len(sources.CATEGORIES), s.get("runs"), s.get("priced"))) + + +CMDS = {"/start": cmd_start, "/help": cmd_start, "/new": cmd_new, "/abuse": cmd_abuse, + "/find": cmd_find, "/src": cmd_src, "/add": cmd_add, "/run": cmd_run, + "/auto": cmd_auto, "/stats": cmd_stats} + + +def handle(update): + msg = update.get("message") or update.get("edited_message") or {} + chat = msg.get("chat") or {} + chat_id = chat.get("id") + text = (msg.get("text") or "").strip() + uid = (msg.get("from") or {}).get("id") + if not chat_id or not text: + return + if OWNERS and uid not in OWNERS: + return + if not OWNERS: + OWNERS.add(uid) # первый написавший становится владельцем до рестарта + parts = text.split(None, 1) + cmd = parts[0].lower().split("@")[0] + arg = parts[1] if len(parts) > 1 else "" + fn = CMDS.get(cmd) + if not fn: + return _send(chat_id, "не знаю такую команду. /help") + try: + fn(chat_id, arg) + except Exception as e: + _send(chat_id, "ошибка: %s" % _esc(type(e).__name__ + ": " + str(e)[:160])) + + +def run_forever(): + if not TOKEN: + raise SystemExit("нужен OMNI_BOT_TOKEN") + offset = 0 + print("bot up", flush=True) + while True: + try: + r = _api("getUpdates", offset=offset, timeout=30) + except Exception as e: + print("poll fail", e, flush=True) + time.sleep(5) + continue + for up in r.get("result", []): + offset = up["update_id"] + 1 + handle(up) + time.sleep(0.3) + + +if __name__ == "__main__": + run_forever() \ No newline at end of file diff --git a/omni/discover.py b/omni/discover.py new file mode 100644 index 0000000..7607497 --- /dev/null +++ b/omni/discover.py @@ -0,0 +1,164 @@ +"""Автодобавление источников: находит фиды по домашним страницам и каталогам. + +Логика: берём затравки (хосты уже подключённых источников + известные каталоги +сообществ), вытаскиваем ссылки на фиды (link rel=alternate, /feed, /rss, +/latest.json, подфорумы), прогоняем каждый кандидат реальным адаптером и +добавляем только те, что отдали записи. Всё добавленное помечается auto и +попадает в панель как обычный источник. +""" +from __future__ import annotations + +import json +import re +import time +import urllib.parse +from concurrent.futures import ThreadPoolExecutor + +from bs4 import BeautifulSoup + +from . import feeds, net, source_state, sources + +FEED_PATHS = ("/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", + "/latest.json", "/?feed=rss2", "/feed/", "/blog/feed") +SEED_CATALOGS = [ + # каталоги сообществ и подфорумов, откуда удобно цеплять новые фиды + "https://www.reddit.com/r/LocalLLaMA/about.json", + "https://news.ycombinator.com/", + "https://lobste.rs/", +] +UA = getattr(feeds, "UA", {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}) + + +def _hosts(): + seen = {} + for s in sources.all_sources(): + try: + h = urllib.parse.urlsplit(s.url).netloc.lower() + except Exception: + continue + if not h or h in seen: + continue + seen[h] = s.category + return seen + + +def _feed_links(html, base): + out = [] + soup = BeautifulSoup(html, "lxml") + for ln in soup.find_all("link", href=True): + t = (ln.get("type") or "").lower() + rel = " ".join(ln.get("rel") or []).lower() + if "alternate" in rel and ("rss" in t or "atom" in t or "json" in t): + out.append(urllib.parse.urljoin(base, ln["href"])) + for a in soup.find_all("a", href=True): + href = str(a["href"]) + if re.search(r"(/feed|/rss|\.rss|atom\.xml|/latest\.json)$", href): + out.append(urllib.parse.urljoin(base, href)) + return out + + +def candidates(limit_hosts=60, extra_seeds=()): + hosts = _hosts() + seeds = [] + for h, cat in list(hosts.items())[:limit_hosts]: + seeds.append(("https://%s/" % h, cat)) + for u in list(extra_seeds) + SEED_CATALOGS: + seeds.append((u, "auto")) + return seeds + + +def probe(url, category="auto", limit=6): + """Проверка одного URL реальным адаптером. Возвращает (ok, n, adapter).""" + low = url.lower() + if "t.me/s/" in low: + ad, kind = "tg_web", "html" + elif low.endswith(".json") or "/latest.json" in low: + ad, kind = ("discourse", "json") if "/latest.json" in low else ("generic_json", "json") + elif any(x in low for x in (".rss", "/rss", "/feed", "atom.xml", "feed.xml", "index.xml")): + ad, kind = "rss", "rss" + else: + ad, kind = "html_listing", "html" + src = sources.Source(name="probe", category=category, kind=kind, url=url, + adapter=ad, tags=("auto",), limit=limit, detail=False, extra={}) + fn = feeds.ADAPTERS.get(ad) + if not fn: + return False, 0, ad + try: + items = fn(src, limit) or [] + except Exception: + return False, 0, ad + return len(items) > 0, len(items), ad + + +def discover(limit_hosts=40, workers=8, max_add=40, dry=False): + seeds = candidates(limit_hosts=limit_hosts) + found = [] + def scan(seed): + base, cat = seed + try: + r = net.get(base, headers=UA) + if r.status != 200: + return [] + except Exception: + return [] + links = _feed_links(r.text[:400000], base) + dom = urllib.parse.urlsplit(base) + for p in FEED_PATHS: + links.append("%s://%s%s" % (dom.scheme, dom.netloc, p)) + # подфорумы Discourse/Reddit + if "reddit.com" in base: + for sub in ("LocalLLaMA", "MachineLearning", "mcp", "selfhosted"): + links.append("https://www.reddit.com/r/%s/.rss" % sub) + return [(u, cat) for u in dict.fromkeys(links)] + + with ThreadPoolExecutor(max_workers=workers) as ex: + for res in ex.map(scan, seeds): + found.extend(res) + print("кандидатов найдено:", len(found), flush=True) + + known = {s.url for s in sources.all_sources()} + found = [(u, c) for u, c in found if u not in known] + print("после дедупа:", len(found), flush=True) + + def check(pair): + url, cat = pair + ok, n, ad = probe(url, cat) + return url, cat, ok, n, ad + + # только машинные фиды: html-листинги на автодобавлении дают мусор + best = {} + with ThreadPoolExecutor(max_workers=workers) as ex: + for url, cat, ok, n, ad in ex.map(check, found): + if not ok or ad == "html_listing" or n < 3: + continue + dom = urllib.parse.urlsplit(url).netloc.lower() + key = (dom, ad) + if key not in best or n > best[key][2]: + best[key] = (url, cat, n, ad) + + added = [] + for (dom, ad), (url, cat, n, _ad) in sorted(best.items(), key=lambda kv: -kv[1][2]): + name = "auto_" + re.sub(r"[^a-z0-9]+", "_", dom).strip("_")[:34] + if any(a["name"] == name for a in added): + continue + rec = {"name": name, "category": cat if cat in ("abuse", "forum", "security") else "community", + "kind": "rss" if ad == "rss" else ("json" if ad in ("generic_json", "discourse") else "html"), + "url": url, "adapter": ad, "tags": ["auto"], "lim": 15, "detail": False} + added.append(rec) + if len(added) >= max_add: + break + print("живых фидов:", len(added), flush=True) + if not dry: + for rec in added: + source_state.add_custom(rec["name"], rec["category"], rec["url"], kind=rec["kind"], + adapter=rec["adapter"], tags=rec["tags"], limit=rec["lim"], + detail=rec["detail"]) + return {"found": len(found), "alive": len(added), "added": added if dry else [a["name"] for a in added]} + + +if __name__ == "__main__": + import sys + lim = int(sys.argv[1]) if len(sys.argv) > 1 else 30 + res = discover(limit_hosts=lim) + print(json.dumps({"found": res["found"], "alive": res["alive"], "added": res["added"][:20]}, + ensure_ascii=False, indent=1)) \ No newline at end of file diff --git a/omni/feeds.py b/omni/feeds.py index 81b5381..f07a31e 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -336,7 +336,8 @@ def _xenforo_latest(src, limit): if len(title) < 12 or href.rstrip("/").endswith("/threads"): continue if not href.startswith("http"): - href = "https://www.blackhatworld.com" + href + parts = urllib.parse.urlsplit(src.url) + href = "%s://%s%s" % (parts.scheme, parts.netloc, href) out.append(_item(src, "thread", href, title)) if len(out) >= limit: break diff --git a/omni/serve.py b/omni/serve.py index 5781999..849cc54 100644 --- a/omni/serve.py +++ b/omni/serve.py @@ -190,6 +190,42 @@ class Handler(BaseHTTPRequestHandler): "categories": cats, }) + if path == "/api/runs": + try: + limit = max(1, min(500, int((q.get("limit") or ["80"])[0]))) + except ValueError: + limit = 80 + import time as _t + con = store._con() if hasattr(store, "_con") else None + rows = [] + try: + import sqlite3 as _s3 + from .config import DB as _DB + db = _s3.connect(str(_DB), timeout=30) + db.row_factory = _s3.Row + for r in db.execute("SELECT id, ts, provider, source, query, got, kept, status, bytes " + "FROM runs ORDER BY id DESC LIMIT ?", (limit,)): + d = dict(r) + ts = d.get("ts") or 0 + d["when"] = _t.strftime("%Y-%m-%d %H:%M", _t.localtime(ts)) if ts else "" + rows.append(d) + total = db.execute("SELECT count(*) FROM runs").fetchone()[0] + db.close() + except Exception as e: + return self._json({"error": "runs: %s" % e}) + return self._json({"items": rows, "total": total, "shown": len(rows)}) + + if path == "/api/abuse": + from . import abuse as _ab + tag = (q.get("tag") or [""])[0].strip() or None + if tag in ("all", "все"): + tag = None + try: + lim = max(1, min(200, int((q.get("limit") or ["40"])[0]))) + except ValueError: + lim = 40 + return self._json(_ab.scan(limit=lim, tag=tag)) + if path == "/api/sources": cat = (q.get("category") or [None])[0] query = (q.get("q") or [""])[0].strip().lower() @@ -259,6 +295,15 @@ class Handler(BaseHTTPRequestHandler): except Exception: body = {} + if u.path == "/api/discover": + from . import discover as _d + q = urllib.parse.parse_qs(u.query) + hosts = max(5, min(120, int((q.get("hosts") or ["40"])[0]))) + add = max(1, min(60, int((q.get("add") or ["25"])[0]))) + dry = (q.get("dry") or ["0"])[0] not in ("0", "false", "") + res = _d.discover(limit_hosts=hosts, workers=8, max_add=add, dry=dry) + return self._json(res) + if u.path == "/api/feed": jid = _jid("feed") _job(jid, state="running") diff --git a/omni/sources.py b/omni/sources.py index 3c1c129..d3e88b8 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -12,6 +12,8 @@ from dataclasses import dataclass, field from .sources_ext import EXTRA from .sources_ext2 import EXTRA2 +from .sources_ext3 import EXTRA3 +from .sources_abuse import ABUSE_LIVE @dataclass @@ -188,7 +190,7 @@ def _dedup(rows): return out -SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2]) +SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE]) BY_NAME = {s.name: s for s in SOURCES} CATEGORIES = sorted({s.category for s in SOURCES}) diff --git a/omni/sources_abuse.py b/omni/sources_abuse.py new file mode 100644 index 0000000..826cb6e --- /dev/null +++ b/omni/sources_abuse.py @@ -0,0 +1,45 @@ +"""Абузы (живые): форумы, каналы и площадки с промо/сливами/софтом. + +Только структурные движки (RSS/Discourse/XenForo/Telegram) и HTML-форумы, +у которых реально нашлись темы (>=5 ссылок на треды), а не навигация. +""" + + +def A(name, category, kind, url, adapter, tags=(), limit=20): + return {"name": name, "category": category, "kind": kind, "url": url, + "adapter": adapter, "tags": tuple(tags), "limit": limit, "detail": False, "extra": {}} + + +ABUSE_LIVE = [ + A("ab_lolz_live", "abuse", "html", "https://lolz.live/tags/rassylka/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_crackingx_com", "abuse", "html", "https://crackingx.com/forums/5/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_xreactor_org", "abuse", "html", "https://xreactor.org/forums/combolists.152/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_crdpro_cc", "abuse", "html", "https://crdpro.cc/tags/tools/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_blackhatworld_com", "abuse", "html", "https://www.blackhatworld.com/seo/up-to-date-list-of-specialized-software-for-telegram.1642970/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_cracked_io", "abuse", "html", "https://cracked.io/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_t_me", "abuse", "html", "https://t.me/s/Abuz_ai_soft", "tg_web", ('abuse', 'abuse'), limit=20), + A("ab_tuts4you_com", "abuse", "html", "https://tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_forum_tuts4you_com", "abuse", "html", "https://forum.tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_nodeloc_com", "abuse", "json", "https://www.nodeloc.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20), + A("ab_v2ex_com", "abuse", "json", "https://www.v2ex.com/api/topics/hot.json", "generic_json", ('abuse', 'abuse'), limit=20), + A("ab_hostloc_com", "abuse", "rss", "https://hostloc.com/forum.php?mod=rss", "rss", ('abuse', 'abuse'), limit=20), + A("ab_linux_do", "abuse", "json", "https://linux.do/latest.json", "discourse", ('abuse', 'abuse'), limit=20), + A("ab_xss_is", "abuse", "html", "https://xss.is/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_exploit_in", "abuse", "html", "https://exploit.in/challenge/?return=/", "xenforo_latest", ('abuse', 'abuse'), limit=20), + A("ab_fb_killa_pro", "abuse", "html", "https://fb-killa.pro/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_afflift_com", "abuse", "html", "https://afflift.com/f/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_altenens_is", "abuse", "html", "https://altenens.is/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_carders_biz", "abuse", "html", "https://carders.biz/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_ascarding_net", "abuse", "html", "https://ascarding.net/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_zhyk_org", "abuse", "html", "https://zhyk.org/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_zenno_club", "abuse", "html", "https://zenno.club/discussion/tags/arbitrazh-trafika/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_carder_su", "abuse", "html", "https://carder.su/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_cpamafia_pro", "abuse", "html", "https://cpamafia.pro/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_forexfactory_com", "abuse", "html", "https://www.forexfactory.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_casinomeister_com", "abuse", "html", "https://www.casinomeister.com/forums/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_hardforum_com", "abuse", "html", "https://hardforum.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_niflheim_world", "abuse", "html", "https://niflheim.world/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20), + A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20), +] diff --git a/omni/sources_ext3.py b/omni/sources_ext3.py new file mode 100644 index 0000000..f1b4c72 --- /dev/null +++ b/omni/sources_ext3.py @@ -0,0 +1,112 @@ +"""Волна 4 (только живые после реального fetch): релизы репозиториев, рассылки, +Discourse-форумы, подреддиты, CN/JP блоги, HF-задачи. Каждая строка проверена: items > 0.""" + + +def S(name, category, kind, url, adapter="rss", tags=(), limit=20, detail=False, extra=None): + return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter, + "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}} + + +EXTRA3 = [ + # ---- release ---- + S("gh_rel_ollama", "release", "atom", "https://github.com/ollama/ollama/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_llamacpp", "release", "atom", "https://github.com/ggml-org/llama.cpp/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_vllm", "release", "atom", "https://github.com/vllm-project/vllm/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_comfyui", "release", "atom", "https://github.com/comfyanonymous/ComfyUI/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_dify", "release", "atom", "https://github.com/langgenius/dify/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_n8n", "release", "atom", "https://github.com/n8n-io/n8n/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_langchain", "release", "atom", "https://github.com/langchain-ai/langchain/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_autogen", "release", "atom", "https://github.com/microsoft/autogen/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_litellm", "release", "atom", "https://github.com/BerriAI/litellm/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_openwebui", "release", "atom", "https://github.com/open-webui/open-webui/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_ragflow", "release", "atom", "https://github.com/infiniflow/ragflow/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_browseruse", "release", "atom", "https://github.com/browser-use/browser-use/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_playwrightmcp", "release", "atom", "https://github.com/microsoft/playwright-mcp/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_mcpservers", "release", "atom", "https://github.com/modelcontextprotocol/servers/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_dspy", "release", "atom", "https://github.com/stanfordnlp/dspy/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_transformers", "release", "atom", "https://github.com/huggingface/transformers/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_trl", "release", "atom", "https://github.com/huggingface/trl/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_sglang", "release", "atom", "https://github.com/sgl-project/sglang/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_unsloth", "release", "atom", "https://github.com/unslothai/unsloth/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_axolotl", "release", "atom", "https://github.com/axolotl-ai-cloud/axolotl/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_markitdown", "release", "atom", "https://github.com/microsoft/markitdown/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_openai_sdk", "release", "atom", "https://github.com/openai/openai-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_anthropic_sdk", "release", "atom", "https://github.com/anthropics/anthropic-sdk-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_gemini_sdk", "release", "atom", "https://github.com/google-gemini/generative-ai-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_crewai", "release", "atom", "https://github.com/crewAIInc/crewAI/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_agno", "release", "atom", "https://github.com/agno-agi/agno/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_livekit_agents", "release", "atom", "https://github.com/livekit/agents/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_mem0", "release", "atom", "https://github.com/mem0ai/mem0/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_qdrant", "release", "atom", "https://github.com/qdrant/qdrant/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + S("gh_rel_chroma", "release", "atom", "https://github.com/chroma-core/chroma/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False), + # ---- community ---- + S("lesswrong", "community", "rss", "https://www.lesswrong.com/feed.xml?view=curated-rss", "rss", ('ai', 'alignment'), limit=20, detail=False), + S("alignmentforum", "community", "rss", "https://www.alignmentforum.org/feed.xml?view=curated-rss", "rss", ('ai', 'alignment'), limit=20, detail=False), + S("gwern", "community", "rss", "https://gwern.net/feed", "rss", ('ai', 'longform'), limit=20, detail=False), + S("importai", "community", "rss", "https://importai.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False), + S("thezvi", "community", "rss", "https://thezvi.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False), + S("latentspace", "community", "rss", "https://www.latent.space/feed", "rss", ('ai', 'podcast'), limit=20, detail=False), + S("aisupremacy", "community", "rss", "https://aisupremacy.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False), + S("oneusefulthing", "community", "rss", "https://www.oneusefulthing.org/feed", "rss", ('ai', 'practices'), limit=20, detail=False), + S("chinatalk", "community", "rss", "https://www.chinatalk.media/feed", "rss", ('ai', 'china'), limit=20, detail=False), + S("reddit_claudeai", "community", "rss", "https://old.reddit.com/r/ClaudeAI/.rss", "reddit_listing", ('reddit', 'claude'), limit=20, detail=False), + S("reddit_bard", "community", "rss", "https://old.reddit.com/r/Bard/.rss", "reddit_listing", ('reddit', 'gemini'), limit=20, detail=False), + S("reddit_singularity", "community", "rss", "https://old.reddit.com/r/singularity/.rss", "reddit_listing", ('reddit', 'ai'), limit=20, detail=False), + S("reddit_openai_dev", "community", "rss", "https://old.reddit.com/r/OpenAIDev/.rss", "reddit_listing", ('reddit', 'openai'), limit=20, detail=False), + S("reddit_mcp", "community", "rss", "https://old.reddit.com/r/mcp/.rss", "reddit_listing", ('reddit', 'mcp'), limit=20, detail=False), + S("reddit_ai_agents", "community", "rss", "https://old.reddit.com/r/AI_Agents/.rss", "reddit_listing", ('reddit', 'agents'), limit=20, detail=False), + S("reddit_bugbounty", "community", "rss", "https://old.reddit.com/r/bugbounty/.rss", "reddit_listing", ('reddit', 'bugbounty'), limit=20, detail=False), + S("reddit_netsecstudents", "community", "rss", "https://old.reddit.com/r/netsecstudents/.rss", "reddit_listing", ('reddit', 'security'), limit=20, detail=False), + S("reddit_blueteamsec", "community", "rss", "https://old.reddit.com/r/blueteamsec/.rss", "reddit_listing", ('reddit', 'security'), limit=20, detail=False), + S("reddit_prompteng", "community", "rss", "https://old.reddit.com/r/PromptEngineering/.rss", "reddit_listing", ('reddit', 'prompts'), limit=20, detail=False), + S("reddit_oobabooga", "community", "rss", "https://old.reddit.com/r/Oobabooga/.rss", "reddit_listing", ('reddit', 'llm'), limit=20, detail=False), + S("reddit_unsloth", "community", "rss", "https://old.reddit.com/r/unsloth/.rss", "reddit_listing", ('reddit', 'finetune'), limit=20, detail=False), + S("reddit_mlops", "community", "rss", "https://old.reddit.com/r/mlops/.rss", "reddit_listing", ('reddit', 'mlops'), limit=20, detail=False), + S("reddit_devops2", "community", "rss", "https://old.reddit.com/r/devops/.rss", "reddit_listing", ('reddit', 'devops'), limit=20, detail=False), + S("reddit_homelab2", "community", "rss", "https://old.reddit.com/r/homelab/.rss", "reddit_listing", ('reddit', 'homelab'), limit=20, detail=False), + S("reddit_grok2", "community", "rss", "https://old.reddit.com/r/grok/.rss", "reddit_listing", ('reddit', 'grok'), limit=20, detail=False), + # ---- security ---- + S("sysdig", "security", "rss", "https://sysdig.com/feed/", "rss", ('security', 'cloud'), limit=20, detail=False), + S("permiso", "security", "rss", "https://permiso.io/blog/rss.xml", "rss", ('security', 'cloud'), limit=20, detail=False), + S("aquasec", "security", "rss", "https://www.aquasec.com/feed/", "rss", ('security', 'cloud'), limit=20, detail=False), + S("infosecmatter", "security", "rss", "https://www.infosecmatter.com/feed/", "rss", ('security', 'tools'), limit=20, detail=False), + S("cvefeed", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ('security', 'cve'), limit=20, detail=False), + S("graynoise", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ('security', 'intel'), limit=20, detail=False), + S("recordedfuture", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ('security', 'intel'), limit=20, detail=False), + S("cisa_news", "security", "rss", "https://www.cisa.gov/news.xml", "rss", ('security', 'gov'), limit=20, detail=False), + S("msrc", "security", "rss", "https://api.msrc.microsoft.com/update-guide/rss", "rss", ('security', 'microsoft', 'cve'), limit=20, detail=False), + # ---- forum ---- + S("forum_openai", "forum", "json", "https://community.openai.com/latest.json", "discourse", ('forum', 'openai'), limit=20, detail=False), + S("forum_fly", "forum", "json", "https://community.fly.io/latest.json", "discourse", ('forum', 'infra'), limit=20, detail=False), + S("forum_bitwarden", "forum", "json", "https://community.bitwarden.com/latest.json", "discourse", ('forum', 'security'), limit=20, detail=False), + S("forum_cursor", "forum", "json", "https://forum.cursor.com/latest.json", "discourse", ('forum', 'ai', 'ide'), limit=20, detail=False), + S("forum_manjaro", "forum", "json", "https://forum.manjaro.org/latest.json", "discourse", ('forum', 'linux'), limit=20, detail=False), + S("forum_arduino", "forum", "json", "https://forum.arduino.cc/latest.json", "discourse", ('forum', 'hardware'), limit=20, detail=False), + S("forum_kde", "forum", "json", "https://discuss.kde.org/latest.json", "discourse", ('forum', 'linux'), limit=20, detail=False), + S("forum_ray", "forum", "json", "https://discuss.ray.io/latest.json", "discourse", ('forum', 'ml'), limit=20, detail=False), + S("forum_wandb", "forum", "json", "https://community.wandb.ai/latest.json", "discourse", ('forum', 'ml'), limit=20, detail=False), + S("forum_grafana2", "forum", "json", "https://community.grafana.com/latest.json", "discourse", ('forum', 'observability'), limit=20, detail=False), + S("forum_traefik2", "forum", "json", "https://community.traefik.io/latest.json", "discourse", ('forum', 'infra'), limit=20, detail=False), + S("forum_nvidia", "forum", "json", "https://forums.developer.nvidia.com/latest.json", "discourse", ('forum', 'nvidia'), limit=20, detail=False), + # ---- company_cn ---- + S("qbitai", "company_cn", "rss", "https://www.qbitai.com/feed", "rss", ('cn', 'ai', 'news'), limit=20, detail=False), + S("infoq_cn", "company_cn", "rss", "https://www.infoq.cn/feed", "rss", ('cn', 'dev'), limit=20, detail=False), + S("leiphone", "company_cn", "rss", "https://www.leiphone.com/feed", "rss", ('cn', 'tech'), limit=20, detail=False), + # ---- models ---- + S("hf_models_summary", "models", "json", "https://huggingface.co/api/models?pipeline_tag=summarization&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_translate", "models", "json", "https://huggingface.co/api/models?pipeline_tag=translation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_similarity", "models", "json", "https://huggingface.co/api/models?pipeline_tag=sentence-similarity&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'embed'), limit=20, detail=False), + S("hf_models_tokencls", "models", "json", "https://huggingface.co/api/models?pipeline_tag=token-classification&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_zeroshot", "models", "json", "https://huggingface.co/api/models?pipeline_tag=zero-shot-classification&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_segment", "models", "json", "https://huggingface.co/api/models?pipeline_tag=image-segmentation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False), + S("hf_models_objectdet", "models", "json", "https://huggingface.co/api/models?pipeline_tag=object-detection&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False), + S("hf_models_depth", "models", "json", "https://huggingface.co/api/models?pipeline_tag=depth-estimation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False), + S("hf_models_timeseries", "models", "json", "https://huggingface.co/api/models?pipeline_tag=time-series-forecasting&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_rl", "models", "json", "https://huggingface.co/api/models?pipeline_tag=reinforcement-learning&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'rl'), limit=20, detail=False), + S("hf_models_qa", "models", "json", "https://huggingface.co/api/models?pipeline_tag=question-answering&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False), + S("hf_models_img2text", "models", "json", "https://huggingface.co/api/models?pipeline_tag=image-to-text&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vlm'), limit=20, detail=False), + S("hf_models_rerank2", "models", "json", "https://huggingface.co/api/models?pipeline_tag=text-ranking&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'rag'), limit=20, detail=False), + # ---- benchmark ---- + S("aider_board", "benchmark", "rss", "https://aider.chat/feed.xml", "rss", ('benchmark', 'code'), limit=20, detail=False), + S("arcprize", "benchmark", "rss", "https://arcprize.org/feed.xml", "rss", ('benchmark', 'reasoning'), limit=20, detail=False), +] diff --git a/web/index.html b/web/index.html index f2746b2..835c301 100644 --- a/web/index.html +++ b/web/index.html @@ -229,37 +229,93 @@ select:focus,input:focus{border-color:rgba(124,108,244,.6)} .thead,.row{grid-template-columns:38px minmax(140px,1fr) 84px 80px 64px} .thead span:nth-child(4),.row .catpill{display:none} } + +/* ---- меню v2 ---- */ +.brand .ver{margin-left:auto;font:10.5px var(--mono);color:var(--dim2);background:#151a24;border:1px solid var(--line);border-radius:20px;padding:2px 8px} +.menu-btn{display:flex;align-items:center;gap:9px;margin:0 0 12px;padding:9px 11px;width:100%;border:1px solid var(--line); + background:#11151d;color:var(--dim);border-radius:11px;font:12.5px inherit;cursor:pointer;transition:.15s} +.menu-btn:hover{border-color:var(--line2);color:var(--txt);background:#161c26} +.menu-btn i{font-style:normal;color:var(--acc2)} +.menu-btn .kbd{margin-left:auto} +.kbd{font:10px var(--mono);color:var(--dim2);border:1px solid var(--line2);border-radius:5px;padding:1px 5px;background:#0d1117} +.nav button .kbd{margin-left:auto;margin-right:6px;opacity:.55} +.nav button:hover .kbd{opacity:1} +.sect{display:flex;align-items:center;justify-content:space-between;gap:8px} +.sect .mini{border:0;background:transparent;color:var(--dim2);cursor:pointer;font-size:11px;padding:0 2px} +.sect .mini:hover{color:var(--txt)} +.cats.collapsed{display:none} +.quick{display:grid;gap:6px;margin:2px 0 10px} +.quick button{display:flex;align-items:center;gap:9px;padding:8px 10px;border-radius:9px;border:1px solid var(--line); + background:#0f131a;color:var(--dim);font:12.5px inherit;cursor:pointer;text-align:left;transition:.14s} +.quick button:hover{border-color:var(--line2);color:var(--txt);background:#151b25;transform:translateX(2px)} +.quick button i{font-style:normal;color:var(--acc);width:12px;text-align:center} +.ttlwrap{min-width:0} +.crumbs{display:flex;gap:6px;flex-wrap:wrap;margin-bottom:5px} +.crumbs span{font:10.5px var(--mono);color:var(--dim2);border:1px solid var(--line);border-radius:20px;padding:2px 8px;background:#0d1117} +.crumbs span.on{color:#cfd6ec;border-color:#3b3660;background:#181633} +.palette{position:fixed;inset:0;background:rgba(4,5,8,.72);backdrop-filter:blur(6px);display:none;z-index:80;padding-top:14vh;justify-content:center} +.palette.on{display:flex} +.pal-box{width:min(620px,92vw);background:#0f131b;border:1px solid var(--line2);border-radius:16px; + box-shadow:0 30px 80px rgba(0,0,0,.6);overflow:hidden;align-self:flex-start} +.pal-box input{width:100%;border:0;border-bottom:1px solid var(--line);background:transparent;color:var(--txt); + padding:15px 18px;font:14px inherit;outline:none} +.pal-list{max-height:52vh;overflow:auto;padding:6px} +.pal-item{display:flex;align-items:center;gap:10px;padding:9px 12px;border-radius:9px;cursor:pointer;color:var(--dim);font-size:13px} +.pal-item .tag{font:10px var(--mono);color:var(--dim2);border:1px solid var(--line);border-radius:5px;padding:1px 6px} +.pal-item .cnt{margin-left:auto;font:11px var(--mono);color:var(--dim2)} +.pal-item.sel,.pal-item:hover{background:#171d29;color:var(--txt)} +.pal-foot{display:flex;gap:6px;align-items:center;padding:9px 14px;border-top:1px solid var(--line);font:10.5px var(--mono);color:var(--dim2)} +.thead.runs,.row.runs{grid-template-columns:132px minmax(120px,1fr) minmax(120px,1.2fr) 90px 100px 110px} +.runs-time{font:11.5px var(--mono);color:var(--dim2)} +.chip b{font:10.5px var(--mono);color:var(--dim2);font-weight:500;margin-left:3px} +.chip.on b{color:#cbc4ff}
-
+
+
+ +
+
↑↓ выбор ↵ открыть esc закрыть
+
+
+
работаю…
@@ -392,6 +478,12 @@ function setCat(name) { S.cat = name; S.offset = 0; renderCats(S.stats ? S.stats.categories : []); syncChips(); + crumb(); + try { + const u = new URL(location.href); + name && name !== 'all' ? u.searchParams.set('cat', name) : u.searchParams.delete('cat'); + history.replaceState(null, '', u.toString()); + } catch (e) {} if (S.view === 'feed') loadItems(true); else loadSources(); $('#f-cat').value = name; } @@ -403,11 +495,14 @@ async function loadStats() { S.stats = s; $('#nav-feed-n').textContent = fmt(s.items); $('#nav-src-n').textContent = fmt(s.sources); + $('#nav-runs-n').textContent = fmt(s.runs); + $('#brand-n').textContent = fmt(s.alive) + '/' + fmt(s.sources); $('#aside-foot').innerHTML = `${fmt(s.sources)} источников · ${fmt(s.sources_enabled)} вкл
` + `${fmt(s.alive)} живых · ${fmt(s.tested)} проверено
` + `${fmt(s.runs)} прогонов · ${fmt(s.priced)} с ценой`; renderCats(s.categories); + buildChips(s.categories); const ks = [ ['записей', s.items, 'в sqlite + fts5', ''], ['источников', s.sources, `${s.sources_enabled} включено`, ''], @@ -555,21 +650,144 @@ async function check(category, name) { } } +/* ---------- прогоны ---------- */ +async function loadRuns() { + const rows = $('#runs-rows'); + rows.innerHTML = '
'; + const r = await api('/api/runs?limit=100'); + if (r.error) { rows.innerHTML = ''; return toast(r.error, 'err'); } + const items = r.items || []; + const ok = items.filter(x => String(x.status || '').toLowerCase().startsWith('ok')).length; + const got = items.reduce((a, x) => a + (x.got || 0), 0); + const kept = items.reduce((a, x) => a + (x.kept || 0), 0); + $('#runs-strip').innerHTML = + `
всего прогонов${fmt(r.total || items.length)}
` + + `
успешных${fmt(ok)}
` + + `
пустых${fmt(items.length - ok)}
` + + `
получено${fmt(got)}
` + + `
сохранено${fmt(kept)}
` + + `
конверсия${got ? Math.round(kept / got * 100) : 0}%
`; + rows.innerHTML = ''; + if (!items.length) { rows.innerHTML = '
прогонов нет. нажми «Собрать ленту».
'; return; } + items.forEach(x => { + const d = el('div', 'row runs'); + const st = String(x.status || '').toLowerCase(); + d.innerHTML = + `${esc(x.when || '')}` + + `${esc(x.source || x.provider || '—')}` + + `${esc(String(x.query || '—').slice(0, 62))}` + + `${fmt(x.got || 0)}` + + `${fmt(x.kept || 0)}` + + `${esc(x.status || '—')}`; + rows.appendChild(d); + }); +} + +function crumb() { + const c = $('#crumbs'); + if (!c) return; + const v = {feed: 'лента', src: 'источники', runs: 'прогоны'}[S.view] || S.view; + let h = `${esc(v)}`; + if (S.cat && S.cat !== 'all') h += `${esc(S.cat)}`; + if (S.q) h += `поиск: ${esc(S.q.slice(0, 22))}`; + if (S.view === 'feed') h += `${S.sort === 'score' ? 'по score' : 'по свежести'}`; + if (S.view === 'src' && S.state) h += `${esc(S.state)}`; + c.innerHTML = h; +} + +const VIEWMETA = { + feed: ['Лента', 'свежие записи со всех источников'], + src: ['Источники', 'подключение, здоровье, включение'], + runs: ['Прогоны', 'история запусков парсера'], + abuse: ['Абузы', 'промо, триалы, ключи, сливы, софт'] +}; + +/* ---------- абузы ---------- */ +const ABUSE_LABELS = {keys: 'ключи', promo: 'промо', trial: 'триалы', leak: 'сливы', tool: 'софт', resale: 'продажа', news: 'новости'}; +S.abuseTag = null; S.abuse = null; + +async function loadAbuse(tag) { + if (tag !== undefined) S.abuseTag = tag; + const q = new URLSearchParams({limit: 60}); + if (S.abuseTag) q.set('tag', S.abuseTag); + const [d, st] = await Promise.all([api('/api/abuse?' + q), S.stats ? Promise.resolve(S.stats) : api('/api/stats')]); + S.abuse = d; + const tags = Object.entries(d.tags || {}).sort((a, b) => b[1] - a[1]); + const strip = $('#abuse-strip'); + if (strip) strip.innerHTML = + `${d.total} находок` + + `${tags.length} тегов` + + `в базе ${fmt(st.items)}` + + `источников ${fmt(st.sources)}`; + const box = $('#abuse-tags'); + if (box) { + box.innerHTML = `` + + tags.map(([k, v]) => ``).join(''); + box.querySelectorAll('.chip').forEach(b => b.onclick = () => loadAbuse(b.dataset.t === 'all' ? null : b.dataset.t)); + } + const cards = $('#abuse-cards'); + if (!cards) return; + if (!d.items.length) { cards.innerHTML = '
пусто — запусти прогон категории abuse
'; return; } + cards.innerHTML = d.items.map(it => ` +
+
${esc(it.source)} + ${it.abuse_score}
+ ${esc(it.title)} +
${it.tags.map(t => `${esc(ABUSE_LABELS[t] || t)}`).join('')} + ${esc((it.published || '').slice(0, 10))}
+
`).join(''); +} + +$('#btn-discover') && ($('#btn-discover').onclick = async (e) => { + const b = e.currentTarget; const t0 = b.textContent; + b.disabled = true; b.textContent = 'ищу фиды…'; + try { + const r = await api('/api/discover?hosts=40&add=25', {method: 'POST'}); + b.textContent = `+${(r.added || []).length} подключено`; + const nt = $('#discover-note'); if (nt) nt.textContent = `кандидатов ${r.found || 0} · живых ${r.alive || 0}`; + setTimeout(() => { b.disabled = false; b.textContent = t0; loadStats(); }, 4000); + } catch (err) { b.textContent = 'ошибка: ' + err.message; b.disabled = false; setTimeout(() => b.textContent = t0, 4000); } +}); + /* ---------- view switching ---------- */ function setView(v) { + if (!VIEWMETA[v]) v = 'feed'; S.view = v; - $('#nav-feed').classList.toggle('on', v === 'feed'); - $('#nav-src').classList.toggle('on', v === 'src'); - $('#view-feed').style.display = v === 'feed' ? '' : 'none'; - $('#view-src').style.display = v === 'src' ? '' : 'none'; + ['feed', 'src', 'runs', 'abuse'].forEach(k => { + const b = $('#nav-' + k); if (b) b.classList.toggle('on', k === v); + const sec = $('#view-' + k); if (sec) sec.style.display = k === v ? '' : 'none'; + }); $('#btn-feed').style.display = v === 'feed' ? '' : 'none'; - $('#feed-tools').style.display = v === 'feed' ? '' : 'none'; - if (v === 'feed') loadItems(true); else loadSources(); + const ft = $('#feed-tools'); if (ft) ft.style.display = v === 'feed' ? '' : 'none'; + if (v === 'abuse') loadAbuse().catch(e => console.error('abuse', e)); + const m = VIEWMETA[v]; + $('#ttl').innerHTML = `${m[0]}${m[1]}`; + crumb(); + try { + const u = new URL(location.href); + u.searchParams.set('view', v); + S.cat && S.cat !== 'all' ? u.searchParams.set('cat', S.cat) : u.searchParams.delete('cat'); + S.q ? u.searchParams.set('q', S.q) : u.searchParams.delete('q'); + history.replaceState(null, '', u.toString()); + } catch (e) {} + if (v === 'feed') loadItems(true); + else if (v === 'src') loadSources(); + else loadRuns(); } /* ---------- wire ---------- */ $('#nav-feed').onclick = () => setView('feed'); $('#nav-src').onclick = () => setView('src'); +$('#nav-runs').onclick = () => setView('runs'); +$('#nav-abuse').onclick = () => { setView('abuse'); loadAbuse(); }; +$('#q-feed').onclick = runFeed; +$('#q-check').onclick = () => check(null, null); +$('#q-add').onclick = () => $('#modal').classList.add('on'); +$('#cats-toggle').onclick = e => { + const c = $('#cats'); const hid = c.classList.toggle('collapsed'); + e.target.textContent = hid ? '▸' : '▾'; + try { localStorage.setItem('omni.cats', hid ? '0' : '1'); } catch (er) {} +}; $('#btn-feed').onclick = runFeed; $('#btn-more').onclick = () => loadItems(false); $('#btn-check-all').onclick = () => check(null, null); @@ -588,8 +806,17 @@ $('#q').oninput = e => { }, 320); }; document.addEventListener('keydown', e => { - if (e.key === '/' && document.activeElement !== $('#q')) { e.preventDefault(); $('#q').focus(); } - if (e.key === 'Escape') $('#modal').classList.remove('on'); + const typing = /^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement.tagName); + if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === 'k') { e.preventDefault(); openPalette(); return; } + if (e.key === '/' && !typing) { e.preventDefault(); $('#q').focus(); return; } + if (e.key === 'Escape') { $('#modal').classList.remove('on'); $('#palette').classList.remove('on'); return; } + if (!typing && !e.ctrlKey && !e.metaKey) { + if (e.key === '1') setView('feed'); + if (e.key === '2') setView('src'); + if (e.key === '3') setView('runs'); + if (e.key === '4') { setView('abuse'); loadAbuse(); } + if (e.key === 'r' && !e.shiftKey) runFeed(); + } }); $('#btn-add').onclick = () => $('#modal').classList.add('on'); @@ -608,27 +835,107 @@ $('#a-save').onclick = async () => { loadSources(); loadStats(); }; -/* feed toolbar */ -$('#feed-tools').innerHTML = - ['all', 'community', 'papers', 'security', 'company', 'forum', 'models', 'benchmark'] - .map(c => ``).join('') + - '' + - '' + - ''; -document.querySelectorAll('#feed-tools .chip').forEach(b => { - b.onclick = () => { - if (b.dataset.k === 'sort') { S.sort = b.dataset.sort; syncChips(); loadItems(true); } - else setCat(b.dataset.cat); - }; -}); -syncChips(); +/* feed toolbar: чипы строятся из живого реестра категорий */ +function buildChips(cats) { + const box = $('#feed-tools'); + if (!box) return; + const list = cats || (S.stats && S.stats.categories) || []; + const tot = {}; list.forEach(c => { tot[c.name] = c.total; }); + box.innerHTML = + `` + + list.map(c => ``).join('') + + '' + + '' + + ''; + box.querySelectorAll('.chip').forEach(b => { + b.onclick = () => { + if (b.dataset.k === 'sort') { S.sort = b.dataset.sort; syncChips(); loadItems(true); } + else setCat(b.dataset.cat); + }; + }); + syncChips(); +} + +/* ---------- палитра ---------- */ +let PAL = [], palSel = 0; + +function palItems() { + const out = [ + {tag: 'вид', label: 'Лента', hint: 'свежие записи', run: () => setView('feed')}, + {tag: 'вид', label: 'Источники', hint: 'менеджер источников', run: () => setView('src')}, + {tag: 'вид', label: 'Прогоны', hint: 'история запусков', run: () => setView('runs')}, + {tag: 'вид', label: 'Абузы', hint: 'промо/триалы/ключи', run: () => { setView('abuse'); loadAbuse(); }}, + {tag: 'действие', label: 'Собрать ленту', hint: 'опрашивает включённые источники', run: runFeed}, + {tag: 'действие', label: 'Проверить все источники', hint: 'health-check', run: () => check(null, null)}, + {tag: 'действие', label: 'Добавить источник', hint: 'своя лента или API', run: () => $('#modal').classList.add('on')}, + {tag: 'действие', label: 'Включить всё в выборке', hint: 'bulk enable', run: async () => { await post('/api/sources/toggle_all', {enabled: true, category: S.cat}); toast('все включены', 'ok'); loadSources(); loadStats(); }}, + {tag: 'действие', label: 'Выключить всё в выборке', hint: 'bulk disable', run: async () => { await post('/api/sources/toggle_all', {enabled: false, category: S.cat}); toast('все выключены', 'ok'); loadSources(); loadStats(); }} + ]; + ((S.stats && S.stats.categories) || []).forEach(c => out.push({ + tag: 'категория', label: c.name, hint: `${c.alive}/${c.total} живых`, q: c.name, + run: () => { setView('feed'); setCat(c.name); } + })); + (S.sources || []).forEach(x => out.push({ + tag: 'источник', label: x.name, + hint: `${x.category} · ${x.stored || 0} записей` + (x.note ? ' · ' + x.note : ''), + hay: `${x.name} ${x.category} ${(x.tags || []).join(' ')}`, + run: () => { S.srcQ = x.name; setView('src'); $('#q').value = x.name; S.q = x.name; loadSources(); } + })); + return out; +} + +function renderPal(q) { + const box = $('#pal-list'); + PAL = palItems(); + const needle = (q || '').toLowerCase().trim(); + const list = needle ? PAL.filter(x => (x.label + ' ' + x.tag + ' ' + (x.hint || '') + ' ' + (x.hay || '')).toLowerCase().includes(needle)) : PAL; + PAL = list.slice(0, 60); + if (palSel >= PAL.length) palSel = 0; + box.innerHTML = PAL.length ? '' : '
ничего не нашлось
'; + PAL.forEach((x, i) => { + const d = el('div', 'pal-item' + (i === palSel ? ' sel' : '')); + d.innerHTML = `${esc(x.tag)}${esc(x.label)}${esc(x.hint || '')}`; + d.onclick = () => { $('#palette').classList.remove('on'); x.run(); }; + box.appendChild(d); + }); +} + +async function openPalette() { + $('#palette').classList.add('on'); + palSel = 0; + $('#pal-q').value = ''; + renderPal(''); + setTimeout(() => $('#pal-q').focus(), 30); + if (!(S.sources || []).length) { + try { + const r = await api('/api/sources?category=all'); + if (r && r.items) { S.sources = r.items; renderPal($('#pal-q').value); } + } catch (e) {} + } +} + +$('#btn-palette').onclick = openPalette; +$('#btn-palette2').onclick = openPalette; +$('#palette').onclick = e => { if (e.target.id === 'palette') $('#palette').classList.remove('on'); }; +$('#pal-q').oninput = e => { palSel = 0; renderPal(e.target.value); }; +$('#pal-q').onkeydown = e => { + if (e.key === 'ArrowDown') { e.preventDefault(); palSel = Math.min(palSel + 1, PAL.length - 1); renderPal($('#pal-q').value); } + if (e.key === 'ArrowUp') { e.preventDefault(); palSel = Math.max(palSel - 1, 0); renderPal($('#pal-q').value); } + if (e.key === 'Enter' && PAL[palSel]) { $('#palette').classList.remove('on'); PAL[palSel].run(); } +}; + +try { if (localStorage.getItem('omni.cats') === '0') { $('#cats').classList.add('collapsed'); $('#cats-toggle').textContent = '▸'; } } catch (e) {} (async () => { const p = new URLSearchParams(location.search); if (p.get('cat')) S.cat = p.get('cat'); if (p.get('q')) { S.q = p.get('q'); $('#q').value = S.q; } await loadStats(); - if (p.get('view') === 'src') { setView('src'); } else { await loadItems(true); } + const v = p.get('view'); + if (v === 'abuse') { setView('abuse'); setTimeout(() => loadAbuse().catch(e => console.error('abuse', e)), 300); } + else if (v === 'src' || v === 'runs') setView(v); + else { setView('feed'); } + if (p.get('pal')) setTimeout(openPalette, 500); })();