omni: абузы (31 живой источник + классификатор), авто-добавление фидов, Telegram-бот, вкладка абузов; 589 источников
This commit is contained in:
reform
149
omni/abuse.py
Normal file
149
omni/abuse.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""Разбор абузов: промо, триалы, ключи, сливы, софт, реселл.
|
||||
|
||||
Работает поверх уже собранной базы (items) и поверх новых абуз-источников:
|
||||
каждая запись получает набор тегов и вес, панель показывает только то,
|
||||
что реально пахнет абузом, а не весь поток.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .config import DB
|
||||
|
||||
# тег -> (человеческое имя, вес, регулярки)
|
||||
RULES = [
|
||||
("keys", "ключи", 40, [
|
||||
r"sk-[A-Za-z0-9_\-]{20,}", r"gh[pousr]_[A-Za-z0-9]{20,}", r"xox[baprs]-[A-Za-z0-9\-]{10,}",
|
||||
r"(разда|выда|дарю|бесплатн|free|giveaway)[^.]{0,40}(ключ|key|токен|token)",
|
||||
r"(ключ|key|токен)[^.]{0,25}(бесплатн|free|дарю|раздач|giveaway)",
|
||||
]),
|
||||
("promo", "промо", 26, [
|
||||
r"промокод", r"promo\s?code", r"промо[\s-]?код", r"скидк[аи]\s+\d{1,3}", r"\d{1,3}\s?%\s?(off|скидк)",
|
||||
r"redeem\s?(code|код)", r"invite\s?code", r"реф(еральн|код)", r"бонус\s+на\s+сч[её]т", r"кэшб[еэ]к|cashback",
|
||||
]),
|
||||
("trial", "триалы", 24, [
|
||||
r"trial", r"триал", r"free\s+credits?", r"бесплатн\w*\s+(кредит|токен|баланс|доступ|период)",
|
||||
r"free\s+tier", r"\$\d+\s+free", r"试用", r"免费", r"пробн\w+\s+период", r"без\s+депозита", r"no\s+deposit",
|
||||
]),
|
||||
("leak", "сливы", 26, [
|
||||
r"слив(?!\s*(сок|ки|очн|тест|трафик))", r"leaked\s+(database|db|dump|combo)", r"db\s?dump",
|
||||
r"combo[\s-]?list", r"stealer\s?log", r"утечк\w*\s+баз", r"база\s+(пассов|почт|данных)",
|
||||
r"пароли\s+от\s+\w+\.(com|ru|net)",
|
||||
]),
|
||||
("tool", "софт", 18, [
|
||||
r"софт", r"кряк", r"crack(ed)?", r"bypass", r"обход\s+(блокиров|бан|защит)",
|
||||
r"авторег", r"autoreg", r"panel", r"панел[ьи]\s", r"рассылк", r"sender", r"скрипт",
|
||||
]),
|
||||
("resale", "продажа", 14, [
|
||||
r"продаю", r"продам", r"куплю\s+аккаунт", r"for\s+sale", r"цена\s*[:$]", r"\$\d+\s?/\s?(мес|mo|1k|1000)",
|
||||
r"реселлер", r"resell",
|
||||
]),
|
||||
("news", "новости", 6, [
|
||||
r"анонс", r"выпустил", r"released?", r"launch", r"обновлени", r"release notes",
|
||||
r"новая модель", r"new model",
|
||||
]),
|
||||
]
|
||||
|
||||
COMPILED = [(tag, label, w, [re.compile(p, re.I) for p in pats]) for tag, label, w, pats in RULES]
|
||||
TAG_LABEL = {t: l for t, l, _, _ in RULES}
|
||||
STRONG = {"keys", "promo", "trial", "resale"}
|
||||
# категории-источники, которые вообще имеют смысл для абузов
|
||||
ABUSE_CATS = {"abuse", "forum", "community", "company_cn"}
|
||||
# «статьи»: слова про ключи и утечки там — текст, а не раздача
|
||||
NOISE = re.compile(r"(blog|devto|medium|substack|newsletter|docs?|wiki)", re.I)
|
||||
|
||||
|
||||
def _source_meta():
|
||||
try:
|
||||
from . import sources as _s
|
||||
return {x.name: x.category for x in _s.all_sources()}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
SRC_CAT = _source_meta()
|
||||
|
||||
|
||||
def classify(title: str, text: str, tags: str = ""):
|
||||
blob = (" ".join([title or "", text or "", tags or ""]))[:4000]
|
||||
hits = []
|
||||
score = 0
|
||||
for tag, label, w, pats in COMPILED:
|
||||
n = 0
|
||||
for p in pats:
|
||||
m = p.search(blob)
|
||||
if m:
|
||||
n += 1
|
||||
if n:
|
||||
hits.append(tag)
|
||||
score += w * min(n, 3)
|
||||
return score, hits
|
||||
|
||||
|
||||
def _qualifies(source, title, text, tags):
|
||||
sc, hits = classify(title, text, tags)
|
||||
if not hits:
|
||||
return 0, []
|
||||
strong = bool(set(hits) & STRONG)
|
||||
cat = SRC_CAT.get(source)
|
||||
blob = (title or "") + " " + (text or "")
|
||||
raw_key = bool(re.search(r"(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
|
||||
if NOISE.search(source or "") and not raw_key:
|
||||
return 0, []
|
||||
if cat is None:
|
||||
# старые записи хранят категорию вместо имени источника
|
||||
if not raw_key and not str(source or "").startswith(("ab_", "tg_", "reddit_", "discourse_", "xenforo")):
|
||||
return 0, []
|
||||
elif cat not in ABUSE_CATS and not raw_key:
|
||||
return 0, []
|
||||
if strong and sc >= 24:
|
||||
return sc, hits
|
||||
if len(hits) >= 2 and sc >= 40:
|
||||
return sc, hits
|
||||
return 0, []
|
||||
|
||||
|
||||
def _rows(limit=400, tag=None, q=None, since_days=None):
|
||||
sql = "SELECT id, source, url, title, text, published, tags, score FROM items"
|
||||
where, args = [], []
|
||||
if q:
|
||||
where.append("(title LIKE ? OR text LIKE ? OR source LIKE ?)")
|
||||
args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q]
|
||||
if since_days:
|
||||
where.append("published >= date('now', ?)")
|
||||
args.append("-%d day" % int(since_days))
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY id DESC LIMIT ?"
|
||||
args.append(int(limit) * 6)
|
||||
con = sqlite3.connect(str(DB), timeout=30)
|
||||
try:
|
||||
cur = con.execute(sql, args)
|
||||
out = []
|
||||
for r in cur:
|
||||
sc, hits = _qualifies(r[1], r[3], r[4], r[6] or "")
|
||||
if not sc:
|
||||
continue
|
||||
if tag and tag not in hits:
|
||||
continue
|
||||
out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
|
||||
"text": (r[4] or "")[:600], "published": r[5],
|
||||
"tags": hits, "abuse_score": sc, "rank": r[7]})
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
|
||||
"""limit — сколько отдать, window — сколько последних записей просканировать."""
|
||||
items = _rows(limit=window, tag=tag, q=q, since_days=since_days)
|
||||
items.sort(key=lambda x: (-x["abuse_score"], -(x["id"] or 0)))
|
||||
counts = {}
|
||||
for it in items:
|
||||
for t in it["tags"]:
|
||||
counts[t] = counts.get(t, 0) + 1
|
||||
return {"items": items[:limit], "total": len(items), "tags": counts,
|
||||
"labels": TAG_LABEL}
|
||||
237
omni/abuse_sources.py
Normal file
237
omni/abuse_sources.py
Normal file
@@ -0,0 +1,237 @@
|
||||
"""Абузы: форумы, каналы и площадки с промо/сливами/триалами/ключами.
|
||||
|
||||
Сгенерировано из реестров парсера абузов; адаптер выведен из движка сайта,
|
||||
каждая строка проходит реальную проверку fetch перед включением.
|
||||
"""
|
||||
|
||||
|
||||
def A(name, category, kind, url, adapter, tags=(), limit=20):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url,
|
||||
"adapter": adapter, "tags": tuple(tags), "limit": limit, "detail": False, "extra": {}}
|
||||
|
||||
|
||||
ABUSE = [
|
||||
# ---- abuse (222) ----
|
||||
A("ab_lolz_live", "abuse", "html", "https://lolz.live/tags/rassylka/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crackingx_com", "abuse", "html", "https://crackingx.com/forums/5/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xreactor_org", "abuse", "html", "https://xreactor.org/forums/combolists.152/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crdpro_cc", "abuse", "html", "https://crdpro.cc/tags/tools/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_trafficultras_com", "abuse", "html", "https://trafficultras.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_blackhatworld_com", "abuse", "html", "https://www.blackhatworld.com/seo/up-to-date-list-of-specialized-software-for-telegram.1642970/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_addset_ru", "abuse", "html", "https://addset.ru/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_fb_killa_pro", "abuse", "html", "https://fb-killa.pro/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forum_x_com", "abuse", "html", "https://forum-x.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_afflift_com", "abuse", "html", "https://afflift.com/f/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cracked_io", "abuse", "html", "https://cracked.io/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_altenens_is", "abuse", "html", "https://altenens.is/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cardvilla_cc", "abuse", "html", "https://cardvilla.cc/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cybercarders_eu", "abuse", "html", "https://cybercarders.eu/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_shadowcarders_com", "abuse", "html", "https://shadowcarders.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_carders_biz", "abuse", "html", "https://carders.biz/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_ascarding_net", "abuse", "html", "https://ascarding.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xakerplus_com", "abuse", "html", "https://xakerplus.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_lozerix_com", "abuse", "html", "https://lozerix.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_zhyk_org", "abuse", "html", "https://zhyk.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_zenno_club", "abuse", "html", "https://zenno.club/discussion/tags/arbitrazh-trafika/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_partnerkin_com", "abuse", "html", "https://partnerkin.com/blog/articles", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_afftimes_com", "abuse", "html", "https://afftimes.com/articles/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpa_rip", "abuse", "html", "https://cpa.rip/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpa_live", "abuse", "html", "https://cpa.live/articles/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_gdetraffic_com", "abuse", "html", "https://gdetraffic.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_affy_group", "abuse", "html", "https://affy.group/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_conversion_im", "abuse", "html", "https://conversion.im/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_warriorforum_com", "abuse", "html", "https://www.warriorforum.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forums_digitalpoint_com", "abuse", "html", "https://forums.digitalpoint.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpaelites_com", "abuse", "html", "https://www.cpaelites.com/forum/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_hackforums_net", "abuse", "html", "https://hackforums.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cracking_org", "abuse", "html", "https://cracking.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crackingsoul_com", "abuse", "html", "https://crackingsoul.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tele_sender_com", "abuse", "html", "https://tele-sender.com/blog/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crmchat_ai", "abuse", "html", "https://crmchat.ai/blog/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_teleboost_app", "abuse", "html", "https://teleboost.app/blog/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_reddit_com", "abuse", "html", "https://www.reddit.com/r/TelegramBots/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_lzt_market", "abuse", "html", "https://lzt.market/telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_sourceforge_net", "abuse", "html", "https://sourceforge.net/directory/telegram-bots/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_t_me", "abuse", "html", "https://t.me/s/Abuz_ai_soft", "tg_web", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_habr_com", "abuse", "html", "https://habr.com/ru/search/?q=telegram+парсинг+софт&target_type=posts&order=date", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_teletype_in", "abuse", "html", "https://teletype.in/search?query=telegram+софт", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_youtube_com", "abuse", "html", "https://www.youtube.com/results?search_query=telegram+marketing+software+2026&sp=CAISBAgCEAE%253D", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_medium_com", "abuse", "html", "https://medium.com/search?q=telegram+bot+automation+2026", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tgstat_ru", "abuse", "html", "https://tgstat.ru/ratings/chats/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_carder_su", "abuse", "html", "https://carder.su/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_procarders_com", "abuse", "html", "https://procarders.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_darkpro_net", "abuse", "html", "https://darkpro.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tuts4you_com", "abuse", "html", "https://tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forum_tuts4you_com", "abuse", "html", "https://forum.tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forum_exetools_com", "abuse", "html", "https://forum.exetools.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tlgrm_eu", "abuse", "html", "https://tlgrm.eu/channels", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telemetr_me", "abuse", "html", "https://telemetr.me/channels", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpamafia_pro", "abuse", "html", "https://cpamafia.pro/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpamafia_top", "abuse", "html", "https://cpamafia.top/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_m_vk_com", "abuse", "html", "https://m.vk.com/search?c%5Bq%5D=продвижение+telegram+софт&c%5Bsection%5D=auto", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forobeta_com", "abuse", "html", "https://forobeta.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forexfactory_com", "abuse", "html", "https://www.forexfactory.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_gpwa_org", "abuse", "html", "https://www.gpwa.org/forum/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_casinomeister_com", "abuse", "html", "https://www.casinomeister.com/forums/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forums_babypips_com", "abuse", "html", "https://forums.babypips.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_github_com", "abuse", "html", "https://github.com/ofmtools/awesome-ofm-tools", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_searchengines_guru", "abuse", "html", "https://searchengines.guru/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_hardforum_com", "abuse", "html", "https://hardforum.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_bitcointalk_org", "abuse", "html", "https://bitcointalk.org/index.php?board=238.0", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_sendpulse_com", "abuse", "html", "https://sendpulse.com/blog/telegram-marketing-tools", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_softwaresuggest_com", "abuse", "html", "https://www.softwaresuggest.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tacticaldatamanager_com", "abuse", "html", "https://tacticaldatamanager.com/blog", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_statiko_io", "abuse", "html", "https://statiko.io/blog/telegram-anti-spam-bot", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tghero_pro", "abuse", "html", "https://tghero.pro/telegram-marketing-tools/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_enreach_ai", "abuse", "html", "https://www.enreach.ai/insights/best-telegram-outreach-tools", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nulledbb_com", "abuse", "html", "https://nulledbb.com/syndication.php?limit=50", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cracking_forums_com", "abuse", "html", "http://www.cracking-forums.com/syndication.php?limit=50", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_eleaks_to", "abuse", "html", "https://eleaks.to/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crackiansleaks_com", "abuse", "html", "https://crackiansleaks.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_niflheim_world", "abuse", "html", "https://niflheim.world/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nodeloc_com", "abuse", "json", "https://www.nodeloc.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_v2ex_com", "abuse", "json", "https://www.v2ex.com/api/topics/hot.json", "generic_json", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_hostloc_com", "abuse", "rss", "https://hostloc.com/forum.php?mod=rss", "rss", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_findaicredits_com", "abuse", "html", "https://www.findaicredits.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_bhf_io", "abuse", "html", "https://bhf.io/en-gb", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_leak_sx", "abuse", "html", "https://leak.sx/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forum_hashkiller_io", "abuse", "html", "https://forum.hashkiller.io/index.php", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_linux_do", "abuse", "json", "https://linux.do/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xss_is", "abuse", "html", "https://xss.is/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_qqshill_com", "abuse", "html", "https://qqshill.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_mavibot_ai", "abuse", "html", "https://mavibot.ai/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_kenzabyte_com", "abuse", "html", "https://www.kenzabyte.com/telegram-auto-account-creator-version-history/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_ltesocks_io", "abuse", "html", "https://ltesocks.io/blog/specific-examples-of-mobile-proxies-en-en/top-programs-for-automatic-telegram-account-registration/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_dicloak_com", "abuse", "html", "https://dicloak.com/blog-detail/the-ultimate-guide-to-using-a-telegram-bulk-account-creator-in-2026", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_multilogin_com", "abuse", "html", "https://multilogin.com/multiple-accounting/create-multiple-telegram-accounts/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_quackr_io", "abuse", "html", "https://quackr.io/blog/create-unlimited-telegram-accounts", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_genfarmer_com", "abuse", "html", "https://genfarmer.com/tool-for-mass-telegram-registration-and-automatic-telegram-account-nurturing-en/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_like_tg", "abuse", "html", "https://www.like.tg/blog/telegram-bulk-registration-tool-best-two-thousand-twenty-six-stable-solution-captcha-guide", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_dexpose_io", "abuse", "html", "https://www.dexpose.io/dark-web-forums/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cybelangel_com", "abuse", "html", "https://cybelangel.com/blog/dark-web-telegram-channels-attackers-use-most-2026/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxyscrape_com", "abuse", "html", "https://proxyscrape.com/online-proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_localtonet_com", "abuse", "html", "https://localtonet.com/tools/proxy-check", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxy_seller_com", "abuse", "html", "https://proxy-seller.com/tools/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_gologin_com", "abuse", "html", "https://gologin.com/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxyway_com", "abuse", "html", "https://proxyway.com/tools/free-online-proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_chromewebstore_google_com", "abuse", "html", "https://chromewebstore.google.com/detail/tg-member-extractor/gphncajfmfojhafagcemjjacbekahdoh", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_apify_com", "abuse", "html", "https://apify.com/agentx/telegram-member-scraper", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_energent_ai", "abuse", "html", "https://www.energent.ai/use-cases/en/telegram-scraper", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_thunderbit_com", "abuse", "html", "https://thunderbit.com/template/telegram-scraper", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_axiom_ai", "abuse", "html", "https://axiom.ai/scrape/telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_flozic_ai", "abuse", "html", "https://www.flozic.ai/agents/conversational/telegram-bot", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_manybot_io", "abuse", "html", "https://manybot.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_botmother_com", "abuse", "html", "https://botmother.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_chatloom_app", "abuse", "html", "https://chatloom.app/en/channels/telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_chatbotbuilder_net", "abuse", "html", "https://chatbotbuilder.net/telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_saascounter_com", "abuse", "html", "https://www.saascounter.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_schedulala_com", "abuse", "html", "https://schedulala.com/blog/telegram-marketing-guide", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_technologycounter_com", "abuse", "html", "https://technologycounter.com/telegram-marketing-software", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_omisoft_net", "abuse", "html", "https://omisoft.net/blog/telegram-mini-apps-for-business/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_adsly_pro", "abuse", "html", "https://adsly.pro/guides/managed-service-vs-saas-cost/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_optimum_web_com", "abuse", "html", "https://www.optimum-web.com/blog/telegram-ai-bot-for-business-2026/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegramsender_com", "abuse", "html", "https://telegramsender.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xreacher_com", "abuse", "html", "https://xreacher.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_zapier_com", "abuse", "html", "https://zapier.com/apps/outreach/integrations/telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_n8n_io", "abuse", "html", "https://n8n.io/workflows/13732-run-email-outreach-campaigns-from-telegram-with-gmail-and-google-sheets/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_albato_com", "abuse", "html", "https://albato.com/connect/outreach-with-telegramprivate", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_entergram_com", "abuse", "html", "https://www.entergram.com/blog/telegram-lead-management-complete-guide", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_breakcold_com", "abuse", "html", "https://www.breakcold.com/blog/crm-integration-with-telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_kommo_com", "abuse", "html", "https://www.kommo.com/integrations/telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_gramsales_com", "abuse", "html", "https://gramsales.com/blog/best-crm-tools-for-telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_synchubai_com", "abuse", "html", "https://synchubai.com/blog/telegram-messaging-crm/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_messaggio_com", "abuse", "html", "https://messaggio.com/messaging-in-telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_vc_ru", "abuse", "html", "https://vc.ru/id5905098/2879981-deskgram-2-moschnyj-soft-dlja-telegram-spama", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_unisender_com", "abuse", "html", "https://www.unisender.com/ru/blog/servisy-dlya-rassylki-v-telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_socradar_io", "abuse", "html", "https://socradar.io/blog/top-10-deep-web-and-dark-web-forums/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nordlayer_com", "abuse", "html", "https://nordlayer.com/blog/top-dark-web-telegram-groups-and-channels/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_play_google_com", "abuse", "html", "https://play.google.com/store/apps/details?id=com.boardgames.bd.mtproto", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxyempire_io", "abuse", "html", "https://proxyempire.io/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxybros_com", "abuse", "html", "https://proxybros.com/proxies/proxy-for-telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_adfasten_com", "abuse", "html", "https://www.adfasten.com/telegram-marketing", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_chatimize_com", "abuse", "html", "https://chatimize.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tg_on_com", "abuse", "html", "https://tg-on.com/articles/top-programm-rassylka-telegram.html", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_botract_com", "abuse", "html", "https://www.botract.com/blog/how-to-choose-telegram-bot-builder", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_respond_io", "abuse", "html", "https://respond.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_sendgecko_io", "abuse", "html", "https://sendgecko.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_community_thermaltake_com", "abuse", "html", "https://community.thermaltake.com/index.php?/topic/692572-the-ultimate-2026-guide-to-mass-telegram-account-creation-zero-bans/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_discuss_flarum_org", "abuse", "html", "https://discuss.flarum.org/d/39670-what-forum-spam-looks-like-in-2026-summer-edition", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_texapi_net", "abuse", "html", "https://texapi.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pixelscan_net", "abuse", "html", "https://pixelscan.net/blog/best-telegram-batch-registration-software-in-2026/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pipedream_com", "abuse", "html", "https://pipedream.com/docs", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_p1sms_kz", "abuse", "html", "https://p1sms.kz/telegram-rassylki", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nodemaven_com", "abuse", "html", "https://nodemaven.com/tools/proxy-bandwidth-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_infatica_io", "abuse", "html", "https://infatica.io/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_flowxo_com", "abuse", "html", "https://flowxo.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pas7_com_ua", "abuse", "html", "https://pas7.com.ua/blog/en/telegram-bot-development-cost-2026", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telelog_org", "abuse", "html", "https://telelog.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_help_nuelink_com", "abuse", "html", "https://help.nuelink.com/article/228-best-telegram-scheduling-tools", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_wooxy_com", "abuse", "html", "https://wooxy.com/telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_chatbots_studio", "abuse", "html", "https://chatbots.studio/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_app_flow_ai", "abuse", "html", "https://app.flow.ai", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_kelacyber_com", "abuse", "html", "https://www.kelacyber.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_infobip_com", "abuse", "html", "https://www.infobip.com/blog/best-marketing-automation-platforms", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_atompark_com", "abuse", "html", "https://www.atompark.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_foresiet_com", "abuse", "html", "https://foresiet.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_vovsoft_com", "abuse", "html", "https://vovsoft.com/software/proxy-checker/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pocodot_ai", "abuse", "html", "https://pocodot.ai/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_richads_com", "abuse", "html", "https://richads.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxyline_net", "abuse", "html", "https://proxyline.net/proxy-checker", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pypi_org", "abuse", "html", "https://pypi.org/project/TGSessionsConverter/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_opentele_readthedocs_io", "abuse", "html", "https://opentele.readthedocs.io/en/latest/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_growity_ai", "abuse", "html", "https://growity.ai/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_pantheon_run", "abuse", "html", "https://pantheon.run/learn/best-telegram-crm", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_platform_openai_com", "abuse", "html", "https://platform.openai.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_dev_to", "abuse", "html", "https://dev.to/scofieldidehen/if-you-use-telegram-in-2026-read-this-now-4770", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_codester_com", "abuse", "html", "https://www.codester.com/items/54880/telegram-marketing-tools-bot", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_greasyfork_org", "abuse", "html", "https://greasyfork.org/en/scripts/538176-telegram-scraper-menu-commands-v2-3-3-human-like-defaults", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_medevel_com", "abuse", "html", "https://medevel.com/telegram-scraper-20/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_blog_checkpoint_com", "abuse", "html", "https://blog.checkpoint.com/research/telegrams-crackdown-in-2026-and-why-cyber-criminals-are-still-winning/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_make_com", "abuse", "html", "https://www.make.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_gopulsar_io", "abuse", "html", "https://gopulsar.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tgproxy_io", "abuse", "html", "https://tgproxy.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_texsender_net", "abuse", "html", "https://texsender.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_communicat_o_com", "abuse", "html", "https://www.communicat-o.com/crm-integration-for-telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegramscraper_shop", "abuse", "html", "https://telegramscraper.shop/blog/best-telegram-scraper-2026", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxyhorizon_com", "abuse", "html", "https://www.proxyhorizon.com/blog/proxies-for-telegram", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_ctiacademy_io", "abuse", "html", "https://ctiacademy.io/blog/telegram-cybercrime-marketplace", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tgsender_wadesk_io", "abuse", "html", "https://tgsender.wadesk.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_coronium_io", "abuse", "html", "https://www.coronium.io/partners/software/telegram-prime-review", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_stackoverflow_com", "abuse", "html", "https://stackoverflow.com/questions/61657596/telethon-first-steps-in-python", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tgninja_com", "abuse", "html", "https://tgninja.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_malware_news", "abuse", "html", "https://malware.news/t/top-dark-web-telegram-groups-channels-2026/104664", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telega_io", "abuse", "html", "https://telega.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_fiverr_com", "abuse", "html", "https://www.fiverr.com/gigs/telegram-invite", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_quora_com", "abuse", "html", "https://www.quora.com/What-bulk-messaging-software-is-best-for-Telegram-Line-and-WhatsApp", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_bolster_ai", "abuse", "html", "https://bolster.ai/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_capterra_com", "abuse", "html", "https://www.capterra.com/p/180347/Telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_manychat_com", "abuse", "html", "https://manychat.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_outpost24_com", "abuse", "html", "https://outpost24.com/blog/carding-ecosystem-fall-financial-cybercrime/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegramvpn_org", "abuse", "html", "https://telegramvpn.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_iricrm_com", "abuse", "html", "https://iricrm.com/telegram-crm/?lang=en", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_deepstrike_io", "abuse", "html", "https://deepstrike.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_stealthmole_com", "abuse", "html", "https://www.stealthmole.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_f6s_com", "abuse", "html", "https://www.f6s.com/software/category/no-code-telegram-bot-builder", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_exploit_in", "abuse", "html", "https://exploit.in/challenge/?return=/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_checkpoint_com", "abuse", "html", "https://www.checkpoint.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_blog_invitemember_com", "abuse", "html", "https://blog.invitemember.com/5-essential-telegram-bots-for-creators/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tracexlabs_com", "abuse", "html", "https://tracexlabs.com/knowledge-base/top-10-deep-web-and-dark-web-forums.html", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_proxies_sx", "abuse", "html", "https://www.proxies.sx/partners/automation-platforms/telegram-expert", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_checkproxy_org", "abuse", "html", "https://checkproxy.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_automatedclientacquisition_com", "abuse", "html", "https://www.automatedclientacquisition.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_stormycloud_org", "abuse", "html", "https://stormycloud.org/mtproto/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_bot_market_net", "abuse", "html", "https://bot-market.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telebotbuilder_com", "abuse", "html", "https://telebotbuilder.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cavyro_com", "abuse", "html", "https://cavyro.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_positechglobal_com", "abuse", "html", "https://positechglobal.com/en/research/analytics/cybercriminal-market-in-telegram/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_trypromptlab_com", "abuse", "html", "https://trypromptlab.com/blog/best-crm-with-telegram-integration", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_flowcastle_ai", "abuse", "html", "https://flowcastle.ai/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegramsoftware_pro", "abuse", "html", "https://telegramsoftware.pro/blog/telegram-session-to-tdata-converter/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_darkbase_cc", "abuse", "html", "https://darkbase.cc/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nocturne_cards", "abuse", "html", "https://nocturne.cards/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_ads_telegram_org", "abuse", "html", "https://ads.telegram.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_teleform_io", "abuse", "html", "https://www.teleform.io/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
]
|
||||
217
omni/bot.py
Normal file
217
omni/bot.py
Normal file
@@ -0,0 +1,217 @@
|
||||
"""Telegram-бот поверх парсера: лента, абузы, источники, прогон, автодобавление.
|
||||
|
||||
Запуск: OMNI_BOT_TOKEN=... python -m omni.bot
|
||||
Доступ: OMNI_BOT_OWNERS=7448683285,123 (если пусто — только первый написавший)
|
||||
|
||||
Бот не постит никуда сам: только отвечает в личный чат тем, кто в списке.
|
||||
Команды:
|
||||
/new [n] последние записи
|
||||
/abuse [тег] абузы: promo trial keys leak tool resale
|
||||
/find <текст> поиск по базе
|
||||
/src <текст> поиск источника
|
||||
/add <url> добавить источник (сначала реальная проверка фида)
|
||||
/run <категория> прогнать категорию
|
||||
/auto [n] автодобавление новых фидов
|
||||
/stats цифры
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
from . import abuse as abuse_mod
|
||||
from . import discover as discover_mod
|
||||
from . import feeds, net, source_state, sources, store
|
||||
|
||||
API = "https://api.telegram.org/bot%s/" % os.environ.get("OMNI_BOT_TOKEN", "")
|
||||
TOKEN = os.environ.get("OMNI_BOT_TOKEN", "")
|
||||
OWNERS = {int(x) for x in re.findall(r"\d{5,}", os.environ.get("OMNI_BOT_OWNERS", ""))}
|
||||
MAX_LEN = 3600
|
||||
|
||||
|
||||
def _api(method, **params):
|
||||
if not TOKEN:
|
||||
raise SystemExit("OMNI_BOT_TOKEN не задан")
|
||||
url = API + method
|
||||
data = urllib.parse.urlencode(params).encode()
|
||||
r = net.get(url, headers={"Content-Type": "application/x-www-form-urlencoded"}) if False else None
|
||||
req = urllib.request.Request(url, data=data)
|
||||
with urllib.request.urlopen(req, timeout=40) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def _send(chat_id, text, preview=True):
|
||||
text = text[:MAX_LEN]
|
||||
return _api("sendMessage", chat_id=chat_id, text=text, parse_mode="HTML",
|
||||
disable_web_page_preview="false" if preview else "true")
|
||||
|
||||
|
||||
def _esc(s):
|
||||
return html.escape(str(s or ""), quote=False)
|
||||
|
||||
|
||||
def fmt_item(it):
|
||||
title = _esc((it.get("title") or "")[:150])
|
||||
src = _esc(it.get("source") or "")
|
||||
url = it.get("url") or ""
|
||||
line = "<b>%s</b>\n<code>%s</code>" % (title, src)
|
||||
if url:
|
||||
line += ' · <a href="%s">открыть</a>' % _esc(url)
|
||||
return line
|
||||
|
||||
|
||||
def cmd_start(chat_id, arg):
|
||||
_send(chat_id, "<b>omni</b> — парсер-радар\n\n"
|
||||
"/new 5 — последние записи\n"
|
||||
"/abuse promo — абузы по тегу\n"
|
||||
"/find claude — поиск\n"
|
||||
"/src reddit — источник\n"
|
||||
"/add <url> — подключить фид\n"
|
||||
"/run abuse — прогнать категорию\n"
|
||||
"/auto 10 — автодобавление\n"
|
||||
"/stats — цифры")
|
||||
|
||||
|
||||
def cmd_new(chat_id, arg):
|
||||
n = int(arg) if arg.strip().isdigit() else 8
|
||||
rows = store.browse2(limit=min(n, 20), sort="fresh")
|
||||
items = rows.get("items", rows) if isinstance(rows, dict) else rows
|
||||
if not items:
|
||||
return _send(chat_id, "пусто")
|
||||
_send(chat_id, "последние %d:\n\n" % len(items) + "\n\n".join(fmt_item(i) for i in items))
|
||||
|
||||
|
||||
def cmd_abuse(chat_id, arg):
|
||||
tag = arg.strip().split()[0] if arg.strip() else None
|
||||
d = abuse_mod.scan(limit=10, tag=tag)
|
||||
if not d["items"]:
|
||||
return _send(chat_id, "абузов не видно")
|
||||
head = "абузы · %s · %d\n\n" % ("все" if not tag else tag, d["total"])
|
||||
body = []
|
||||
for it in d["items"]:
|
||||
body.append("<b>%s</b>\n<code>%s</code> · %s" % (
|
||||
_esc(it["title"][:120]), _esc(it["source"]), _esc(",".join(it["tags"]))))
|
||||
_send(chat_id, head + "\n\n".join(body))
|
||||
|
||||
|
||||
def cmd_find(chat_id, arg):
|
||||
q = arg.strip()
|
||||
if not q:
|
||||
return _send(chat_id, "что искать?")
|
||||
rows = store.search(q, limit=8)
|
||||
items = rows.get("items", rows) if isinstance(rows, dict) else rows
|
||||
if not items:
|
||||
return _send(chat_id, "нет совпадений")
|
||||
_send(chat_id, "поиск: %s\n\n" % _esc(q) + "\n\n".join(fmt_item(i) for i in items))
|
||||
|
||||
|
||||
def cmd_src(chat_id, arg):
|
||||
q = arg.strip().lower()
|
||||
all_s = sources.all_sources()
|
||||
hits = [s for s in all_s if q and (q in s.name.lower() or q in s.url.lower() or q in s.category.lower())]
|
||||
if not hits:
|
||||
return _send(chat_id, "нет такого источника")
|
||||
out = ["<b>%s</b> · %s · %s" % (_esc(s.name), _esc(s.category), _esc(s.adapter)) for s in hits[:12]]
|
||||
_send(chat_id, "источники (%d):\n\n" % len(hits) + "\n".join(out))
|
||||
|
||||
|
||||
def cmd_add(chat_id, arg):
|
||||
url = arg.strip()
|
||||
if not url.startswith("http"):
|
||||
return _send(chat_id, "нужен url фида")
|
||||
ok, n, ad = discover_mod.probe(url, "community", limit=6)
|
||||
if not ok:
|
||||
return _send(chat_id, "фид пустой или недоступен: %s" % _esc(ad))
|
||||
name = "manual_" + re.sub(r"[^a-z0-9]+", "_", urllib.parse.urlsplit(url).netloc.lower())[:32]
|
||||
source_state.add_custom(name, "community", url, kind="rss" if ad == "rss" else "html",
|
||||
adapter=ad, tags=["manual"], limit=15)
|
||||
_send(chat_id, "добавлен <code>%s</code> · %s · %d записей" % (_esc(name), _esc(ad), n))
|
||||
|
||||
|
||||
def cmd_run(chat_id, arg):
|
||||
cat = arg.strip() or "abuse"
|
||||
srcs = [s for s in sources.all_sources() if s.category == cat]
|
||||
if not srcs:
|
||||
return _send(chat_id, "нет категории %s" % _esc(cat))
|
||||
_send(chat_id, "прогон %s (%d источников)…" % (_esc(cat), len(srcs)))
|
||||
got = new = 0
|
||||
for s in srcs[:80]:
|
||||
try:
|
||||
items = feeds.fetch(s, 10)
|
||||
except Exception:
|
||||
items = []
|
||||
got += len(items)
|
||||
if items:
|
||||
new += store.add(items)
|
||||
_send(chat_id, "готово: собрано %d, новых %d" % (got, new))
|
||||
|
||||
|
||||
def cmd_auto(chat_id, arg):
|
||||
n = int(arg) if arg.strip().isdigit() else 15
|
||||
res = discover_mod.discover(limit_hosts=30, workers=8, max_add=min(n, 30))
|
||||
_send(chat_id, "автодобавление: кандидатов %d, живых %d\n%s" % (
|
||||
res["found"], res["alive"], _esc(", ".join(res["added"][:10]))))
|
||||
|
||||
|
||||
def cmd_stats(chat_id, arg):
|
||||
s = store.stats()
|
||||
a = sources.all_sources()
|
||||
alive = source_state.alive_count() if hasattr(source_state, "alive_count") else "?"
|
||||
_send(chat_id, "<b>omni</b>\nзаписей %s · источников %d · живых %s\nкатегорий %d · прогонов %s · с ценой %s" % (
|
||||
s.get("items"), len(a), alive, len(sources.CATEGORIES), s.get("runs"), s.get("priced")))
|
||||
|
||||
|
||||
CMDS = {"/start": cmd_start, "/help": cmd_start, "/new": cmd_new, "/abuse": cmd_abuse,
|
||||
"/find": cmd_find, "/src": cmd_src, "/add": cmd_add, "/run": cmd_run,
|
||||
"/auto": cmd_auto, "/stats": cmd_stats}
|
||||
|
||||
|
||||
def handle(update):
|
||||
msg = update.get("message") or update.get("edited_message") or {}
|
||||
chat = msg.get("chat") or {}
|
||||
chat_id = chat.get("id")
|
||||
text = (msg.get("text") or "").strip()
|
||||
uid = (msg.get("from") or {}).get("id")
|
||||
if not chat_id or not text:
|
||||
return
|
||||
if OWNERS and uid not in OWNERS:
|
||||
return
|
||||
if not OWNERS:
|
||||
OWNERS.add(uid) # первый написавший становится владельцем до рестарта
|
||||
parts = text.split(None, 1)
|
||||
cmd = parts[0].lower().split("@")[0]
|
||||
arg = parts[1] if len(parts) > 1 else ""
|
||||
fn = CMDS.get(cmd)
|
||||
if not fn:
|
||||
return _send(chat_id, "не знаю такую команду. /help")
|
||||
try:
|
||||
fn(chat_id, arg)
|
||||
except Exception as e:
|
||||
_send(chat_id, "ошибка: %s" % _esc(type(e).__name__ + ": " + str(e)[:160]))
|
||||
|
||||
|
||||
def run_forever():
|
||||
if not TOKEN:
|
||||
raise SystemExit("нужен OMNI_BOT_TOKEN")
|
||||
offset = 0
|
||||
print("bot up", flush=True)
|
||||
while True:
|
||||
try:
|
||||
r = _api("getUpdates", offset=offset, timeout=30)
|
||||
except Exception as e:
|
||||
print("poll fail", e, flush=True)
|
||||
time.sleep(5)
|
||||
continue
|
||||
for up in r.get("result", []):
|
||||
offset = up["update_id"] + 1
|
||||
handle(up)
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_forever()
|
||||
164
omni/discover.py
Normal file
164
omni/discover.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Автодобавление источников: находит фиды по домашним страницам и каталогам.
|
||||
|
||||
Логика: берём затравки (хосты уже подключённых источников + известные каталоги
|
||||
сообществ), вытаскиваем ссылки на фиды (link rel=alternate, /feed, /rss,
|
||||
/latest.json, подфорумы), прогоняем каждый кандидат реальным адаптером и
|
||||
добавляем только те, что отдали записи. Всё добавленное помечается auto и
|
||||
попадает в панель как обычный источник.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from . import feeds, net, source_state, sources
|
||||
|
||||
FEED_PATHS = ("/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml",
|
||||
"/latest.json", "/?feed=rss2", "/feed/", "/blog/feed")
|
||||
SEED_CATALOGS = [
|
||||
# каталоги сообществ и подфорумов, откуда удобно цеплять новые фиды
|
||||
"https://www.reddit.com/r/LocalLLaMA/about.json",
|
||||
"https://news.ycombinator.com/",
|
||||
"https://lobste.rs/",
|
||||
]
|
||||
UA = getattr(feeds, "UA", {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
|
||||
|
||||
|
||||
def _hosts():
|
||||
seen = {}
|
||||
for s in sources.all_sources():
|
||||
try:
|
||||
h = urllib.parse.urlsplit(s.url).netloc.lower()
|
||||
except Exception:
|
||||
continue
|
||||
if not h or h in seen:
|
||||
continue
|
||||
seen[h] = s.category
|
||||
return seen
|
||||
|
||||
|
||||
def _feed_links(html, base):
|
||||
out = []
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for ln in soup.find_all("link", href=True):
|
||||
t = (ln.get("type") or "").lower()
|
||||
rel = " ".join(ln.get("rel") or []).lower()
|
||||
if "alternate" in rel and ("rss" in t or "atom" in t or "json" in t):
|
||||
out.append(urllib.parse.urljoin(base, ln["href"]))
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = str(a["href"])
|
||||
if re.search(r"(/feed|/rss|\.rss|atom\.xml|/latest\.json)$", href):
|
||||
out.append(urllib.parse.urljoin(base, href))
|
||||
return out
|
||||
|
||||
|
||||
def candidates(limit_hosts=60, extra_seeds=()):
|
||||
hosts = _hosts()
|
||||
seeds = []
|
||||
for h, cat in list(hosts.items())[:limit_hosts]:
|
||||
seeds.append(("https://%s/" % h, cat))
|
||||
for u in list(extra_seeds) + SEED_CATALOGS:
|
||||
seeds.append((u, "auto"))
|
||||
return seeds
|
||||
|
||||
|
||||
def probe(url, category="auto", limit=6):
|
||||
"""Проверка одного URL реальным адаптером. Возвращает (ok, n, adapter)."""
|
||||
low = url.lower()
|
||||
if "t.me/s/" in low:
|
||||
ad, kind = "tg_web", "html"
|
||||
elif low.endswith(".json") or "/latest.json" in low:
|
||||
ad, kind = ("discourse", "json") if "/latest.json" in low else ("generic_json", "json")
|
||||
elif any(x in low for x in (".rss", "/rss", "/feed", "atom.xml", "feed.xml", "index.xml")):
|
||||
ad, kind = "rss", "rss"
|
||||
else:
|
||||
ad, kind = "html_listing", "html"
|
||||
src = sources.Source(name="probe", category=category, kind=kind, url=url,
|
||||
adapter=ad, tags=("auto",), limit=limit, detail=False, extra={})
|
||||
fn = feeds.ADAPTERS.get(ad)
|
||||
if not fn:
|
||||
return False, 0, ad
|
||||
try:
|
||||
items = fn(src, limit) or []
|
||||
except Exception:
|
||||
return False, 0, ad
|
||||
return len(items) > 0, len(items), ad
|
||||
|
||||
|
||||
def discover(limit_hosts=40, workers=8, max_add=40, dry=False):
|
||||
seeds = candidates(limit_hosts=limit_hosts)
|
||||
found = []
|
||||
def scan(seed):
|
||||
base, cat = seed
|
||||
try:
|
||||
r = net.get(base, headers=UA)
|
||||
if r.status != 200:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
links = _feed_links(r.text[:400000], base)
|
||||
dom = urllib.parse.urlsplit(base)
|
||||
for p in FEED_PATHS:
|
||||
links.append("%s://%s%s" % (dom.scheme, dom.netloc, p))
|
||||
# подфорумы Discourse/Reddit
|
||||
if "reddit.com" in base:
|
||||
for sub in ("LocalLLaMA", "MachineLearning", "mcp", "selfhosted"):
|
||||
links.append("https://www.reddit.com/r/%s/.rss" % sub)
|
||||
return [(u, cat) for u in dict.fromkeys(links)]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for res in ex.map(scan, seeds):
|
||||
found.extend(res)
|
||||
print("кандидатов найдено:", len(found), flush=True)
|
||||
|
||||
known = {s.url for s in sources.all_sources()}
|
||||
found = [(u, c) for u, c in found if u not in known]
|
||||
print("после дедупа:", len(found), flush=True)
|
||||
|
||||
def check(pair):
|
||||
url, cat = pair
|
||||
ok, n, ad = probe(url, cat)
|
||||
return url, cat, ok, n, ad
|
||||
|
||||
# только машинные фиды: html-листинги на автодобавлении дают мусор
|
||||
best = {}
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for url, cat, ok, n, ad in ex.map(check, found):
|
||||
if not ok or ad == "html_listing" or n < 3:
|
||||
continue
|
||||
dom = urllib.parse.urlsplit(url).netloc.lower()
|
||||
key = (dom, ad)
|
||||
if key not in best or n > best[key][2]:
|
||||
best[key] = (url, cat, n, ad)
|
||||
|
||||
added = []
|
||||
for (dom, ad), (url, cat, n, _ad) in sorted(best.items(), key=lambda kv: -kv[1][2]):
|
||||
name = "auto_" + re.sub(r"[^a-z0-9]+", "_", dom).strip("_")[:34]
|
||||
if any(a["name"] == name for a in added):
|
||||
continue
|
||||
rec = {"name": name, "category": cat if cat in ("abuse", "forum", "security") else "community",
|
||||
"kind": "rss" if ad == "rss" else ("json" if ad in ("generic_json", "discourse") else "html"),
|
||||
"url": url, "adapter": ad, "tags": ["auto"], "lim": 15, "detail": False}
|
||||
added.append(rec)
|
||||
if len(added) >= max_add:
|
||||
break
|
||||
print("живых фидов:", len(added), flush=True)
|
||||
if not dry:
|
||||
for rec in added:
|
||||
source_state.add_custom(rec["name"], rec["category"], rec["url"], kind=rec["kind"],
|
||||
adapter=rec["adapter"], tags=rec["tags"], limit=rec["lim"],
|
||||
detail=rec["detail"])
|
||||
return {"found": len(found), "alive": len(added), "added": added if dry else [a["name"] for a in added]}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
lim = int(sys.argv[1]) if len(sys.argv) > 1 else 30
|
||||
res = discover(limit_hosts=lim)
|
||||
print(json.dumps({"found": res["found"], "alive": res["alive"], "added": res["added"][:20]},
|
||||
ensure_ascii=False, indent=1))
|
||||
@@ -336,7 +336,8 @@ def _xenforo_latest(src, limit):
|
||||
if len(title) < 12 or href.rstrip("/").endswith("/threads"):
|
||||
continue
|
||||
if not href.startswith("http"):
|
||||
href = "https://www.blackhatworld.com" + href
|
||||
parts = urllib.parse.urlsplit(src.url)
|
||||
href = "%s://%s%s" % (parts.scheme, parts.netloc, href)
|
||||
out.append(_item(src, "thread", href, title))
|
||||
if len(out) >= limit:
|
||||
break
|
||||
|
||||
@@ -190,6 +190,42 @@ class Handler(BaseHTTPRequestHandler):
|
||||
"categories": cats,
|
||||
})
|
||||
|
||||
if path == "/api/runs":
|
||||
try:
|
||||
limit = max(1, min(500, int((q.get("limit") or ["80"])[0])))
|
||||
except ValueError:
|
||||
limit = 80
|
||||
import time as _t
|
||||
con = store._con() if hasattr(store, "_con") else None
|
||||
rows = []
|
||||
try:
|
||||
import sqlite3 as _s3
|
||||
from .config import DB as _DB
|
||||
db = _s3.connect(str(_DB), timeout=30)
|
||||
db.row_factory = _s3.Row
|
||||
for r in db.execute("SELECT id, ts, provider, source, query, got, kept, status, bytes "
|
||||
"FROM runs ORDER BY id DESC LIMIT ?", (limit,)):
|
||||
d = dict(r)
|
||||
ts = d.get("ts") or 0
|
||||
d["when"] = _t.strftime("%Y-%m-%d %H:%M", _t.localtime(ts)) if ts else ""
|
||||
rows.append(d)
|
||||
total = db.execute("SELECT count(*) FROM runs").fetchone()[0]
|
||||
db.close()
|
||||
except Exception as e:
|
||||
return self._json({"error": "runs: %s" % e})
|
||||
return self._json({"items": rows, "total": total, "shown": len(rows)})
|
||||
|
||||
if path == "/api/abuse":
|
||||
from . import abuse as _ab
|
||||
tag = (q.get("tag") or [""])[0].strip() or None
|
||||
if tag in ("all", "все"):
|
||||
tag = None
|
||||
try:
|
||||
lim = max(1, min(200, int((q.get("limit") or ["40"])[0])))
|
||||
except ValueError:
|
||||
lim = 40
|
||||
return self._json(_ab.scan(limit=lim, tag=tag))
|
||||
|
||||
if path == "/api/sources":
|
||||
cat = (q.get("category") or [None])[0]
|
||||
query = (q.get("q") or [""])[0].strip().lower()
|
||||
@@ -259,6 +295,15 @@ class Handler(BaseHTTPRequestHandler):
|
||||
except Exception:
|
||||
body = {}
|
||||
|
||||
if u.path == "/api/discover":
|
||||
from . import discover as _d
|
||||
q = urllib.parse.parse_qs(u.query)
|
||||
hosts = max(5, min(120, int((q.get("hosts") or ["40"])[0])))
|
||||
add = max(1, min(60, int((q.get("add") or ["25"])[0])))
|
||||
dry = (q.get("dry") or ["0"])[0] not in ("0", "false", "")
|
||||
res = _d.discover(limit_hosts=hosts, workers=8, max_add=add, dry=dry)
|
||||
return self._json(res)
|
||||
|
||||
if u.path == "/api/feed":
|
||||
jid = _jid("feed")
|
||||
_job(jid, state="running")
|
||||
|
||||
@@ -12,6 +12,8 @@ from dataclasses import dataclass, field
|
||||
|
||||
from .sources_ext import EXTRA
|
||||
from .sources_ext2 import EXTRA2
|
||||
from .sources_ext3 import EXTRA3
|
||||
from .sources_abuse import ABUSE_LIVE
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -188,7 +190,7 @@ def _dedup(rows):
|
||||
return out
|
||||
|
||||
|
||||
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2])
|
||||
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE])
|
||||
|
||||
BY_NAME = {s.name: s for s in SOURCES}
|
||||
CATEGORIES = sorted({s.category for s in SOURCES})
|
||||
|
||||
45
omni/sources_abuse.py
Normal file
45
omni/sources_abuse.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Абузы (живые): форумы, каналы и площадки с промо/сливами/софтом.
|
||||
|
||||
Только структурные движки (RSS/Discourse/XenForo/Telegram) и HTML-форумы,
|
||||
у которых реально нашлись темы (>=5 ссылок на треды), а не навигация.
|
||||
"""
|
||||
|
||||
|
||||
def A(name, category, kind, url, adapter, tags=(), limit=20):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url,
|
||||
"adapter": adapter, "tags": tuple(tags), "limit": limit, "detail": False, "extra": {}}
|
||||
|
||||
|
||||
ABUSE_LIVE = [
|
||||
A("ab_lolz_live", "abuse", "html", "https://lolz.live/tags/rassylka/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crackingx_com", "abuse", "html", "https://crackingx.com/forums/5/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xreactor_org", "abuse", "html", "https://xreactor.org/forums/combolists.152/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_crdpro_cc", "abuse", "html", "https://crdpro.cc/tags/tools/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_blackhatworld_com", "abuse", "html", "https://www.blackhatworld.com/seo/up-to-date-list-of-specialized-software-for-telegram.1642970/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cracked_io", "abuse", "html", "https://cracked.io/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_t_me", "abuse", "html", "https://t.me/s/Abuz_ai_soft", "tg_web", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_tuts4you_com", "abuse", "html", "https://tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forum_tuts4you_com", "abuse", "html", "https://forum.tuts4you.com/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_nodeloc_com", "abuse", "json", "https://www.nodeloc.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_v2ex_com", "abuse", "json", "https://www.v2ex.com/api/topics/hot.json", "generic_json", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_hostloc_com", "abuse", "rss", "https://hostloc.com/forum.php?mod=rss", "rss", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_linux_do", "abuse", "json", "https://linux.do/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_xss_is", "abuse", "html", "https://xss.is/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_exploit_in", "abuse", "html", "https://exploit.in/challenge/?return=/", "xenforo_latest", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_fb_killa_pro", "abuse", "html", "https://fb-killa.pro/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_afflift_com", "abuse", "html", "https://afflift.com/f/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_altenens_is", "abuse", "html", "https://altenens.is/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_carders_biz", "abuse", "html", "https://carders.biz/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_ascarding_net", "abuse", "html", "https://ascarding.net/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_zhyk_org", "abuse", "html", "https://zhyk.org/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_zenno_club", "abuse", "html", "https://zenno.club/discussion/tags/arbitrazh-trafika/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_carder_su", "abuse", "html", "https://carder.su/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_cpamafia_pro", "abuse", "html", "https://cpamafia.pro/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_forexfactory_com", "abuse", "html", "https://www.forexfactory.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_casinomeister_com", "abuse", "html", "https://www.casinomeister.com/forums/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_hardforum_com", "abuse", "html", "https://hardforum.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_niflheim_world", "abuse", "html", "https://niflheim.world/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20),
|
||||
]
|
||||
112
omni/sources_ext3.py
Normal file
112
omni/sources_ext3.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Волна 4 (только живые после реального fetch): релизы репозиториев, рассылки,
|
||||
Discourse-форумы, подреддиты, CN/JP блоги, HF-задачи. Каждая строка проверена: items > 0."""
|
||||
|
||||
|
||||
def S(name, category, kind, url, adapter="rss", tags=(), limit=20, detail=False, extra=None):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
||||
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
||||
|
||||
|
||||
EXTRA3 = [
|
||||
# ---- release ----
|
||||
S("gh_rel_ollama", "release", "atom", "https://github.com/ollama/ollama/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_llamacpp", "release", "atom", "https://github.com/ggml-org/llama.cpp/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_vllm", "release", "atom", "https://github.com/vllm-project/vllm/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_comfyui", "release", "atom", "https://github.com/comfyanonymous/ComfyUI/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_dify", "release", "atom", "https://github.com/langgenius/dify/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_n8n", "release", "atom", "https://github.com/n8n-io/n8n/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_langchain", "release", "atom", "https://github.com/langchain-ai/langchain/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_autogen", "release", "atom", "https://github.com/microsoft/autogen/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_litellm", "release", "atom", "https://github.com/BerriAI/litellm/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_openwebui", "release", "atom", "https://github.com/open-webui/open-webui/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_ragflow", "release", "atom", "https://github.com/infiniflow/ragflow/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_browseruse", "release", "atom", "https://github.com/browser-use/browser-use/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_playwrightmcp", "release", "atom", "https://github.com/microsoft/playwright-mcp/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_mcpservers", "release", "atom", "https://github.com/modelcontextprotocol/servers/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_dspy", "release", "atom", "https://github.com/stanfordnlp/dspy/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_transformers", "release", "atom", "https://github.com/huggingface/transformers/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_trl", "release", "atom", "https://github.com/huggingface/trl/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_sglang", "release", "atom", "https://github.com/sgl-project/sglang/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_unsloth", "release", "atom", "https://github.com/unslothai/unsloth/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_axolotl", "release", "atom", "https://github.com/axolotl-ai-cloud/axolotl/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_markitdown", "release", "atom", "https://github.com/microsoft/markitdown/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_openai_sdk", "release", "atom", "https://github.com/openai/openai-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_anthropic_sdk", "release", "atom", "https://github.com/anthropics/anthropic-sdk-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_gemini_sdk", "release", "atom", "https://github.com/google-gemini/generative-ai-python/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_crewai", "release", "atom", "https://github.com/crewAIInc/crewAI/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_agno", "release", "atom", "https://github.com/agno-agi/agno/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_livekit_agents", "release", "atom", "https://github.com/livekit/agents/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_mem0", "release", "atom", "https://github.com/mem0ai/mem0/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_qdrant", "release", "atom", "https://github.com/qdrant/qdrant/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
S("gh_rel_chroma", "release", "atom", "https://github.com/chroma-core/chroma/releases.atom", "rss", ('release', 'github', 'update'), limit=10, detail=False),
|
||||
# ---- community ----
|
||||
S("lesswrong", "community", "rss", "https://www.lesswrong.com/feed.xml?view=curated-rss", "rss", ('ai', 'alignment'), limit=20, detail=False),
|
||||
S("alignmentforum", "community", "rss", "https://www.alignmentforum.org/feed.xml?view=curated-rss", "rss", ('ai', 'alignment'), limit=20, detail=False),
|
||||
S("gwern", "community", "rss", "https://gwern.net/feed", "rss", ('ai', 'longform'), limit=20, detail=False),
|
||||
S("importai", "community", "rss", "https://importai.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False),
|
||||
S("thezvi", "community", "rss", "https://thezvi.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False),
|
||||
S("latentspace", "community", "rss", "https://www.latent.space/feed", "rss", ('ai', 'podcast'), limit=20, detail=False),
|
||||
S("aisupremacy", "community", "rss", "https://aisupremacy.substack.com/feed", "rss", ('ai', 'newsletter'), limit=20, detail=False),
|
||||
S("oneusefulthing", "community", "rss", "https://www.oneusefulthing.org/feed", "rss", ('ai', 'practices'), limit=20, detail=False),
|
||||
S("chinatalk", "community", "rss", "https://www.chinatalk.media/feed", "rss", ('ai', 'china'), limit=20, detail=False),
|
||||
S("reddit_claudeai", "community", "rss", "https://old.reddit.com/r/ClaudeAI/.rss", "reddit_listing", ('reddit', 'claude'), limit=20, detail=False),
|
||||
S("reddit_bard", "community", "rss", "https://old.reddit.com/r/Bard/.rss", "reddit_listing", ('reddit', 'gemini'), limit=20, detail=False),
|
||||
S("reddit_singularity", "community", "rss", "https://old.reddit.com/r/singularity/.rss", "reddit_listing", ('reddit', 'ai'), limit=20, detail=False),
|
||||
S("reddit_openai_dev", "community", "rss", "https://old.reddit.com/r/OpenAIDev/.rss", "reddit_listing", ('reddit', 'openai'), limit=20, detail=False),
|
||||
S("reddit_mcp", "community", "rss", "https://old.reddit.com/r/mcp/.rss", "reddit_listing", ('reddit', 'mcp'), limit=20, detail=False),
|
||||
S("reddit_ai_agents", "community", "rss", "https://old.reddit.com/r/AI_Agents/.rss", "reddit_listing", ('reddit', 'agents'), limit=20, detail=False),
|
||||
S("reddit_bugbounty", "community", "rss", "https://old.reddit.com/r/bugbounty/.rss", "reddit_listing", ('reddit', 'bugbounty'), limit=20, detail=False),
|
||||
S("reddit_netsecstudents", "community", "rss", "https://old.reddit.com/r/netsecstudents/.rss", "reddit_listing", ('reddit', 'security'), limit=20, detail=False),
|
||||
S("reddit_blueteamsec", "community", "rss", "https://old.reddit.com/r/blueteamsec/.rss", "reddit_listing", ('reddit', 'security'), limit=20, detail=False),
|
||||
S("reddit_prompteng", "community", "rss", "https://old.reddit.com/r/PromptEngineering/.rss", "reddit_listing", ('reddit', 'prompts'), limit=20, detail=False),
|
||||
S("reddit_oobabooga", "community", "rss", "https://old.reddit.com/r/Oobabooga/.rss", "reddit_listing", ('reddit', 'llm'), limit=20, detail=False),
|
||||
S("reddit_unsloth", "community", "rss", "https://old.reddit.com/r/unsloth/.rss", "reddit_listing", ('reddit', 'finetune'), limit=20, detail=False),
|
||||
S("reddit_mlops", "community", "rss", "https://old.reddit.com/r/mlops/.rss", "reddit_listing", ('reddit', 'mlops'), limit=20, detail=False),
|
||||
S("reddit_devops2", "community", "rss", "https://old.reddit.com/r/devops/.rss", "reddit_listing", ('reddit', 'devops'), limit=20, detail=False),
|
||||
S("reddit_homelab2", "community", "rss", "https://old.reddit.com/r/homelab/.rss", "reddit_listing", ('reddit', 'homelab'), limit=20, detail=False),
|
||||
S("reddit_grok2", "community", "rss", "https://old.reddit.com/r/grok/.rss", "reddit_listing", ('reddit', 'grok'), limit=20, detail=False),
|
||||
# ---- security ----
|
||||
S("sysdig", "security", "rss", "https://sysdig.com/feed/", "rss", ('security', 'cloud'), limit=20, detail=False),
|
||||
S("permiso", "security", "rss", "https://permiso.io/blog/rss.xml", "rss", ('security', 'cloud'), limit=20, detail=False),
|
||||
S("aquasec", "security", "rss", "https://www.aquasec.com/feed/", "rss", ('security', 'cloud'), limit=20, detail=False),
|
||||
S("infosecmatter", "security", "rss", "https://www.infosecmatter.com/feed/", "rss", ('security', 'tools'), limit=20, detail=False),
|
||||
S("cvefeed", "security", "rss", "https://cvefeed.io/rssfeed/latest.xml", "rss", ('security', 'cve'), limit=20, detail=False),
|
||||
S("graynoise", "security", "rss", "https://www.greynoise.io/blog/rss.xml", "rss", ('security', 'intel'), limit=20, detail=False),
|
||||
S("recordedfuture", "security", "rss", "https://www.recordedfuture.com/feed", "rss", ('security', 'intel'), limit=20, detail=False),
|
||||
S("cisa_news", "security", "rss", "https://www.cisa.gov/news.xml", "rss", ('security', 'gov'), limit=20, detail=False),
|
||||
S("msrc", "security", "rss", "https://api.msrc.microsoft.com/update-guide/rss", "rss", ('security', 'microsoft', 'cve'), limit=20, detail=False),
|
||||
# ---- forum ----
|
||||
S("forum_openai", "forum", "json", "https://community.openai.com/latest.json", "discourse", ('forum', 'openai'), limit=20, detail=False),
|
||||
S("forum_fly", "forum", "json", "https://community.fly.io/latest.json", "discourse", ('forum', 'infra'), limit=20, detail=False),
|
||||
S("forum_bitwarden", "forum", "json", "https://community.bitwarden.com/latest.json", "discourse", ('forum', 'security'), limit=20, detail=False),
|
||||
S("forum_cursor", "forum", "json", "https://forum.cursor.com/latest.json", "discourse", ('forum', 'ai', 'ide'), limit=20, detail=False),
|
||||
S("forum_manjaro", "forum", "json", "https://forum.manjaro.org/latest.json", "discourse", ('forum', 'linux'), limit=20, detail=False),
|
||||
S("forum_arduino", "forum", "json", "https://forum.arduino.cc/latest.json", "discourse", ('forum', 'hardware'), limit=20, detail=False),
|
||||
S("forum_kde", "forum", "json", "https://discuss.kde.org/latest.json", "discourse", ('forum', 'linux'), limit=20, detail=False),
|
||||
S("forum_ray", "forum", "json", "https://discuss.ray.io/latest.json", "discourse", ('forum', 'ml'), limit=20, detail=False),
|
||||
S("forum_wandb", "forum", "json", "https://community.wandb.ai/latest.json", "discourse", ('forum', 'ml'), limit=20, detail=False),
|
||||
S("forum_grafana2", "forum", "json", "https://community.grafana.com/latest.json", "discourse", ('forum', 'observability'), limit=20, detail=False),
|
||||
S("forum_traefik2", "forum", "json", "https://community.traefik.io/latest.json", "discourse", ('forum', 'infra'), limit=20, detail=False),
|
||||
S("forum_nvidia", "forum", "json", "https://forums.developer.nvidia.com/latest.json", "discourse", ('forum', 'nvidia'), limit=20, detail=False),
|
||||
# ---- company_cn ----
|
||||
S("qbitai", "company_cn", "rss", "https://www.qbitai.com/feed", "rss", ('cn', 'ai', 'news'), limit=20, detail=False),
|
||||
S("infoq_cn", "company_cn", "rss", "https://www.infoq.cn/feed", "rss", ('cn', 'dev'), limit=20, detail=False),
|
||||
S("leiphone", "company_cn", "rss", "https://www.leiphone.com/feed", "rss", ('cn', 'tech'), limit=20, detail=False),
|
||||
# ---- models ----
|
||||
S("hf_models_summary", "models", "json", "https://huggingface.co/api/models?pipeline_tag=summarization&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_translate", "models", "json", "https://huggingface.co/api/models?pipeline_tag=translation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_similarity", "models", "json", "https://huggingface.co/api/models?pipeline_tag=sentence-similarity&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'embed'), limit=20, detail=False),
|
||||
S("hf_models_tokencls", "models", "json", "https://huggingface.co/api/models?pipeline_tag=token-classification&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_zeroshot", "models", "json", "https://huggingface.co/api/models?pipeline_tag=zero-shot-classification&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_segment", "models", "json", "https://huggingface.co/api/models?pipeline_tag=image-segmentation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False),
|
||||
S("hf_models_objectdet", "models", "json", "https://huggingface.co/api/models?pipeline_tag=object-detection&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False),
|
||||
S("hf_models_depth", "models", "json", "https://huggingface.co/api/models?pipeline_tag=depth-estimation&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vision'), limit=20, detail=False),
|
||||
S("hf_models_timeseries", "models", "json", "https://huggingface.co/api/models?pipeline_tag=time-series-forecasting&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_rl", "models", "json", "https://huggingface.co/api/models?pipeline_tag=reinforcement-learning&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'rl'), limit=20, detail=False),
|
||||
S("hf_models_qa", "models", "json", "https://huggingface.co/api/models?pipeline_tag=question-answering&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf'), limit=20, detail=False),
|
||||
S("hf_models_img2text", "models", "json", "https://huggingface.co/api/models?pipeline_tag=image-to-text&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'vlm'), limit=20, detail=False),
|
||||
S("hf_models_rerank2", "models", "json", "https://huggingface.co/api/models?pipeline_tag=text-ranking&sort=trendingScore&direction=-1&limit=20", "hf_models", ('model', 'hf', 'rag'), limit=20, detail=False),
|
||||
# ---- benchmark ----
|
||||
S("aider_board", "benchmark", "rss", "https://aider.chat/feed.xml", "rss", ('benchmark', 'code'), limit=20, detail=False),
|
||||
S("arcprize", "benchmark", "rss", "https://arcprize.org/feed.xml", "rss", ('benchmark', 'reasoning'), limit=20, detail=False),
|
||||
]
|
||||
Reference in New Issue
Block a user