diff --git a/omni/abuse.py b/omni/abuse.py index 8cf0e89..1965338 100644 --- a/omni/abuse.py +++ b/omni/abuse.py @@ -39,6 +39,46 @@ RULES = [ r"продаю", r"продам", r"куплю\s+аккаунт", r"for\s+sale", r"цена\s*[:$]", r"\$\d+\s?/\s?(мес|mo|1k|1000)", r"реселлер", r"resell", ]), + ("credits", "кредиты/квоты", 22, [ + r"free\s+(?:api\s+)?credits?", r"\b\d+\s?(?:k|m)?\s?tokens?\s+(?:free|бесплатн)", r"free\s+quota", + r"бесплатн\w*\s+(?:лимит|квот|токен)", r"кредит\w*\s+(?:бесплатн|в\s+подарок)", r"赠金", r"额度", + r"unlimited\s+(?:api|access|tokens|usage)", r"безлимит\w*\s+(?:api|ключ|доступ|токен)", + ]), + ("invite", "инвайты/беты", 16, [ + r"\binvite\s?(?:code|link|only)\b", r"инвайт", r"\bwaitlist\b", r"лист\s+ожидан", r"early\s+access", + r"\bbeta\s+(?:access|invite|key)", r"закрыт\w*\s+бет", r"приглашени", + ]), + ("ltd", "lifetime deal", 18, [ + r"lifetime\s+deal", r"\bltd\b", r"пожизненн\w*\s+(?:доступ|лицензи)", r"пакет\s+навсегда", + ]), + ("coupon", "купоны/скидки", 12, [ + r"\bcoupon\b", r"\bdiscount\s?(?:code)?\b", r"\d{1,2}%\s?off", r"дешёв\w*\s+чем", r"\bcheaper\b", + ]), + ("combo", "комбо/базы", 20, [ + r"[\w.\-]+@[\w.\-]+\.[a-z]{2,}\s?:\s?[^\s]{4,}", r"\bcombolist\b", r"\bcombos?\b\s+(?:list|base)", + r"\b\d{3,}\s*(?:k|к|тыс)?\s*(?:строк|lines|аккаунтов|accounts)\b", + ]), + ("crack", "кряки/патчи", 16, [ + r"\bkeygen\b", r"\bpatcher\b", r"\bкряк\b", r"\bcrack(?:ed)?\s+(?:version|by)", r"\bcracked\b", + r"\bnulled\b", r"\bactivated\s+version\b", + ]), + ("selfhost", "селфхост/опенсорс", 8, [ + r"self[\s-]?host(?:ed|ing)?", r"open[\s-]?source\s+alternative", r"альтернатив\w*\s+(?:для|к)\s+\w+", + r"docker\s+(?:run|compose)", r"\bна\s+своём\s+сервер", + ]), + ("freeapi", "бесплатные API", 24, [ + r"free\s+(?:llm\s+)?api", r"бесплатн\w*\s+api", r"api\s+(?:без|without)\s+(?:ключа|ключа\s+не\s+нужно|key)", + r"no\s+(?:api\s+)?key\s+(?:required|needed)", r"keyless", r"openai[\s-]?compatible\s+(?:endpoint|api)", + r"/v1/chat/completions", r"new[\s-]?api\s+(?:panel|панел)", r"openai\s+proxy", + ]), + ("nokyc", "без KYC/карты", 14, [ + r"no\s+kyc", r"без\s+kyc", r"no\s+card\s+(?:required|needed)", r"без\s+карт\w*", r"без\s+депозит", + r"no\s+sign[\s-]?up", + ]), + ("infra", "прокси/номера/карты", 10, [ + r"\bпрокси\b|\bproxies\b", r"\bsms\s?(?:activ|receive)", r"\bвиртуальн\w*\s+карт", r"\bvcc\b", + r"\bноутбук\s+номер", r"\bномер\w*\s+для\s+смс", + ]), ("news", "новости", 6, [ r"анонс", r"выпустил", r"released?", r"launch", r"обновлени", r"release notes", r"новая модель", r"new model", @@ -47,7 +87,7 @@ RULES = [ COMPILED = [(tag, label, w, [re.compile(p, re.I) for p in pats]) for tag, label, w, pats in RULES] TAG_LABEL = {t: l for t, l, _, _ in RULES} -STRONG = {"keys", "promo", "trial", "resale"} +STRONG = {"keys", "promo", "trial", "resale", "credits", "freeapi", "ltd", "combo"} # на специализированных абуз-площадках хватает одного признака: сам источник тематический ABUSE_SRC_SC = 14 # категории-источники, которые вообще имеют смысл для абузов @@ -106,7 +146,7 @@ def _qualifies(source, title, text, tags): return 0, [] if cat is None: # старые записи хранят категорию вместо имени источника - if not raw_key and not str(source or "").startswith(("ab_", "tg_", "reddit_", "discourse_", "xenforo")): + if not raw_key and not str(source or "").startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo")): return 0, [] elif cat not in ABUSE_CATS and not raw_key: return 0, [] @@ -119,42 +159,89 @@ def _qualifies(source, title, text, tags): return 0, [] +def _connect(): + con = sqlite3.connect(str(DB), timeout=60) + con.execute("PRAGMA busy_timeout=60000") + con.execute("CREATE TABLE IF NOT EXISTS abuse_idx " + "(id INTEGER PRIMARY KEY, score INTEGER, tags TEXT)") + return con + + +def rebuild(full=False, chunk=4000): + """Инкрементальный индекс абузов: classify по новым items, без полного перещёлкивания. + + full=True (или OMNI_ABUSE_REBUILD=1) — пересчитать всю базу после правки правил. + Возвращает число записей, попавших в индекс на этом проходе. + """ + import os + if os.environ.get("OMNI_ABUSE_REBUILD") == "1": + full = True + con = _connect() + try: + if full: + con.execute("DELETE FROM abuse_idx") + con.commit() + last = con.execute("SELECT COALESCE(MAX(id), 0) FROM abuse_idx").fetchone()[0] + added = 0 + while True: + rows = con.execute( + "SELECT id, source, url, title, text, tags FROM items WHERE id > ? ORDER BY id LIMIT ?", + (last, chunk)).fetchall() + if not rows: + break + batch = [] + for r in rows: + sc, hits = _qualifies(r[1], r[3], r[4], r[5] or "") + if sc: + batch.append((r[0], sc, ",".join(hits))) + last = r[0] + if batch: + con.executemany("INSERT OR REPLACE INTO abuse_idx(id, score, tags) VALUES (?,?,?)", batch) + added += len(batch) + con.commit() + return added + finally: + con.close() + + def _rows(limit=400, tag=None, q=None, since_days=None): - sql = "SELECT id, source, url, title, text, published, tags, score FROM items" + """Читает из индекса abuse_idx; индекс добивается свежими записями перед чтением.""" + try: + rebuild() + except Exception: + pass + sql = ("SELECT i.id, i.source, i.url, i.title, i.text, i.published, a.tags, i.score, a.score " + "FROM abuse_idx a JOIN items i ON i.id = a.id") where, args = [], [] + if tag: + where.append("(',' || a.tags || ',') LIKE ?") + args.append("%%," + tag + ",%%") if q: - where.append("(title LIKE ? OR text LIKE ? OR source LIKE ?)") + where.append("(i.title LIKE ? OR i.text LIKE ? OR i.source LIKE ?)") args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q] if since_days: - where.append("published >= date('now', ?)") + where.append("i.published >= date('now', ?)") args.append("-%d day" % int(since_days)) if where: sql += " WHERE " + " AND ".join(where) - sql += " ORDER BY id DESC LIMIT ?" - args.append(int(limit) * 6) - con = sqlite3.connect(str(DB), timeout=30) + sql += " ORDER BY a.score DESC, i.id DESC LIMIT ?" + args.append(int(limit)) + con = _connect() try: - cur = con.execute(sql, args) out = [] - for r in cur: - sc, hits = _qualifies(r[1], r[3], r[4], r[6] or "") - if not sc: - continue - if tag and tag not in hits: - continue + for r in con.execute(sql, args): + hits = [t for t in (r[6] or "").split(",") if t] out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3], "text": (r[4] or "")[:600], "published": r[5], - "tags": hits, "abuse_score": sc, "rank": r[7]}) - if len(out) >= limit: - break + "tags": hits, "abuse_score": r[8], "rank": r[7]}) return out finally: con.close() def scan(limit=120, tag=None, q=None, since_days=None, window=6000): - """limit — сколько отдать, window — сколько последних записей просканировать.""" - items = _rows(limit=window, tag=tag, q=q, since_days=since_days) + """limit — сколько отдать; индекс уже отсортирован по весy, window не режет.""" + items = _rows(limit=max(int(limit), 400), tag=tag, q=q, since_days=since_days) items.sort(key=lambda x: (-x["abuse_score"], -(x["id"] or 0))) counts = {} for it in items: diff --git a/omni/cli.py b/omni/cli.py index f583c20..f4615dc 100644 --- a/omni/cli.py +++ b/omni/cli.py @@ -542,6 +542,13 @@ def main(argv=None): sp.add_argument("--apply", action="store_true") sp.set_defaults(fn=cmd_pump) + sp = sub.add_parser("tgsearch", help="найти Telegram-каналы по ключевым словам через свою сессию") + sp.add_argument("queries", help="запросы через запятую") + sp.add_argument("--limit", type=int, default=40, help="каналов на запрос") + sp.add_argument("--min-members", type=int, default=0, help="отсеять мелкие каналы") + sp.add_argument("--apply", action="store_true", help="дописать найденные каналы в data/radar_sources.json") + sp.set_defaults(fn=cmd_tgsearch) + sp = sub.add_parser("keys", help="ключи парсеров из vault: сервисы, счётчики, остывание") sp.add_argument("--service", default="", help="только один сервис") sp.add_argument("-v", "--verbose", action="store_true", help="по ключам, масками") @@ -561,4 +568,58 @@ def main(argv=None): if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) + + +def cmd_tgsearch(a): + """Поиск каналов по ключевым словам -> пополнение пула источников.""" + from . import tgsess + if not tgsess.enabled(): + print("сессия не настроена: data/tg.json или OMNI_TG_* в env") + return 1 + queries = [q.strip() for q in str(a.queries).split(",") if q.strip()] + found = {} + for q in queries: + try: + rows = tgsess.search(q, limit=a.limit) + except Exception as e: + print(" %-28s ошибка %s" % (q, type(e).__name__)) + continue + for r in rows: + if a.min_members and (r.get("participants") or 0) < a.min_members: + continue + prev = found.get(r["username"]) + if prev: + prev["hits"] = prev.get("hits", 1) + 1 + continue + r["hits"] = 1 + found[r["username"]] = r + print(" %-28s каналов %d (всего уникальных %d)" % (q, len(rows), len(found))) + rows = sorted(found.values(), key=lambda x: (-(x.get("hits") or 0), -(x.get("participants") or 0))) + for r in rows[:20]: + print(" @%-28s %7s участн. hits=%d %s" % (r["username"], r.get("participants") or 0, + r["hits"], (r.get("title") or "")[:40])) + if not a.apply: + print("найдено %d каналов; добавь --apply чтобы дописать в пул" % len(rows)) + return 0 + import json + from pathlib import Path + p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json" + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + d = {"domains": [], "channels": []} + chans = list(d.get("channels") or []) + have = {str(c).lstrip("@").lower() for c in chans} + added = 0 + for r in rows: + u = str(r["username"]).lstrip("@") + if u.lower() in have: + continue + chans.append(u) + have.add(u.lower()) + added += 1 + d["channels"] = chans + p.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8") + print("дописано каналов: %d | всего каналов в пуле: %d" % (added, len(chans))) + return 0 diff --git a/omni/feeds.py b/omni/feeds.py index df125a6..a9b52dd 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -709,6 +709,169 @@ def _x_user(src, limit=12): return xp.profile(handle, limit=max(4, min(limit, 25))) +# ---------------- sitemap_recent: сайты без ленты, но со свежими url ---------------- +def _sitemap_recent(src, limit): + """sitemap.xml/sitemap_index.xml -> свежие url по lastmod. + + extra: {"days": 14, "sitemaps": ["https://x/sitemap.xml"], "must": ["/blog/","/news/"]} + Даёт доступ к сайтам, где нет ни rss, ни списка статей в html. + """ + import datetime as _d + days = int((src.extra or {}).get("days") or 21) + cut = (_d.datetime.utcnow() - _d.timedelta(days=days)).strftime("%Y-%m-%d") + must = tuple((src.extra or {}).get("must") or ()) + starts = list((src.extra or {}).get("sitemaps") or []) + if not starts: + base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url)) + starts = [base + "/sitemap.xml", base + "/sitemap_index.xml", + base + "/sitemap-index.xml", base + "/wp-sitemap.xml"] + urls, seen_maps = [], set() + + def load(u, depth=0): + if u in seen_maps or depth > 2 or len(seen_maps) > 12: + return + seen_maps.add(u) + r = net.get(u, headers=UA) + if r.status != 200 or "<" not in (r.text or ""): + return + sx = BeautifulSoup(r.text, "xml") + for sm in sx.find_all("sitemap"): + loc = (sm.find("loc").get_text(strip=True) if sm.find("loc") else "") + lm = sm.find("lastmod").get_text(strip=True) if sm.find("lastmod") else "" + if loc and (not lm or lm[:10] >= cut): + load(loc, depth + 1) + for ue in sx.find_all("url"): + loc = ue.find("loc").get_text(strip=True) if ue.find("loc") else "" + lm = ue.find("lastmod").get_text(strip=True) if ue.find("lastmod") else "" + if not loc: + continue + if lm and lm[:10] < cut: + continue + if must and not any(m in loc for m in must): + continue + urls.append((loc, lm)) + + for u in starts: + try: + load(u) + except Exception: + continue + # свежие сверху, без подстраниц-дублей + urls.sort(key=lambda x: x[1] or "", reverse=True) + out, seen = [], set() + for loc, lm in urls: + k = loc.rstrip("/") + if k in seen: + continue + seen.add(k) + tail = urllib.parse.urlsplit(loc).path.strip("/").rsplit("/", 1)[-1] + title = urllib.parse.unquote(tail).replace("-", " ").replace("_", " ")[:160] or loc + out.append(_item(src, "page", loc, title, "", "", lm)) + if len(out) >= limit: + break + return out + + +# ---------------- newapi_pricing: панели new-api -> бесплатные модели ---------------- +def _newapi_pricing(src, limit): + """Панель new-api: /api/pricing -> модели с нулевой ценой = бесплатный доступ. + + Ищет не статьи про абузы, а сами раздачи: free-модели на живых панелях. + """ + base = src.url.rstrip("/") + js = None + # часть панелей отдаёт каталог только простому UA (полный Chrome-UA -> пустое тело) + base_hdr = dict(UA) if isinstance(UA, dict) else {"User-Agent": str(UA)} + for ua in (base_hdr.get("User-Agent", ""), "Mozilla/5.0", "curl/8.5.0"): + r = net.get(base + "/api/pricing", headers={"User-Agent": ua, "Accept": "application/json"}) + if r.status != 200 or not (r.text or "").strip().startswith(("{", "[")): + continue + try: + js = json.loads(r.text) + break + except Exception: + continue + if js is None: + return [] + data = js.get("data") if isinstance(js, dict) else None + if not isinstance(data, list): + return [] + free, paid = [], 0 + for m in data: + if not isinstance(m, dict): + continue + name = m.get("model_name") or m.get("model") or "" + ratio = m.get("model_ratio") + cratio = m.get("completion_ratio") + try: + zero = (float(ratio or 0) == 0) and (float(cratio or 0) == 0) + except Exception: + zero = False + if name and zero: + free.append(name) + elif name: + paid += 1 + if not free: + return [] + host = urllib.parse.urlsplit(base).netloc or base + title = "панель %s: %d бесплатных моделей" % (host, len(free)) + body = ("Бесплатные модели (цена 0): " + ", ".join(free[:40]) + + ("\nвсего моделей на панели: %d" % (len(free) + paid))) + return [_item(src, "panel", base + "/api/pricing", title, body, "", "", + {"free_models": free[:60], "free_count": len(free), "paid_count": paid})] + + +def _github_search(src, limit): + """GitHub search API: свежие репозитории по запросу (free api, jailbreak, autoreg).""" + q = (src.extra or {}).get("q") or "free api" + sort = (src.extra or {}).get("sort") or "updated" + url = ("https://api.github.com/search/repositories?q=" + urllib.parse.quote(str(q)) + + "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50)))) + hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"} + tok = ((config.CONFIG.get("keys") or {}).get("github") or []) + if isinstance(tok, list) and tok: + hdr["Authorization"] = "Bearer " + str(tok[0]) + r = net.get(url, headers=hdr) + if r.status != 200: + return [] + try: + js = json.loads(r.text) + except Exception: + return [] + out = [] + for it in (js.get("items") or [])[:limit]: + title = it.get("full_name") or it.get("name") or "" + desc = it.get("description") or "" + url_ = it.get("html_url") or "" + upd = (it.get("updated_at") or "")[:10] + out.append(_item(src, "github", url_, title, desc, upd, "", {})) + return out + + +def _hn_search(src, limit): + """Hacker News (Algolia): поиск по свежим историям — бесплатные API, сливы, LLM-абузы.""" + q = (src.extra or {}).get("q") or "free api" + url = ("https://hn.algolia.com/api/v1/search_by_date?query=" + urllib.parse.quote(str(q)) + + "&tags=story&hitsPerPage=" + str(max(1, min(int(limit), 50)))) + r = net.get(url, headers={"User-Agent": "omni-parse", "Accept": "application/json"}) + if r.status != 200: + return [] + try: + js = json.loads(r.text) + except Exception: + return [] + out = [] + for h in (js.get("hits") or [])[:limit]: + title = h.get("title") or h.get("story_title") or "" + if not title: + continue + link = h.get("url") or ("https://news.ycombinator.com/item?id=%s" % h.get("objectID")) + ts = (h.get("created_at") or "")[:10] + body = (h.get("story_text") or "")[:400] + out.append(_item(src, "hn", link, title, body, ts, "", {})) + return out + + ADAPTERS = { "x_user": _x_user, "rss": _rss, @@ -739,6 +902,10 @@ ADAPTERS = { "cve_circl": _cve_circl, "ghsa": _ghsa, "models_dev": _models_dev, + "sitemap_recent": _sitemap_recent, + "newapi_pricing": _newapi_pricing, + "github_search": _github_search, + "hn_search": _hn_search, } diff --git a/omni/sources.py b/omni/sources.py index edaa0cf..0843efc 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -245,6 +245,42 @@ def radar_sources(): return rows +_PANELS = {"rows": [], "at": 0.0} + + +def panel_sources(): + """Панели new-api из data/panels.json (tools/panel_hunt.py --apply). + + Адаптер newapi_pricing тянет /api/pricing и отдаёт находку: сколько моделей + на панели бесплатны. Это не статьи про абузы, а сами раздачи API. + """ + import json + from pathlib import Path + p = Path(__file__).resolve().parents[1] / "data" / "panels.json" + try: + mtime = p.stat().st_mtime + except OSError: + return [] + if _PANELS["rows"] and _PANELS["at"] == mtime: + return _PANELS["rows"] + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + return [] + rows = [] + for r in d.get("panels") or []: + host = str(r.get("host") or "").strip() + if not host: + continue + name = "pn_" + host.replace(".", "_").replace("-", "_") + rows.append(Source(name=name, category="abuse", kind="site", url="https://" + host, + adapter="newapi_pricing", tags=("panel", "abuse", "freeapi"), + limit=1, detail=False, + extra={"free_count": r.get("free_count") or 0, "models": r.get("models") or 0})) + _PANELS.update(at=mtime, rows=rows) + return rows + + def _gen_sources(): """Сгенерированные наборы: китайские (sources_cn) + дарк-форумы (sources_dark).""" out = [] @@ -272,11 +308,15 @@ def all_sources(): ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")} extra.append(Source(**d)) seen, out = set(), [] - for s in SOURCES + _gen_sources() + extra + radar_sources(): + for s in SOURCES + _gen_sources() + extra + radar_sources() + panel_sources(): + # дедуп строго по url: одна площадка = один источник, даже если пришла из + # реестра, radar_sources и panels.json одновременно (иначе двойной обход и + # двойные записи в базе). Побеждает первое вхождение — порядок списка выше. key = (s.url or "").rstrip("/").lower() if key and key in seen: continue - seen.add(key) + if key: + seen.add(key) out.append(s) return out diff --git a/omni/sources_abuse.py b/omni/sources_abuse.py index e22ccc4..959b5ff 100644 --- a/omni/sources_abuse.py +++ b/omni/sources_abuse.py @@ -44,4 +44,31 @@ ABUSE_LIVE = [ A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20), A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20), A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20), + # --- новые способы: discourse-площадки, reddit-json, rss-агрегаторы, sitemap --- + A("ab_reddit_freebies", "abuse", "json", "https://www.reddit.com/r/freebies/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20), + A("ab_reddit_openaiapi", "abuse", "json", "https://www.reddit.com/r/OpenAIAPI/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=12), + A("ab_reddit_freeai", "abuse", "json", "https://www.reddit.com/r/FreeAI/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=12), + A("ab_rss_slickdeals_free", "abuse", "rss", "https://slickdeals.net/newsearch.php?rss=1&searcharea=deals&searchin=first&q=free", "rss", ('abuse', 'abuse', 'promo'), limit=20), + A("ab_sitemap_cpamafia", "abuse", "site", "https://cpamafia.pro/sitemap.xml", "sitemap_recent", ('abuse', 'abuse', 'sitemap'), limit=20, extra={"days": 30}), + A("ab_sitemap_carders", "abuse", "site", "https://carders.biz/sitemap.xml", "sitemap_recent", ('abuse', 'abuse', 'sitemap'), limit=20, extra={"days": 30}), + + # ---- GitHub search: свежие репозитории-раздачи (свой url на запрос = без дедупа) ---- + A("ab_gh_free_api_keys", "abuse", "site", "https://github.com/search?q=free+api+keys&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free api keys", "sort": "updated"}), + A("ab_gh_api_key_pool", "abuse", "site", "https://github.com/search?q=api+key+pool&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "api key pool", "sort": "updated"}), + A("ab_gh_free_gpt_api", "abuse", "site", "https://github.com/search?q=free+gpt+api&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free gpt api", "sort": "updated"}), + A("ab_gh_claude_api_free", "abuse", "site", "https://github.com/search?q=claude+api+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "claude api free", "sort": "updated"}), + A("ab_gh_one_api_new_api", "abuse", "site", "https://github.com/search?q=one-api+new-api&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "one-api new-api", "sort": "updated"}), + A("ab_gh_jailbreak_prompts", "abuse", "site", "https://github.com/search?q=jailbreak+prompts&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "jailbreak prompts", "sort": "updated"}), + A("ab_gh_account_generator", "abuse", "site", "https://github.com/search?q=account+generator&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "account generator", "sort": "updated"}), + A("ab_gh_proxy_pool_free", "abuse", "site", "https://github.com/search?q=proxy+pool+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "proxy pool free", "sort": "updated"}), + A("ab_gh_openai_reverse_proxy", "abuse", "site", "https://github.com/search?q=openai+reverse+proxy&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "openai reverse proxy", "sort": "updated"}), + A("ab_gh_ai_gateway_free", "abuse", "site", "https://github.com/search?q=ai+gateway+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "ai gateway free", "sort": "updated"}), + + # ---- Hacker News (Algolia): свежие истории про бесплатные API и сливы ---- + A("ab_hn_free_api_keys", "abuse", "site", "https://hn.algolia.com/?query=free+api+keys", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "free api keys"}), + A("ab_hn_openai_api_free", "abuse", "site", "https://hn.algolia.com/?query=openai+api+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "openai api free"}), + A("ab_hn_llm_jailbreak", "abuse", "site", "https://hn.algolia.com/?query=llm+jailbreak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "llm jailbreak"}), + A("ab_hn_account_generator", "abuse", "site", "https://hn.algolia.com/?query=account+generator", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "account generator"}), + A("ab_hn_api_key_leak", "abuse", "site", "https://hn.algolia.com/?query=api+key+leak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "api key leak"}), + A("ab_hn_chatgpt_free", "abuse", "site", "https://hn.algolia.com/?query=chatgpt+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "chatgpt free"}), ] diff --git a/omni/tgsess.py b/omni/tgsess.py index 1a210ad..dd3a9e2 100644 --- a/omni/tgsess.py +++ b/omni/tgsess.py @@ -89,4 +89,46 @@ def fetch(handles, limit=None): api_id = int(os.environ["OMNI_TG_API_ID"]) api_hash = os.environ["OMNI_TG_API_HASH"] lim = int(limit or os.environ.get("OMNI_TG_LIMIT") or 100) - return asyncio.run(_pull(list(handles), lim, session, api_id, api_hash)) \ No newline at end of file + return asyncio.run(_pull(list(handles), lim, session, api_id, api_hash)) + +async def _search(query, limit, session, api_id, api_hash, timeout=60): + """Глобальный поиск каналов/групп по запросу (contacts.SearchRequest).""" + from telethon import TelegramClient + from telethon.tl.functions.contacts import SearchRequest + + client = TelegramClient(session, api_id, api_hash, connection_retries=1, timeout=timeout / 2) + await asyncio.wait_for(client.connect(), timeout) + try: + res = await asyncio.wait_for(client(SearchRequest(q=str(query), limit=int(limit))), timeout) + out = [] + for ch in getattr(res, "chats", []) or []: + uname = getattr(ch, "username", None) + if not uname: + continue + out.append({"username": uname, "title": getattr(ch, "title", "") or "", + "id": getattr(ch, "id", None), + "broadcast": bool(getattr(ch, "broadcast", False)), + "megagroup": bool(getattr(ch, "megagroup", False)), + "participants": getattr(ch, "participants_count", None) or 0, + "query": str(query)}) + return out + finally: + try: + await asyncio.wait_for(client.disconnect(), 10) + except Exception: + pass + + +def search(query, limit=40): + """Каналы по ключевому слову. Пусто, если сессия не настроена.""" + if not enabled(): + return [] + try: + from . import config + config.tg_cfg() + except Exception: + pass + session = os.environ["OMNI_TG_SESSION"] + api_id = int(os.environ["OMNI_TG_API_ID"]) + api_hash = os.environ["OMNI_TG_API_HASH"] + return asyncio.run(_search(query, limit, session, api_id, api_hash)) diff --git a/tools/panel_hunt.py b/tools/panel_hunt.py new file mode 100644 index 0000000..c6f9fc6 --- /dev/null +++ b/tools/panel_hunt.py @@ -0,0 +1,153 @@ +"""Охота за панелями new-api: /api/status -> /api/pricing -> бесплатные модели. + +Панель new-api (одна из самых частых раздач в абуз-каналах) отдаёт каталог +моделей и цены публично. Нулевая цена = бесплатный доступ = находка для абузов. + +Запуск: + python tools/panel_hunt.py # домены из data/radar_sources.json + python tools/panel_hunt.py --hosts a.com b.com + python tools/panel_hunt.py --limit 400 --workers 24 + python tools/panel_hunt.py --apply # записать data/panels.json (источники omni) + +Пишет только пути/счётчики. Ключи и токены не трогает. +""" +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import ssl +import sys +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" +UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0 Safari/537.36" +CTX = ssl.create_default_context() +CTX.check_hostname = False +CTX.verify_mode = ssl.CERT_NONE +MARKERS = ("HeaderNavModules", "system_name", "chats", "quota_per_unit", "display_in_currency", "announcements_enabled") + + +def _get(url, timeout=8, cap=400000): + req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout, context=CTX) as r: + return r.status, r.read(cap).decode("utf-8", "ignore") + + +def _is_newapi(body: str) -> bool: + if '"data"' not in body: + return False + return any(m in body for m in MARKERS) + + +def probe(host: str): + """-> dict | None. Проверяет сам хост и api..""" + for h in (host, "api." + host if not host.startswith("api.") else host): + try: + st, body = _get("https://" + h + "/api/status", timeout=7) + if st != 200 or not _is_newapi(body): + continue + except Exception: + continue + rec = {"host": h, "status": "newapi", "models": 0, "free": [], "free_count": 0} + try: + st, body = _get("https://" + h + "/api/pricing", timeout=12) + if st == 200: + js = json.loads(body) + data = js.get("data") if isinstance(js, dict) else None + if isinstance(data, list): + rec["models"] = len(data) + free = [] + for m in data: + if not isinstance(m, dict): + continue + try: + zero = float(m.get("model_ratio") or 0) == 0 and float(m.get("completion_ratio") or 0) == 0 + except Exception: + zero = False + if zero and m.get("model_name"): + free.append(m["model_name"]) + rec["free"] = free[:80] + rec["free_count"] = len(free) + except Exception: + rec["pricing"] = "err" + return rec + return None + + +def load_hosts(args) -> list: + if args.hosts: + return list(dict.fromkeys(args.hosts)) + p = DATA / "radar_sources.json" + if not p.exists(): + return [] + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + return [] + out = [] + for item in d.get("domains") or []: + dom = item[0] if isinstance(item, list) else (item.get("domain") or "") + if dom: + out.append(str(dom)) + return list(dict.fromkeys(out)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--hosts", nargs="*", default=None) + ap.add_argument("--limit", type=int, default=0, help="0 = все домены") + ap.add_argument("--workers", type=int, default=24) + ap.add_argument("--apply", action="store_true", help="записать data/panels.json") + ap.add_argument("--min-free", type=int, default=0, help="только панели с >= N бесплатных моделей") + a = ap.parse_args() + + hosts = load_hosts(a) + if a.limit: + hosts = hosts[:a.limit] + if not hosts: + print("нет доменов: сначала omni radar --apply") + return 1 + + res = [] + with cf.ThreadPoolExecutor(max(4, a.workers)) as ex: + futs = {ex.submit(probe, h): h for h in hosts} + for i, f in enumerate(cf.as_completed(futs), 1): + try: + r = f.result() + except Exception: + r = None + if r: + res.append(r) + print(" %s | new-api | моделей %d | free %d" % (r["host"], r["models"], r["free_count"])) + if i % 200 == 0: + print("... проверено %d/%d, найдено %d" % (i, len(hosts), len(res)), flush=True) + + res.sort(key=lambda x: x["free_count"], reverse=True) + if a.min_free: + res = [r for r in res if r["free_count"] >= a.min_free] + print("итог: проверено %d | панелей new-api %d | с бесплатными моделями %d | free-моделей всего %d" + % (len(hosts), len(res), sum(1 for r in res if r["free_count"]), sum(r["free_count"] for r in res))) + + if a.apply: + out = DATA / "panels.json" + prev = {} + if out.exists(): + try: + for r in json.loads(out.read_text(encoding="utf-8")).get("panels") or []: + prev[r.get("host")] = r + except Exception: + prev = {} + for r in res: + prev[r["host"]] = r # свежие данные перекрывают старые, мёртвые не затираются + payload = {"panels": sorted(prev.values(), key=lambda x: x.get("free_count") or 0, reverse=True)} + out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") + print("записано: %s (панелей %d)" % (out, len(payload["panels"]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file