реестр: дедуп источников строго по url (2 падавших теста зелёные); инкрементальный abuse-индекс + tgsearch из предыдущей сессии

omni/abuse.py

@@ -39,6 +39,46 @@ RULES = [
         r"продаю", r"продам", r"куплю\s+аккаунт", r"for\s+sale", r"цена\s*[:$]", r"\$\d+\s?/\s?(мес|mo|1k|1000)",
         r"реселлер", r"resell",
     ]),
+    ("credits", "кредиты/квоты", 22, [
+        r"free\s+(?:api\s+)?credits?", r"\b\d+\s?(?:k|m)?\s?tokens?\s+(?:free|бесплатн)", r"free\s+quota",
+        r"бесплатн\w*\s+(?:лимит|квот|токен)", r"кредит\w*\s+(?:бесплатн|в\s+подарок)", r"赠金", r"额度",
+        r"unlimited\s+(?:api|access|tokens|usage)", r"безлимит\w*\s+(?:api|ключ|доступ|токен)",
+    ]),
+    ("invite", "инвайты/беты", 16, [
+        r"\binvite\s?(?:code|link|only)\b", r"инвайт", r"\bwaitlist\b", r"лист\s+ожидан", r"early\s+access",
+        r"\bbeta\s+(?:access|invite|key)", r"закрыт\w*\s+бет", r"приглашени",
+    ]),
+    ("ltd", "lifetime deal", 18, [
+        r"lifetime\s+deal", r"\bltd\b", r"пожизненн\w*\s+(?:доступ|лицензи)", r"пакет\s+навсегда",
+    ]),
+    ("coupon", "купоны/скидки", 12, [
+        r"\bcoupon\b", r"\bdiscount\s?(?:code)?\b", r"\d{1,2}%\s?off", r"дешёв\w*\s+чем", r"\bcheaper\b",
+    ]),
+    ("combo", "комбо/базы", 20, [
+        r"[\w.\-]+@[\w.\-]+\.[a-z]{2,}\s?:\s?[^\s]{4,}", r"\bcombolist\b", r"\bcombos?\b\s+(?:list|base)",
+        r"\b\d{3,}\s*(?:k|к|тыс)?\s*(?:строк|lines|аккаунтов|accounts)\b",
+    ]),
+    ("crack", "кряки/патчи", 16, [
+        r"\bkeygen\b", r"\bpatcher\b", r"\bкряк\b", r"\bcrack(?:ed)?\s+(?:version|by)", r"\bcracked\b",
+        r"\bnulled\b", r"\bactivated\s+version\b",
+    ]),
+    ("selfhost", "селфхост/опенсорс", 8, [
+        r"self[\s-]?host(?:ed|ing)?", r"open[\s-]?source\s+alternative", r"альтернатив\w*\s+(?:для|к)\s+\w+",
+        r"docker\s+(?:run|compose)", r"\bна\s+своём\s+сервер",
+    ]),
+    ("freeapi", "бесплатные API", 24, [
+        r"free\s+(?:llm\s+)?api", r"бесплатн\w*\s+api", r"api\s+(?:без|without)\s+(?:ключа|ключа\s+не\s+нужно|key)",
+        r"no\s+(?:api\s+)?key\s+(?:required|needed)", r"keyless", r"openai[\s-]?compatible\s+(?:endpoint|api)",
+        r"/v1/chat/completions", r"new[\s-]?api\s+(?:panel|панел)", r"openai\s+proxy",
+    ]),
+    ("nokyc", "без KYC/карты", 14, [
+        r"no\s+kyc", r"без\s+kyc", r"no\s+card\s+(?:required|needed)", r"без\s+карт\w*", r"без\s+депозит",
+        r"no\s+sign[\s-]?up",
+    ]),
+    ("infra", "прокси/номера/карты", 10, [
+        r"\bпрокси\b|\bproxies\b", r"\bsms\s?(?:activ|receive)", r"\bвиртуальн\w*\s+карт", r"\bvcc\b",
+        r"\bноутбук\s+номер", r"\bномер\w*\s+для\s+смс",
+    ]),
     ("news", "новости", 6, [
         r"анонс", r"выпустил", r"released?", r"launch", r"обновлени", r"release notes",
         r"новая модель", r"new model",
@@ -47,7 +87,7 @@ RULES = [
 
 COMPILED = [(tag, label, w, [re.compile(p, re.I) for p in pats]) for tag, label, w, pats in RULES]
 TAG_LABEL = {t: l for t, l, _, _ in RULES}
-STRONG = {"keys", "promo", "trial", "resale"}
+STRONG = {"keys", "promo", "trial", "resale", "credits", "freeapi", "ltd", "combo"}
 # на специализированных абуз-площадках хватает одного признака: сам источник тематический
 ABUSE_SRC_SC = 14
 # категории-источники, которые вообще имеют смысл для абузов
@@ -106,7 +146,7 @@ def _qualifies(source, title, text, tags):
         return 0, []
     if cat is None:
         # старые записи хранят категорию вместо имени источника
-        if not raw_key and not str(source or "").startswith(("ab_", "tg_", "reddit_", "discourse_", "xenforo")):
+        if not raw_key and not str(source or "").startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo")):
             return 0, []
     elif cat not in ABUSE_CATS and not raw_key:
         return 0, []
@@ -119,42 +159,89 @@ def _qualifies(source, title, text, tags):
     return 0, []
 
 
+def _connect():
+    con = sqlite3.connect(str(DB), timeout=60)
+    con.execute("PRAGMA busy_timeout=60000")
+    con.execute("CREATE TABLE IF NOT EXISTS abuse_idx "
+                "(id INTEGER PRIMARY KEY, score INTEGER, tags TEXT)")
+    return con
+
+
+def rebuild(full=False, chunk=4000):
+    """Инкрементальный индекс абузов: classify по новым items, без полного перещёлкивания.
+
+    full=True (или OMNI_ABUSE_REBUILD=1) — пересчитать всю базу после правки правил.
+    Возвращает число записей, попавших в индекс на этом проходе.
+    """
+    import os
+    if os.environ.get("OMNI_ABUSE_REBUILD") == "1":
+        full = True
+    con = _connect()
+    try:
+        if full:
+            con.execute("DELETE FROM abuse_idx")
+            con.commit()
+        last = con.execute("SELECT COALESCE(MAX(id), 0) FROM abuse_idx").fetchone()[0]
+        added = 0
+        while True:
+            rows = con.execute(
+                "SELECT id, source, url, title, text, tags FROM items WHERE id > ? ORDER BY id LIMIT ?",
+                (last, chunk)).fetchall()
+            if not rows:
+                break
+            batch = []
+            for r in rows:
+                sc, hits = _qualifies(r[1], r[3], r[4], r[5] or "")
+                if sc:
+                    batch.append((r[0], sc, ",".join(hits)))
+                last = r[0]
+            if batch:
+                con.executemany("INSERT OR REPLACE INTO abuse_idx(id, score, tags) VALUES (?,?,?)", batch)
+                added += len(batch)
+            con.commit()
+        return added
+    finally:
+        con.close()
+
+
 def _rows(limit=400, tag=None, q=None, since_days=None):
-    sql = "SELECT id, source, url, title, text, published, tags, score FROM items"
+    """Читает из индекса abuse_idx; индекс добивается свежими записями перед чтением."""
+    try:
+        rebuild()
+    except Exception:
+        pass
+    sql = ("SELECT i.id, i.source, i.url, i.title, i.text, i.published, a.tags, i.score, a.score "
+           "FROM abuse_idx a JOIN items i ON i.id = a.id")
     where, args = [], []
+    if tag:
+        where.append("(',' || a.tags || ',') LIKE ?")
+        args.append("%%," + tag + ",%%")
     if q:
-        where.append("(title LIKE ? OR text LIKE ? OR source LIKE ?)")
+        where.append("(i.title LIKE ? OR i.text LIKE ? OR i.source LIKE ?)")
         args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q]
     if since_days:
-        where.append("published >= date('now', ?)")
+        where.append("i.published >= date('now', ?)")
         args.append("-%d day" % int(since_days))
     if where:
         sql += " WHERE " + " AND ".join(where)
-    sql += " ORDER BY id DESC LIMIT ?"
+    sql += " ORDER BY a.score DESC, i.id DESC LIMIT ?"
-    args.append(int(limit) * 6)
+    args.append(int(limit))
-    con = sqlite3.connect(str(DB), timeout=30)
+    con = _connect()
     try:
-        cur = con.execute(sql, args)
         out = []
-        for r in cur:
+        for r in con.execute(sql, args):
-            sc, hits = _qualifies(r[1], r[3], r[4], r[6] or "")
+            hits = [t for t in (r[6] or "").split(",") if t]
-            if not sc:
-                continue
-            if tag and tag not in hits:
-                continue
             out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
                         "text": (r[4] or "")[:600], "published": r[5],
-                        "tags": hits, "abuse_score": sc, "rank": r[7]})
+                        "tags": hits, "abuse_score": r[8], "rank": r[7]})
-            if len(out) >= limit:
-                break
         return out
     finally:
         con.close()
 
 
 def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
-    """limit — сколько отдать, window — сколько последних записей просканировать."""
+    """limit — сколько отдать; индекс уже отсортирован по весy, window не режет."""
-    items = _rows(limit=window, tag=tag, q=q, since_days=since_days)
+    items = _rows(limit=max(int(limit), 400), tag=tag, q=q, since_days=since_days)
     items.sort(key=lambda x: (-x["abuse_score"], -(x["id"] or 0)))
     counts = {}
     for it in items:

omni/cli.py

@@ -542,6 +542,13 @@ def main(argv=None):
     sp.add_argument("--apply", action="store_true")
     sp.set_defaults(fn=cmd_pump)
 
+    sp = sub.add_parser("tgsearch", help="найти Telegram-каналы по ключевым словам через свою сессию")
+    sp.add_argument("queries", help="запросы через запятую")
+    sp.add_argument("--limit", type=int, default=40, help="каналов на запрос")
+    sp.add_argument("--min-members", type=int, default=0, help="отсеять мелкие каналы")
+    sp.add_argument("--apply", action="store_true", help="дописать найденные каналы в data/radar_sources.json")
+    sp.set_defaults(fn=cmd_tgsearch)
+
     sp = sub.add_parser("keys", help="ключи парсеров из vault: сервисы, счётчики, остывание")
     sp.add_argument("--service", default="", help="только один сервис")
     sp.add_argument("-v", "--verbose", action="store_true", help="по ключам, масками")
@@ -562,3 +569,57 @@ def main(argv=None):
 
 if __name__ == "__main__":
     sys.exit(main())
+
+
+def cmd_tgsearch(a):
+    """Поиск каналов по ключевым словам -> пополнение пула источников."""
+    from . import tgsess
+    if not tgsess.enabled():
+        print("сессия не настроена: data/tg.json или OMNI_TG_* в env")
+        return 1
+    queries = [q.strip() for q in str(a.queries).split(",") if q.strip()]
+    found = {}
+    for q in queries:
+        try:
+            rows = tgsess.search(q, limit=a.limit)
+        except Exception as e:
+            print("  %-28s ошибка %s" % (q, type(e).__name__))
+            continue
+        for r in rows:
+            if a.min_members and (r.get("participants") or 0) < a.min_members:
+                continue
+            prev = found.get(r["username"])
+            if prev:
+                prev["hits"] = prev.get("hits", 1) + 1
+                continue
+            r["hits"] = 1
+            found[r["username"]] = r
+        print("  %-28s каналов %d (всего уникальных %d)" % (q, len(rows), len(found)))
+    rows = sorted(found.values(), key=lambda x: (-(x.get("hits") or 0), -(x.get("participants") or 0)))
+    for r in rows[:20]:
+        print("   @%-28s %7s участн. hits=%d %s" % (r["username"], r.get("participants") or 0,
+                                                   r["hits"], (r.get("title") or "")[:40]))
+    if not a.apply:
+        print("найдено %d каналов; добавь --apply чтобы дописать в пул" % len(rows))
+        return 0
+    import json
+    from pathlib import Path
+    p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
+    try:
+        d = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        d = {"domains": [], "channels": []}
+    chans = list(d.get("channels") or [])
+    have = {str(c).lstrip("@").lower() for c in chans}
+    added = 0
+    for r in rows:
+        u = str(r["username"]).lstrip("@")
+        if u.lower() in have:
+            continue
+        chans.append(u)
+        have.add(u.lower())
+        added += 1
+    d["channels"] = chans
+    p.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")
+    print("дописано каналов: %d | всего каналов в пуле: %d" % (added, len(chans)))
+    return 0

omni/feeds.py

@@ -709,6 +709,169 @@ def _x_user(src, limit=12):
     return xp.profile(handle, limit=max(4, min(limit, 25)))
 
 
+# ---------------- sitemap_recent: сайты без ленты, но со свежими url ----------------
+def _sitemap_recent(src, limit):
+    """sitemap.xml/sitemap_index.xml -> свежие url по lastmod.
+
+    extra: {"days": 14, "sitemaps": ["https://x/sitemap.xml"], "must": ["/blog/","/news/"]}
+    Даёт доступ к сайтам, где нет ни rss, ни списка статей в html.
+    """
+    import datetime as _d
+    days = int((src.extra or {}).get("days") or 21)
+    cut = (_d.datetime.utcnow() - _d.timedelta(days=days)).strftime("%Y-%m-%d")
+    must = tuple((src.extra or {}).get("must") or ())
+    starts = list((src.extra or {}).get("sitemaps") or [])
+    if not starts:
+        base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
+        starts = [base + "/sitemap.xml", base + "/sitemap_index.xml",
+                  base + "/sitemap-index.xml", base + "/wp-sitemap.xml"]
+    urls, seen_maps = [], set()
+
+    def load(u, depth=0):
+        if u in seen_maps or depth > 2 or len(seen_maps) > 12:
+            return
+        seen_maps.add(u)
+        r = net.get(u, headers=UA)
+        if r.status != 200 or "<" not in (r.text or ""):
+            return
+        sx = BeautifulSoup(r.text, "xml")
+        for sm in sx.find_all("sitemap"):
+            loc = (sm.find("loc").get_text(strip=True) if sm.find("loc") else "")
+            lm = sm.find("lastmod").get_text(strip=True) if sm.find("lastmod") else ""
+            if loc and (not lm or lm[:10] >= cut):
+                load(loc, depth + 1)
+        for ue in sx.find_all("url"):
+            loc = ue.find("loc").get_text(strip=True) if ue.find("loc") else ""
+            lm = ue.find("lastmod").get_text(strip=True) if ue.find("lastmod") else ""
+            if not loc:
+                continue
+            if lm and lm[:10] < cut:
+                continue
+            if must and not any(m in loc for m in must):
+                continue
+            urls.append((loc, lm))
+
+    for u in starts:
+        try:
+            load(u)
+        except Exception:
+            continue
+    # свежие сверху, без подстраниц-дублей
+    urls.sort(key=lambda x: x[1] or "", reverse=True)
+    out, seen = [], set()
+    for loc, lm in urls:
+        k = loc.rstrip("/")
+        if k in seen:
+            continue
+        seen.add(k)
+        tail = urllib.parse.urlsplit(loc).path.strip("/").rsplit("/", 1)[-1]
+        title = urllib.parse.unquote(tail).replace("-", " ").replace("_", " ")[:160] or loc
+        out.append(_item(src, "page", loc, title, "", "", lm))
+        if len(out) >= limit:
+            break
+    return out
+
+
+# ---------------- newapi_pricing: панели new-api -> бесплатные модели ----------------
+def _newapi_pricing(src, limit):
+    """Панель new-api: /api/pricing -> модели с нулевой ценой = бесплатный доступ.
+
+    Ищет не статьи про абузы, а сами раздачи: free-модели на живых панелях.
+    """
+    base = src.url.rstrip("/")
+    js = None
+    # часть панелей отдаёт каталог только простому UA (полный Chrome-UA -> пустое тело)
+    base_hdr = dict(UA) if isinstance(UA, dict) else {"User-Agent": str(UA)}
+    for ua in (base_hdr.get("User-Agent", ""), "Mozilla/5.0", "curl/8.5.0"):
+        r = net.get(base + "/api/pricing", headers={"User-Agent": ua, "Accept": "application/json"})
+        if r.status != 200 or not (r.text or "").strip().startswith(("{", "[")):
+            continue
+        try:
+            js = json.loads(r.text)
+            break
+        except Exception:
+            continue
+    if js is None:
+        return []
+    data = js.get("data") if isinstance(js, dict) else None
+    if not isinstance(data, list):
+        return []
+    free, paid = [], 0
+    for m in data:
+        if not isinstance(m, dict):
+            continue
+        name = m.get("model_name") or m.get("model") or ""
+        ratio = m.get("model_ratio")
+        cratio = m.get("completion_ratio")
+        try:
+            zero = (float(ratio or 0) == 0) and (float(cratio or 0) == 0)
+        except Exception:
+            zero = False
+        if name and zero:
+            free.append(name)
+        elif name:
+            paid += 1
+    if not free:
+        return []
+    host = urllib.parse.urlsplit(base).netloc or base
+    title = "панель %s: %d бесплатных моделей" % (host, len(free))
+    body = ("Бесплатные модели (цена 0): " + ", ".join(free[:40])
+            + ("\nвсего моделей на панели: %d" % (len(free) + paid)))
+    return [_item(src, "panel", base + "/api/pricing", title, body, "", "",
+                  {"free_models": free[:60], "free_count": len(free), "paid_count": paid})]
+
+
+def _github_search(src, limit):
+    """GitHub search API: свежие репозитории по запросу (free api, jailbreak, autoreg)."""
+    q = (src.extra or {}).get("q") or "free api"
+    sort = (src.extra or {}).get("sort") or "updated"
+    url = ("https://api.github.com/search/repositories?q=" + urllib.parse.quote(str(q))
+           + "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50))))
+    hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"}
+    tok = ((config.CONFIG.get("keys") or {}).get("github") or [])
+    if isinstance(tok, list) and tok:
+        hdr["Authorization"] = "Bearer " + str(tok[0])
+    r = net.get(url, headers=hdr)
+    if r.status != 200:
+        return []
+    try:
+        js = json.loads(r.text)
+    except Exception:
+        return []
+    out = []
+    for it in (js.get("items") or [])[:limit]:
+        title = it.get("full_name") or it.get("name") or ""
+        desc = it.get("description") or ""
+        url_ = it.get("html_url") or ""
+        upd = (it.get("updated_at") or "")[:10]
+        out.append(_item(src, "github", url_, title, desc, upd, "", {}))
+    return out
+
+
+def _hn_search(src, limit):
+    """Hacker News (Algolia): поиск по свежим историям — бесплатные API, сливы, LLM-абузы."""
+    q = (src.extra or {}).get("q") or "free api"
+    url = ("https://hn.algolia.com/api/v1/search_by_date?query=" + urllib.parse.quote(str(q))
+           + "&tags=story&hitsPerPage=" + str(max(1, min(int(limit), 50))))
+    r = net.get(url, headers={"User-Agent": "omni-parse", "Accept": "application/json"})
+    if r.status != 200:
+        return []
+    try:
+        js = json.loads(r.text)
+    except Exception:
+        return []
+    out = []
+    for h in (js.get("hits") or [])[:limit]:
+        title = h.get("title") or h.get("story_title") or ""
+        if not title:
+            continue
+        link = h.get("url") or ("https://news.ycombinator.com/item?id=%s" % h.get("objectID"))
+        ts = (h.get("created_at") or "")[:10]
+        body = (h.get("story_text") or "")[:400]
+        out.append(_item(src, "hn", link, title, body, ts, "", {}))
+    return out
+
+
 ADAPTERS = {
     "x_user": _x_user,
     "rss": _rss,
@@ -739,6 +902,10 @@ ADAPTERS = {
     "cve_circl": _cve_circl,
     "ghsa": _ghsa,
     "models_dev": _models_dev,
+    "sitemap_recent": _sitemap_recent,
+    "newapi_pricing": _newapi_pricing,
+    "github_search": _github_search,
+    "hn_search": _hn_search,
 
 }
 

omni/sources.py

@@ -245,6 +245,42 @@ def radar_sources():
     return rows
 
 
+_PANELS = {"rows": [], "at": 0.0}
+
+
+def panel_sources():
+    """Панели new-api из data/panels.json (tools/panel_hunt.py --apply).
+
+    Адаптер newapi_pricing тянет /api/pricing и отдаёт находку: сколько моделей
+    на панели бесплатны. Это не статьи про абузы, а сами раздачи API.
+    """
+    import json
+    from pathlib import Path
+    p = Path(__file__).resolve().parents[1] / "data" / "panels.json"
+    try:
+        mtime = p.stat().st_mtime
+    except OSError:
+        return []
+    if _PANELS["rows"] and _PANELS["at"] == mtime:
+        return _PANELS["rows"]
+    try:
+        d = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return []
+    rows = []
+    for r in d.get("panels") or []:
+        host = str(r.get("host") or "").strip()
+        if not host:
+            continue
+        name = "pn_" + host.replace(".", "_").replace("-", "_")
+        rows.append(Source(name=name, category="abuse", kind="site", url="https://" + host,
+                           adapter="newapi_pricing", tags=("panel", "abuse", "freeapi"),
+                           limit=1, detail=False,
+                           extra={"free_count": r.get("free_count") or 0, "models": r.get("models") or 0}))
+    _PANELS.update(at=mtime, rows=rows)
+    return rows
+
+
 def _gen_sources():
     """Сгенерированные наборы: китайские (sources_cn) + дарк-форумы (sources_dark)."""
     out = []
@@ -272,11 +308,15 @@ def all_sources():
              ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
         extra.append(Source(**d))
     seen, out = set(), []
-    for s in SOURCES + _gen_sources() + extra + radar_sources():
+    for s in SOURCES + _gen_sources() + extra + radar_sources() + panel_sources():
+        # дедуп строго по url: одна площадка = один источник, даже если пришла из
+        # реестра, radar_sources и panels.json одновременно (иначе двойной обход и
+        # двойные записи в базе). Побеждает первое вхождение — порядок списка выше.
         key = (s.url or "").rstrip("/").lower()
         if key and key in seen:
             continue
-        seen.add(key)
+        if key:
+            seen.add(key)
         out.append(s)
     return out
 

omni/sources_abuse.py

@@ -44,4 +44,31 @@ ABUSE_LIVE = [
     A("ab_telegrammassdm_com", "abuse", "html", "https://telegrammassdm.com/", "html_listing", ('abuse', 'abuse'), limit=20),
     A("ab_echo_tg", "abuse", "html", "https://echo.tg/", "html_listing", ('abuse', 'abuse'), limit=20),
     A("ab_seochao_ru", "abuse", "html", "https://seochao.ru/forums/arbitrazh-trafika.85/", "html_listing", ('abuse', 'abuse'), limit=20),
+    # --- новые способы: discourse-площадки, reddit-json, rss-агрегаторы, sitemap ---
+    A("ab_reddit_freebies", "abuse", "json", "https://www.reddit.com/r/freebies/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20),
+    A("ab_reddit_openaiapi", "abuse", "json", "https://www.reddit.com/r/OpenAIAPI/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=12),
+    A("ab_reddit_freeai", "abuse", "json", "https://www.reddit.com/r/FreeAI/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=12),
+    A("ab_rss_slickdeals_free", "abuse", "rss", "https://slickdeals.net/newsearch.php?rss=1&searcharea=deals&searchin=first&q=free", "rss", ('abuse', 'abuse', 'promo'), limit=20),
+    A("ab_sitemap_cpamafia", "abuse", "site", "https://cpamafia.pro/sitemap.xml", "sitemap_recent", ('abuse', 'abuse', 'sitemap'), limit=20, extra={"days": 30}),
+    A("ab_sitemap_carders", "abuse", "site", "https://carders.biz/sitemap.xml", "sitemap_recent", ('abuse', 'abuse', 'sitemap'), limit=20, extra={"days": 30}),
+
+    # ---- GitHub search: свежие репозитории-раздачи (свой url на запрос = без дедупа) ----
+    A("ab_gh_free_api_keys", "abuse", "site", "https://github.com/search?q=free+api+keys&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free api keys", "sort": "updated"}),
+    A("ab_gh_api_key_pool", "abuse", "site", "https://github.com/search?q=api+key+pool&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "api key pool", "sort": "updated"}),
+    A("ab_gh_free_gpt_api", "abuse", "site", "https://github.com/search?q=free+gpt+api&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free gpt api", "sort": "updated"}),
+    A("ab_gh_claude_api_free", "abuse", "site", "https://github.com/search?q=claude+api+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "claude api free", "sort": "updated"}),
+    A("ab_gh_one_api_new_api", "abuse", "site", "https://github.com/search?q=one-api+new-api&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "one-api new-api", "sort": "updated"}),
+    A("ab_gh_jailbreak_prompts", "abuse", "site", "https://github.com/search?q=jailbreak+prompts&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "jailbreak prompts", "sort": "updated"}),
+    A("ab_gh_account_generator", "abuse", "site", "https://github.com/search?q=account+generator&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "account generator", "sort": "updated"}),
+    A("ab_gh_proxy_pool_free", "abuse", "site", "https://github.com/search?q=proxy+pool+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "proxy pool free", "sort": "updated"}),
+    A("ab_gh_openai_reverse_proxy", "abuse", "site", "https://github.com/search?q=openai+reverse+proxy&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "openai reverse proxy", "sort": "updated"}),
+    A("ab_gh_ai_gateway_free", "abuse", "site", "https://github.com/search?q=ai+gateway+free&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "ai gateway free", "sort": "updated"}),
+
+    # ---- Hacker News (Algolia): свежие истории про бесплатные API и сливы ----
+    A("ab_hn_free_api_keys", "abuse", "site", "https://hn.algolia.com/?query=free+api+keys", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "free api keys"}),
+    A("ab_hn_openai_api_free", "abuse", "site", "https://hn.algolia.com/?query=openai+api+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "openai api free"}),
+    A("ab_hn_llm_jailbreak", "abuse", "site", "https://hn.algolia.com/?query=llm+jailbreak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "llm jailbreak"}),
+    A("ab_hn_account_generator", "abuse", "site", "https://hn.algolia.com/?query=account+generator", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "account generator"}),
+    A("ab_hn_api_key_leak", "abuse", "site", "https://hn.algolia.com/?query=api+key+leak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "api key leak"}),
+    A("ab_hn_chatgpt_free", "abuse", "site", "https://hn.algolia.com/?query=chatgpt+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "chatgpt free"}),
 ]

omni/tgsess.py

@@ -90,3 +90,45 @@ def fetch(handles, limit=None):
     api_hash = os.environ["OMNI_TG_API_HASH"]
     lim = int(limit or os.environ.get("OMNI_TG_LIMIT") or 100)
     return asyncio.run(_pull(list(handles), lim, session, api_id, api_hash))
+
+async def _search(query, limit, session, api_id, api_hash, timeout=60):
+    """Глобальный поиск каналов/групп по запросу (contacts.SearchRequest)."""
+    from telethon import TelegramClient
+    from telethon.tl.functions.contacts import SearchRequest
+
+    client = TelegramClient(session, api_id, api_hash, connection_retries=1, timeout=timeout / 2)
+    await asyncio.wait_for(client.connect(), timeout)
+    try:
+        res = await asyncio.wait_for(client(SearchRequest(q=str(query), limit=int(limit))), timeout)
+        out = []
+        for ch in getattr(res, "chats", []) or []:
+            uname = getattr(ch, "username", None)
+            if not uname:
+                continue
+            out.append({"username": uname, "title": getattr(ch, "title", "") or "",
+                        "id": getattr(ch, "id", None),
+                        "broadcast": bool(getattr(ch, "broadcast", False)),
+                        "megagroup": bool(getattr(ch, "megagroup", False)),
+                        "participants": getattr(ch, "participants_count", None) or 0,
+                        "query": str(query)})
+        return out
+    finally:
+        try:
+            await asyncio.wait_for(client.disconnect(), 10)
+        except Exception:
+            pass
+
+
+def search(query, limit=40):
+    """Каналы по ключевому слову. Пусто, если сессия не настроена."""
+    if not enabled():
+        return []
+    try:
+        from . import config
+        config.tg_cfg()
+    except Exception:
+        pass
+    session = os.environ["OMNI_TG_SESSION"]
+    api_id = int(os.environ["OMNI_TG_API_ID"])
+    api_hash = os.environ["OMNI_TG_API_HASH"]
+    return asyncio.run(_search(query, limit, session, api_id, api_hash))

tools/panel_hunt.py

@@ -0,0 +1,153 @@
+"""Охота за панелями new-api: /api/status -> /api/pricing -> бесплатные модели.
+
+Панель new-api (одна из самых частых раздач в абуз-каналах) отдаёт каталог
+моделей и цены публично. Нулевая цена = бесплатный доступ = находка для абузов.
+
+Запуск:
+    python tools/panel_hunt.py                  # домены из data/radar_sources.json
+    python tools/panel_hunt.py --hosts a.com b.com
+    python tools/panel_hunt.py --limit 400 --workers 24
+    python tools/panel_hunt.py --apply          # записать data/panels.json (источники omni)
+
+Пишет только пути/счётчики. Ключи и токены не трогает.
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import ssl
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DATA = ROOT / "data"
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0 Safari/537.36"
+CTX = ssl.create_default_context()
+CTX.check_hostname = False
+CTX.verify_mode = ssl.CERT_NONE
+MARKERS = ("HeaderNavModules", "system_name", "chats", "quota_per_unit", "display_in_currency", "announcements_enabled")
+
+
+def _get(url, timeout=8, cap=400000):
+    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout, context=CTX) as r:
+        return r.status, r.read(cap).decode("utf-8", "ignore")
+
+
+def _is_newapi(body: str) -> bool:
+    if '"data"' not in body:
+        return False
+    return any(m in body for m in MARKERS)
+
+
+def probe(host: str):
+    """-> dict | None. Проверяет сам хост и api.<host>."""
+    for h in (host, "api." + host if not host.startswith("api.") else host):
+        try:
+            st, body = _get("https://" + h + "/api/status", timeout=7)
+            if st != 200 or not _is_newapi(body):
+                continue
+        except Exception:
+            continue
+        rec = {"host": h, "status": "newapi", "models": 0, "free": [], "free_count": 0}
+        try:
+            st, body = _get("https://" + h + "/api/pricing", timeout=12)
+            if st == 200:
+                js = json.loads(body)
+                data = js.get("data") if isinstance(js, dict) else None
+                if isinstance(data, list):
+                    rec["models"] = len(data)
+                    free = []
+                    for m in data:
+                        if not isinstance(m, dict):
+                            continue
+                        try:
+                            zero = float(m.get("model_ratio") or 0) == 0 and float(m.get("completion_ratio") or 0) == 0
+                        except Exception:
+                            zero = False
+                        if zero and m.get("model_name"):
+                            free.append(m["model_name"])
+                    rec["free"] = free[:80]
+                    rec["free_count"] = len(free)
+        except Exception:
+            rec["pricing"] = "err"
+        return rec
+    return None
+
+
+def load_hosts(args) -> list:
+    if args.hosts:
+        return list(dict.fromkeys(args.hosts))
+    p = DATA / "radar_sources.json"
+    if not p.exists():
+        return []
+    try:
+        d = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return []
+    out = []
+    for item in d.get("domains") or []:
+        dom = item[0] if isinstance(item, list) else (item.get("domain") or "")
+        if dom:
+            out.append(str(dom))
+    return list(dict.fromkeys(out))
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--hosts", nargs="*", default=None)
+    ap.add_argument("--limit", type=int, default=0, help="0 = все домены")
+    ap.add_argument("--workers", type=int, default=24)
+    ap.add_argument("--apply", action="store_true", help="записать data/panels.json")
+    ap.add_argument("--min-free", type=int, default=0, help="только панели с >= N бесплатных моделей")
+    a = ap.parse_args()
+
+    hosts = load_hosts(a)
+    if a.limit:
+        hosts = hosts[:a.limit]
+    if not hosts:
+        print("нет доменов: сначала omni radar --apply")
+        return 1
+
+    res = []
+    with cf.ThreadPoolExecutor(max(4, a.workers)) as ex:
+        futs = {ex.submit(probe, h): h for h in hosts}
+        for i, f in enumerate(cf.as_completed(futs), 1):
+            try:
+                r = f.result()
+            except Exception:
+                r = None
+            if r:
+                res.append(r)
+                print("  %s | new-api | моделей %d | free %d" % (r["host"], r["models"], r["free_count"]))
+            if i % 200 == 0:
+                print("... проверено %d/%d, найдено %d" % (i, len(hosts), len(res)), flush=True)
+
+    res.sort(key=lambda x: x["free_count"], reverse=True)
+    if a.min_free:
+        res = [r for r in res if r["free_count"] >= a.min_free]
+    print("итог: проверено %d | панелей new-api %d | с бесплатными моделями %d | free-моделей всего %d"
+          % (len(hosts), len(res), sum(1 for r in res if r["free_count"]), sum(r["free_count"] for r in res)))
+
+    if a.apply:
+        out = DATA / "panels.json"
+        prev = {}
+        if out.exists():
+            try:
+                for r in json.loads(out.read_text(encoding="utf-8")).get("panels") or []:
+                    prev[r.get("host")] = r
+            except Exception:
+                prev = {}
+        for r in res:
+            prev[r["host"]] = r  # свежие данные перекрывают старые, мёртвые не затираются
+        payload = {"panels": sorted(prev.values(), key=lambda x: x.get("free_count") or 0, reverse=True)}
+        out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
+        print("записано: %s (панелей %d)" % (out, len(payload["panels"])))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())