реестр: дедуп источников строго по url (2 падавших теста зелёные); инкрементальный abuse-индекс + tgsearch из предыдущей сессии

omni/feeds.py

@@ -709,6 +709,169 @@ def _x_user(src, limit=12):
     return xp.profile(handle, limit=max(4, min(limit, 25)))
 
 
+# ---------------- sitemap_recent: сайты без ленты, но со свежими url ----------------
+def _sitemap_recent(src, limit):
+    """sitemap.xml/sitemap_index.xml -> свежие url по lastmod.
+
+    extra: {"days": 14, "sitemaps": ["https://x/sitemap.xml"], "must": ["/blog/","/news/"]}
+    Даёт доступ к сайтам, где нет ни rss, ни списка статей в html.
+    """
+    import datetime as _d
+    days = int((src.extra or {}).get("days") or 21)
+    cut = (_d.datetime.utcnow() - _d.timedelta(days=days)).strftime("%Y-%m-%d")
+    must = tuple((src.extra or {}).get("must") or ())
+    starts = list((src.extra or {}).get("sitemaps") or [])
+    if not starts:
+        base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
+        starts = [base + "/sitemap.xml", base + "/sitemap_index.xml",
+                  base + "/sitemap-index.xml", base + "/wp-sitemap.xml"]
+    urls, seen_maps = [], set()
+
+    def load(u, depth=0):
+        if u in seen_maps or depth > 2 or len(seen_maps) > 12:
+            return
+        seen_maps.add(u)
+        r = net.get(u, headers=UA)
+        if r.status != 200 or "<" not in (r.text or ""):
+            return
+        sx = BeautifulSoup(r.text, "xml")
+        for sm in sx.find_all("sitemap"):
+            loc = (sm.find("loc").get_text(strip=True) if sm.find("loc") else "")
+            lm = sm.find("lastmod").get_text(strip=True) if sm.find("lastmod") else ""
+            if loc and (not lm or lm[:10] >= cut):
+                load(loc, depth + 1)
+        for ue in sx.find_all("url"):
+            loc = ue.find("loc").get_text(strip=True) if ue.find("loc") else ""
+            lm = ue.find("lastmod").get_text(strip=True) if ue.find("lastmod") else ""
+            if not loc:
+                continue
+            if lm and lm[:10] < cut:
+                continue
+            if must and not any(m in loc for m in must):
+                continue
+            urls.append((loc, lm))
+
+    for u in starts:
+        try:
+            load(u)
+        except Exception:
+            continue
+    # свежие сверху, без подстраниц-дублей
+    urls.sort(key=lambda x: x[1] or "", reverse=True)
+    out, seen = [], set()
+    for loc, lm in urls:
+        k = loc.rstrip("/")
+        if k in seen:
+            continue
+        seen.add(k)
+        tail = urllib.parse.urlsplit(loc).path.strip("/").rsplit("/", 1)[-1]
+        title = urllib.parse.unquote(tail).replace("-", " ").replace("_", " ")[:160] or loc
+        out.append(_item(src, "page", loc, title, "", "", lm))
+        if len(out) >= limit:
+            break
+    return out
+
+
+# ---------------- newapi_pricing: панели new-api -> бесплатные модели ----------------
+def _newapi_pricing(src, limit):
+    """Панель new-api: /api/pricing -> модели с нулевой ценой = бесплатный доступ.
+
+    Ищет не статьи про абузы, а сами раздачи: free-модели на живых панелях.
+    """
+    base = src.url.rstrip("/")
+    js = None
+    # часть панелей отдаёт каталог только простому UA (полный Chrome-UA -> пустое тело)
+    base_hdr = dict(UA) if isinstance(UA, dict) else {"User-Agent": str(UA)}
+    for ua in (base_hdr.get("User-Agent", ""), "Mozilla/5.0", "curl/8.5.0"):
+        r = net.get(base + "/api/pricing", headers={"User-Agent": ua, "Accept": "application/json"})
+        if r.status != 200 or not (r.text or "").strip().startswith(("{", "[")):
+            continue
+        try:
+            js = json.loads(r.text)
+            break
+        except Exception:
+            continue
+    if js is None:
+        return []
+    data = js.get("data") if isinstance(js, dict) else None
+    if not isinstance(data, list):
+        return []
+    free, paid = [], 0
+    for m in data:
+        if not isinstance(m, dict):
+            continue
+        name = m.get("model_name") or m.get("model") or ""
+        ratio = m.get("model_ratio")
+        cratio = m.get("completion_ratio")
+        try:
+            zero = (float(ratio or 0) == 0) and (float(cratio or 0) == 0)
+        except Exception:
+            zero = False
+        if name and zero:
+            free.append(name)
+        elif name:
+            paid += 1
+    if not free:
+        return []
+    host = urllib.parse.urlsplit(base).netloc or base
+    title = "панель %s: %d бесплатных моделей" % (host, len(free))
+    body = ("Бесплатные модели (цена 0): " + ", ".join(free[:40])
+            + ("\nвсего моделей на панели: %d" % (len(free) + paid)))
+    return [_item(src, "panel", base + "/api/pricing", title, body, "", "",
+                  {"free_models": free[:60], "free_count": len(free), "paid_count": paid})]
+
+
+def _github_search(src, limit):
+    """GitHub search API: свежие репозитории по запросу (free api, jailbreak, autoreg)."""
+    q = (src.extra or {}).get("q") or "free api"
+    sort = (src.extra or {}).get("sort") or "updated"
+    url = ("https://api.github.com/search/repositories?q=" + urllib.parse.quote(str(q))
+           + "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50))))
+    hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"}
+    tok = ((config.CONFIG.get("keys") or {}).get("github") or [])
+    if isinstance(tok, list) and tok:
+        hdr["Authorization"] = "Bearer " + str(tok[0])
+    r = net.get(url, headers=hdr)
+    if r.status != 200:
+        return []
+    try:
+        js = json.loads(r.text)
+    except Exception:
+        return []
+    out = []
+    for it in (js.get("items") or [])[:limit]:
+        title = it.get("full_name") or it.get("name") or ""
+        desc = it.get("description") or ""
+        url_ = it.get("html_url") or ""
+        upd = (it.get("updated_at") or "")[:10]
+        out.append(_item(src, "github", url_, title, desc, upd, "", {}))
+    return out
+
+
+def _hn_search(src, limit):
+    """Hacker News (Algolia): поиск по свежим историям — бесплатные API, сливы, LLM-абузы."""
+    q = (src.extra or {}).get("q") or "free api"
+    url = ("https://hn.algolia.com/api/v1/search_by_date?query=" + urllib.parse.quote(str(q))
+           + "&tags=story&hitsPerPage=" + str(max(1, min(int(limit), 50))))
+    r = net.get(url, headers={"User-Agent": "omni-parse", "Accept": "application/json"})
+    if r.status != 200:
+        return []
+    try:
+        js = json.loads(r.text)
+    except Exception:
+        return []
+    out = []
+    for h in (js.get("hits") or [])[:limit]:
+        title = h.get("title") or h.get("story_title") or ""
+        if not title:
+            continue
+        link = h.get("url") or ("https://news.ycombinator.com/item?id=%s" % h.get("objectID"))
+        ts = (h.get("created_at") or "")[:10]
+        body = (h.get("story_text") or "")[:400]
+        out.append(_item(src, "hn", link, title, body, ts, "", {}))
+    return out
+
+
 ADAPTERS = {
     "x_user": _x_user,
     "rss": _rss,
@@ -739,6 +902,10 @@ ADAPTERS = {
     "cve_circl": _cve_circl,
     "ghsa": _ghsa,
     "models_dev": _models_dev,
+    "sitemap_recent": _sitemap_recent,
+    "newapi_pricing": _newapi_pricing,
+    "github_search": _github_search,
+    "hn_search": _hn_search,
 
 }