абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days

tools/check_candidates.py

@@ -0,0 +1,79 @@
+"""Проверка кандидатов в источники абузов: сколько items реально отдаёт адаптер."""
+import json
+import sys
+
+sys.path.insert(0, ".")
+
+from omni import feeds
+from types import SimpleNamespace
+
+CANDS = [
+    # TG-превью
+    ("ab_tg_free_apis", "tg_web", "https://t.me/s/free_apis", {}),
+    ("ab_tg_besplatno_chatgpt", "tg_web", "https://t.me/s/besplatno_chatgpt", {}),
+    ("ab_tg_gpt_frees", "tg_web", "https://t.me/s/gpt_frees", {}),
+    ("ab_tg_promokody", "tg_web", "https://t.me/s/promokody", {}),
+    ("ab_tg_promokodq", "tg_web", "https://t.me/s/promokodq", {}),
+    ("ab_tg_free_softiki", "tg_web", "https://t.me/s/free_softiki", {}),
+    ("ab_tg_baza_dannykh", "tg_web", "https://t.me/s/baza_dannykh_ru", {}),
+    ("ab_tg_donzo", "tg_web", "https://t.me/s/virtualnyydondon_aktivatsiadonzo", {}),
+    ("ab_tg_cracksoftiki", "tg_web", "https://t.me/s/cracksoftiki", {}),
+    ("ab_tg_cracksoftwux", "tg_web", "https://t.me/s/crackSoftiowux", {}),
+    ("ab_tg_taynikstore", "tg_web", "https://t.me/s/taynikstoree", {}),
+    ("ab_tg_nodelocrus", "tg_web", "https://t.me/s/NODELOCRUS", {}),
+    ("ab_tg_aivision", "tg_web", "https://t.me/s/a_vision_ai", {}),
+    ("ab_tg_freeapi_chat", "tg_web", "https://t.me/s/freeapi_chat", {}),
+    ("ab_tg_vpn_free", "tg_web", "https://t.me/s/vpn_free_servers", {}),
+    ("ab_tg_sms_free", "tg_web", "https://t.me/s/freenumbers", {}),
+    # reddit
+    ("ab_reddit_freebies2", "reddit_listing", "https://www.reddit.com/r/freebies/.json?limit=25", {}),
+    ("ab_reddit_selfhosted", "reddit_listing", "https://www.reddit.com/r/selfhosted/.json?limit=25", {}),
+    ("ab_reddit_localllama", "reddit_listing", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25", {}),
+    ("ab_reddit_chatgptcoding", "reddit_listing", "https://www.reddit.com/r/ChatGPTCoding/.json?limit=25", {}),
+    ("ab_reddit_beermoney", "reddit_listing", "https://www.reddit.com/r/beermoney/.json?limit=25", {}),
+    ("ab_reddit_sideloaded", "reddit_listing", "https://www.reddit.com/r/sideloaded/.json?limit=25", {}),
+    ("ab_reddit_piracy2", "reddit_listing", "https://www.reddit.com/r/Piracy/.json?limit=25", {}),
+    ("ab_reddit_freegame", "reddit_listing", "https://www.reddit.com/r/FreeGameFindings/.json?limit=25", {}),
+    # discourse
+    ("ab_dis_n8n", "discourse", "https://community.n8n.io/latest.json", {}),
+    ("ab_dis_jina", "discourse", "https://discourse.jina.ai/latest.json", {}),
+    ("ab_dis_cloudflare", "discourse", "https://community.cloudflare.com/latest.json", {}),
+    ("ab_dis_vercel", "discourse", "https://community.vercel.com/latest.json", {}),
+    # rss
+    ("ab_rss_slickdeals2", "rss", "https://slickdeals.net/newsearch.php?mode=frontpage&searcharea=deals&searchin=first&rss=1", {}),
+    ("ab_rss_dealnews", "rss", "https://www.dealnews.com/rss/", {}),
+    ("ab_rss_freeStuffTimes", "rss", "https://feeds.feedburner.com/free-stuff-times", {}),
+    ("ab_rss_humble", "rss", "https://www.humblebundle.com/rss", {}),
+    # sitemap
+    ("ab_sitemap_cracked", "sitemap_recent", "https://cracked.io/sitemap.xml", {"days": 30}),
+    ("ab_sitemap_nulled", "sitemap_recent", "https://nulled.to/sitemap.xml", {"days": 30}),
+    # github / hn
+    ("ab_gh_freeapi_keys", "github_search", "https://api.github.com/search/repositories", {"q": "free api keys pool", "sort": "updated"}),
+    ("ab_gh_leak_pool", "github_search", "https://api.github.com/search/repositories", {"q": "leaked api keys checker", "sort": "updated"}),
+    ("ab_hn_free_tokens", "hn_search", "https://hn.algolia.com/api/v1/search_by_date", {"q": "free tokens api"}),
+]
+
+out = []
+for name, adapter, url, extra in CANDS:
+    fn = feeds.ADAPTERS.get(adapter)
+    if not fn:
+        out.append((name, adapter, -1, "нет адаптера"))
+        continue
+    src = SimpleNamespace(name=name, category="abuse", kind="site", url=url,
+                          adapter=adapter, tags=("abuse",), limit=8, detail=False,
+                          extra=dict(extra))
+    try:
+        items = fn(src, 8)
+        n = len(items)
+        sample = (items[0].title or "")[:60] if items else ""
+    except Exception as e:
+        n, sample = -2, "ERR %s" % str(e)[:60]
+    out.append((name, adapter, n, sample))
+
+ok = [x for x in out if x[2] > 0]
+print("проверено:", len(out), "| живых:", len(ok))
+for name, adapter, n, sample in out:
+    print("%-28s %-16s %3s  %s" % (name, adapter, n, sample))
+json.dump([{"name": n, "adapter": a, "url": u, "extra": e, "items": c}
+           for (n, a, c, _), (_, _, u, e) in zip(out, CANDS)], open("data/_cand_checked.json", "w", encoding="utf-8"),
+          ensure_ascii=False, indent=1)