parser: ETag/304-кэш, 61 авто-источник из 81 кандидата (+RSS AI/security), правила farm/weights, тесты sources_auto
This commit is contained in:
Vlad Gothbreach
@@ -18,6 +18,9 @@ RULES = [
|
||||
r"sk-[A-Za-z0-9_\-]{20,}", r"gh[pousr]_[A-Za-z0-9]{20,}", r"xox[baprs]-[A-Za-z0-9\-]{10,}",
|
||||
r"(разда|выда|дарю|бесплатн|free|giveaway)[^.]{0,40}(ключ|key|токен|token)",
|
||||
r"(ключ|key|токен)[^.]{0,25}(бесплатн|free|дарю|раздач|giveaway)",
|
||||
r"sk-ws-[A-Za-z0-9_\-]{16,}", r"ek_live_[A-Za-z0-9_\-]{10,}", r"abl_[A-Za-z0-9]{16,}",
|
||||
r"xpl_[A-Za-z0-9]{16,}", r"AIza[0-9A-Za-z_\-]{30,}", r"ak_[A-Za-z0-9]{20,}",
|
||||
r"kr/claude", r"sk-or-v1-[A-Za-z0-9]{20,}",
|
||||
]),
|
||||
("promo", "промо", 26, [
|
||||
r"промокод", r"promo\s?code", r"промо[\s-]?код", r"скидк[аи]\s+\d{1,3}", r"\d{1,3}\s?%\s?(off|скидк)",
|
||||
@@ -37,6 +40,8 @@ RULES = [
|
||||
("tool", "софт", 18, [
|
||||
r"софт", r"кряк", r"crack(ed)?", r"bypass", r"обход\s+(блокиров|бан|защит)",
|
||||
r"авторег", r"autoreg", r"panel", r"панел[ьи]\s", r"рассылк", r"sender", r"скрипт",
|
||||
r"new-?api", r"one-?api", r"sub2api", r"chat2api", r"gpt-?load", r"api\s*(?:реселл|resell)",
|
||||
r"openai[\s-]?compatib", r"reverse\s+proxy", r"shim",
|
||||
]),
|
||||
("resale", "продажа", 14, [
|
||||
r"продаю", r"продам", r"куплю\s+аккаунт", r"\bfor\s+sale\b", r"цена\s*[:$]", r"\$\d+\s?/\s?(мес|mo|1k|1000)",
|
||||
@@ -54,6 +59,15 @@ RULES = [
|
||||
r"free\s+(?:api\s+)?credits?", r"\b\d+\s?(?:k|m)?\s?tokens?\s+(?:free|бесплатн)", r"free\s+quota",
|
||||
r"бесплатн\w*\s+(?:лимит|квот|токен)", r"кредит\w*\s+(?:бесплатн|в\s+подарок)", r"赠金", r"额度",
|
||||
r"unlimited\s+(?:api|access|tokens|usage)", r"безлимит\w*\s+(?:api|ключ|доступ|токен)",
|
||||
r"\$\d{2,4}\s+(?:in\s+)?(?:free\s+)?credits?", r"credits?\s+\$\d{2,4}", r"\$\d{2,4}\s*(?:free|в\s+подарок)",
|
||||
]),
|
||||
("farm", "фермы/автоматизация", 18, [
|
||||
r"авторег", r"autoreg", r"farm(?:er|ing)?", r"много\s*аккаунт", r"массов\w+\s+регистрац",
|
||||
r"аккаунт\w*\s+оптом", r"прокси[\s-]?пул", r"proxy\s+pool", r"ротаци\w*\s+(?:ip|прокси)",
|
||||
]),
|
||||
("weights", "модели/веса", 20, [
|
||||
r"abliterated", r"uncensored", r"GGUF", r"weights?[^.]{0,25}(?:download|слив|huggingface)",
|
||||
r"без\s+цензур", r"обход\s+модераци",
|
||||
]),
|
||||
("invite", "инвайты/беты", 16, [
|
||||
r"\binvite\s?(?:code|link|only)\b", r"инвайт", r"\bwaitlist\b", r"лист\s+ожидан", r"early\s+access",
|
||||
@@ -98,7 +112,8 @@ RULES = [
|
||||
|
||||
COMPILED = [(tag, label, w, [re.compile(p, re.I) for p in pats]) for tag, label, w, pats in RULES]
|
||||
TAG_LABEL = {t: l for t, l, _, _ in RULES}
|
||||
STRONG = {"keys", "promo", "trial", "resale", "accs", "credits", "freeapi", "ltd", "combo"}
|
||||
STRONG = {"keys", "promo", "trial", "resale", "accs", "credits", "freeapi", "ltd", "combo",
|
||||
"farm", "weights"}
|
||||
# на специализированных абуз-площадках хватает одного признака: сам источник тематический
|
||||
ABUSE_SRC_SC = 14
|
||||
# категории-источники, которые вообще имеют смысл для абузов
|
||||
|
||||
110
omni/net.py
110
omni/net.py
@@ -10,6 +10,78 @@ _cfg = config.load()
|
||||
_lock = threading.Lock()
|
||||
_last = {} # host -> ts
|
||||
|
||||
# ---------- условный GET: ETag / If-Modified-Since ----------
|
||||
_DATA = __import__("pathlib").Path(__file__).resolve().parents[1] / "data"
|
||||
_COND = _DATA / "http_cond.json"
|
||||
_BODIES = _DATA / "http_cache"
|
||||
_COND_DB = None
|
||||
_cond_lock = threading.Lock()
|
||||
|
||||
|
||||
def _cond_load():
|
||||
global _COND_DB
|
||||
if _COND_DB is None:
|
||||
try:
|
||||
import json as _j
|
||||
_COND_DB = _j.loads(_COND.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
_COND_DB = {}
|
||||
return _COND_DB
|
||||
|
||||
|
||||
def _cond_save():
|
||||
try:
|
||||
import json as _j
|
||||
with _cond_lock:
|
||||
_BODIES.mkdir(parents=True, exist_ok=True)
|
||||
_COND.write_text(_j.dumps(_COND_DB, ensure_ascii=False), encoding="utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _body_path(key):
|
||||
import hashlib
|
||||
return _BODIES / (hashlib.sha1(key.encode("utf-8", "replace")).hexdigest() + ".body")
|
||||
|
||||
|
||||
def _cond_ask(key, h):
|
||||
rec = _cond_load().get(key) or {}
|
||||
if rec.get("etag"):
|
||||
h["If-None-Match"] = rec["etag"]
|
||||
if rec.get("lm"):
|
||||
h["If-Modified-Since"] = rec["lm"]
|
||||
return bool(rec.get("etag") or rec.get("lm"))
|
||||
|
||||
|
||||
def _cond_store(key, resp):
|
||||
if resp.status != 200 or not resp.text or len(resp.text) > 1_500_000:
|
||||
return
|
||||
hdr = {str(k).lower(): v for k, v in (resp.headers or {}).items()}
|
||||
etag, lm = hdr.get("etag"), hdr.get("last-modified")
|
||||
if not etag and not lm:
|
||||
return
|
||||
try:
|
||||
_BODIES.mkdir(parents=True, exist_ok=True)
|
||||
p = _body_path(key)
|
||||
p.write_text(resp.text, encoding="utf-8")
|
||||
except Exception:
|
||||
return
|
||||
_cond_load()[key] = {"etag": etag, "lm": lm, "body": p.name,
|
||||
"ts": int(time.time()), "size": len(resp.text)}
|
||||
_cond_save()
|
||||
|
||||
|
||||
def _cond_hit(key):
|
||||
rec = _cond_load().get(key)
|
||||
if not rec or not rec.get("body"):
|
||||
return None
|
||||
try:
|
||||
txt = (_BODIES / rec["body"]).read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
return None
|
||||
return Resp(key.split("|")[0], 200, txt, "cache-304", headers={"x-omni-cache": "304"})
|
||||
|
||||
|
||||
BLOCK_MARKS = ("just a moment", "cf-browser-verification", "attention required",
|
||||
"ddos-guard", "checking your browser", "enable javascript and cookies")
|
||||
|
||||
@@ -118,9 +190,11 @@ def post_json(url, payload, headers=None, timeout=None):
|
||||
|
||||
|
||||
def get(url, headers=None, timeout=None, json_mode=False, proxy=None):
|
||||
"""Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.
|
||||
"""Лестница: httpx -> curl_cffi -> curl_cffi с прокси + условный GET (ETag/304).
|
||||
|
||||
proxy=... — принудительный прокси для конкретного источника (напр. CF-форумы через DE).
|
||||
Повторный запрос неизменного документа отдаётся из data/http_cache (304), это экономит
|
||||
и трафик, и время прогона (RSS/HTML отдают ETag/Last-Modified почти всегда).
|
||||
"""
|
||||
timeout = timeout or _cfg.get("timeout", 25)
|
||||
host = urllib.parse.urlsplit(url).netloc
|
||||
@@ -135,23 +209,43 @@ def get(url, headers=None, timeout=None, json_mode=False, proxy=None):
|
||||
if ck:
|
||||
h["Cookie"] = ck
|
||||
|
||||
key = url + ("|" + proxy if proxy else "")
|
||||
conditional = _cond_ask(key, h)
|
||||
|
||||
def finish(resp):
|
||||
if conditional and resp.status in (304,):
|
||||
hit = _cond_hit(key)
|
||||
if hit:
|
||||
return hit
|
||||
if resp.status == 200:
|
||||
_cond_store(key, resp)
|
||||
return resp
|
||||
|
||||
if proxy:
|
||||
forced = _try_curl(url, h, timeout, proxy)
|
||||
if forced.ok and not forced.blocked:
|
||||
return forced
|
||||
return finish(forced)
|
||||
alt = _try_httpx(url, h, timeout, proxy)
|
||||
return alt if alt.status == 200 else forced
|
||||
return finish(alt if alt.status == 200 else forced)
|
||||
|
||||
first = _try_httpx(url, h, timeout, None)
|
||||
if conditional and first.status == 304:
|
||||
hit = _cond_hit(key)
|
||||
if hit:
|
||||
return hit
|
||||
if first.ok and not first.blocked:
|
||||
return first
|
||||
return finish(first)
|
||||
second = _try_curl(url, h, timeout, None)
|
||||
if conditional and second.status == 304:
|
||||
hit = _cond_hit(key)
|
||||
if hit:
|
||||
return hit
|
||||
if second.ok and not second.blocked:
|
||||
return second
|
||||
return finish(second)
|
||||
proxy = _pick_proxy()
|
||||
if proxy:
|
||||
third = _try_curl(url, h, timeout, proxy)
|
||||
if third.status == 200:
|
||||
return third
|
||||
return third
|
||||
return second if second.status else first
|
||||
return finish(third)
|
||||
return finish(third)
|
||||
return finish(second if second.status else first)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
|
||||
(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.
|
||||
|
||||
Обновлено: 2026-10-27 16:41 UTC · живых 46 из 52
|
||||
Обновлено: 2026-10-27 17:19 UTC · живых 61 из 81
|
||||
"""
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ AUTO = [
|
||||
A("aa_gitlab_llm", "json", "https://gitlab.com/api/v4/projects?search=llm&sort=desc&order_by=created_at&per_page=25", "generic_json", ("abuse"), extra={"url_key": "web_url"}),
|
||||
A("aa_npm_free_api", "json", "https://registry.npmjs.org/-/v1/search?text=free%20api%20keys&size=25", "generic_json", ("abuse"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}),
|
||||
A("aa_crates_proxy", "json", "https://crates.io/api/v1/crates?q=openai%20proxy&per_page=20", "generic_json", ("abuse"), extra={"rows": "crates", "base": "https://crates.io/crates/", "slug_key": "name"}),
|
||||
A("aa_json_gh_advisories", "json", "https://api.github.com/advisories?per_page=30", "generic_json", ("abuse"), extra={"url_key": "html_url", "title_key": "summary", "date_key": "published_at"}),
|
||||
|
||||
# ---------- Reddit через pullpush.io (без ключа) ----------
|
||||
A("aa_pp_freeai", "json", "https://api.pullpush.io/reddit/search/submission/?subreddit=FreeAI&size=25", "generic_json", ("abuse"), extra={"rows": "data", "url_key": "url", "title_key": "title", "text_key": "selftext", "date_key": "created_utc"}),
|
||||
@@ -63,6 +64,20 @@ AUTO = [
|
||||
A("aa_rss_hf_blog", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("abuse")),
|
||||
A("aa_rss_tc_ai", "rss", "https://techcrunch.com/category/artificial-intelligence/feed/", "rss", ("abuse")),
|
||||
A("aa_rss_52pojie", "rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", ("abuse")),
|
||||
A("aa_rss_latentspace", "rss", "https://www.latent.space/feed", "rss", ("abuse")),
|
||||
A("aa_rss_interconnects", "rss", "https://www.interconnects.ai/feed", "rss", ("abuse")),
|
||||
A("aa_rss_importai", "rss", "https://importai.substack.com/feed", "rss", ("abuse")),
|
||||
A("aa_rss_tldr_ai", "rss", "https://tldr.tech/api/rss/ai", "rss", ("abuse")),
|
||||
A("aa_rss_smol", "rss", "https://news.smol.ai/rss.xml", "rss", ("abuse")),
|
||||
A("aa_rss_aisupremacy", "rss", "https://aisupremacy.substack.com/feed", "rss", ("abuse")),
|
||||
A("aa_rss_snyk", "rss", "https://snyk.io/blog/feed/", "rss", ("abuse")),
|
||||
A("aa_rss_jfrog", "rss", "https://jfrog.com/blog/feed/", "rss", ("abuse")),
|
||||
A("aa_rss_wiz", "rss", "https://www.wiz.io/blog/rss.xml", "rss", ("abuse")),
|
||||
A("aa_rss_upguard", "rss", "https://www.upguard.com/blog/rss.xml", "rss", ("abuse")),
|
||||
A("aa_rss_securityweek", "rss", "https://www.securityweek.com/feed/", "rss", ("abuse")),
|
||||
A("aa_rss_cloudflare", "rss", "https://blog.cloudflare.com/rss/", "rss", ("abuse")),
|
||||
A("aa_rss_bleeping", "rss", "https://www.bleepingcomputer.com/feed/", "rss", ("abuse")),
|
||||
A("aa_rss_thn", "rss", "https://feeds.feedburner.com/TheHackersNews", "rss", ("abuse")),
|
||||
|
||||
# ---------- Telegram-превью ----------
|
||||
A("aa_tg_ai_newz", "html", "https://t.me/s/ai_newz", "tg_web", ("abuse", "tg")),
|
||||
|
||||
Reference in New Issue
Block a user