"""Сетевой слой: httpx -> curl_cffi (TLS Chrome) -> прокси. Троттлинг по хосту.""" import random import threading import time import urllib.parse from . import config _cfg = config.load() _lock = threading.Lock() _last = {} # host -> ts BLOCK_MARKS = ("just a moment", "cf-browser-verification", "attention required", "ddos-guard", "checking your browser", "enable javascript and cookies") class Resp: __slots__ = ("url", "status", "text", "mode", "error", "headers", "elapsed") def __init__(self, url, status=0, text="", mode="", error="", headers=None, elapsed=0.0): self.url, self.status, self.text = url, status, text self.mode, self.error, self.headers, self.elapsed = mode, error, headers or {}, elapsed @property def ok(self): return self.status == 200 and len(self.text) > 50 @property def blocked(self): if self.status in (403, 429, 503): return True low = self.text[:4000].lower() return any(m in low for m in BLOCK_MARKS) def json(self): try: import json return json.loads(self.text) except Exception: return None def __repr__(self): return f"" def _throttle(host): """Слот резервируется под локом ДО сна: иначе N потоков проходят одновременно. Замер до фикса: 24 вызова на хост при 0.3с == 0.9с вместо 6.9с.""" delay = float(_cfg.get("per_host_delay", 0.6)) with _lock: now = time.time() prev = _last.get(host, 0) wait = max(0.0, delay - (now - prev)) _last[host] = now + wait if wait > 0: time.sleep(wait) def _pick_proxy(): ps = config.proxies() return random.choice(ps) if ps else None def _try_httpx(url, headers, timeout, proxy): try: import httpx kw = {"headers": headers, "timeout": timeout, "follow_redirects": True} if proxy: kw["proxy"] = proxy with httpx.Client(**kw) as c: r = c.get(url) return Resp(url, r.status_code, r.text[: _cfg.get("max_bytes", 4_000_000)], "proxy+httpx" if proxy else "httpx", headers=dict(r.headers)) except Exception as e: return Resp(url, 0, "", "httpx", f"{type(e).__name__}: {e}"[:160]) def _try_curl(url, headers, timeout, proxy, impersonate="chrome131"): try: from curl_cffi import requests as cr kw = {"headers": headers, "timeout": timeout, "impersonate": impersonate, "allow_redirects": True} if proxy: kw["proxies"] = {"http": proxy, "https": proxy} r = cr.get(url, **kw) return Resp(url, r.status_code, (r.text or "")[: _cfg.get("max_bytes", 4_000_000)], "proxy+curl" if proxy else "curl", headers=dict(r.headers)) except Exception as e: return Resp(url, 0, "", "curl", f"{type(e).__name__}: {e}"[:160]) def post_json(url, payload, headers=None, timeout=None): """POST JSON: httpx, при блоке/ошибке повтор через curl_cffi.""" import json timeout = timeout or _cfg.get("timeout", 25) host = urllib.parse.urlsplit(url).netloc _throttle(host) h = {"User-Agent": _cfg["ua"], "Content-Type": "application/json", "Accept": "application/json"} h.update(headers or {}) body = json.dumps(payload).encode() try: import httpx with httpx.Client(timeout=timeout, follow_redirects=True) as c: r = c.post(url, content=body, headers=h) txt = r.text[:_cfg.get("max_bytes", 4_000_000)] if r.status_code == 200: return Resp(url, r.status_code, txt, "httpx-post", headers=dict(r.headers)) except Exception as e: txt, code = "", 0 err = f"{type(e).__name__}: {e}"[:160] _ = err try: from curl_cffi import requests as cr r = cr.post(url, data=body, headers=h, timeout=timeout, impersonate="chrome131") return Resp(url, r.status_code, (r.text or "")[:_cfg.get("max_bytes", 4_000_000)], "curl-post", headers=dict(r.headers)) except Exception as e: return Resp(url, 0, "", "curl-post", f"{type(e).__name__}: {e}"[:160]) def get(url, headers=None, timeout=None, json_mode=False): """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.""" timeout = timeout or _cfg.get("timeout", 25) host = urllib.parse.urlsplit(url).netloc _throttle(host) h = {"User-Agent": _cfg["ua"], "Accept": "text/html,application/json;q=0.9,*/*;q=0.8", "Accept-Language": "ru,en;q=0.8"} if json_mode: h["Accept"] = "application/json" h["Content-Type"] = "application/json" h.update(headers or {}) ck = config.cookies_for(host) if ck: h["Cookie"] = ck first = _try_httpx(url, h, timeout, None) if first.ok and not first.blocked: return first second = _try_curl(url, h, timeout, None) if second.ok and not second.blocked: return second proxy = _pick_proxy() if proxy: third = _try_curl(url, h, timeout, proxy) if third.status == 200: return third return third return second if second.status else first