prod: чистая лента (дедуп перепечаток, анти-шум в score, категорийный вес), счётчик живых/мёртвых источников, таймауты net
This commit is contained in:
@@ -133,7 +133,8 @@ def _pick_proxy():
|
||||
return random.choice(ps) if ps else None
|
||||
|
||||
|
||||
def _try_httpx(url, headers, timeout, proxy):
|
||||
def _try_httpx(url, headers, timeout, proxy, _min=10):
|
||||
timeout = max(_min, min(timeout, 12))
|
||||
try:
|
||||
import httpx
|
||||
kw = {"headers": headers, "timeout": timeout, "follow_redirects": True}
|
||||
@@ -148,6 +149,7 @@ def _try_httpx(url, headers, timeout, proxy):
|
||||
|
||||
|
||||
def _try_curl(url, headers, timeout, proxy, impersonate="chrome131"):
|
||||
timeout = max(timeout, 8)
|
||||
try:
|
||||
from curl_cffi import requests as cr
|
||||
kw = {"headers": headers, "timeout": timeout, "impersonate": impersonate, "allow_redirects": True}
|
||||
|
||||
Reference in New Issue
Block a user