parser: ETag/304-кэш, 61 авто-источник из 81 кандидата (+RSS AI/security), правила farm/weights, тесты sources_auto

omni/net.py

@@ -10,6 +10,78 @@ _cfg = config.load()
 _lock = threading.Lock()
 _last = {}          # host -> ts
 
+# ---------- условный GET: ETag / If-Modified-Since ----------
+_DATA = __import__("pathlib").Path(__file__).resolve().parents[1] / "data"
+_COND = _DATA / "http_cond.json"
+_BODIES = _DATA / "http_cache"
+_COND_DB = None
+_cond_lock = threading.Lock()
+
+
+def _cond_load():
+    global _COND_DB
+    if _COND_DB is None:
+        try:
+            import json as _j
+            _COND_DB = _j.loads(_COND.read_text(encoding="utf-8"))
+        except Exception:
+            _COND_DB = {}
+    return _COND_DB
+
+
+def _cond_save():
+    try:
+        import json as _j
+        with _cond_lock:
+            _BODIES.mkdir(parents=True, exist_ok=True)
+            _COND.write_text(_j.dumps(_COND_DB, ensure_ascii=False), encoding="utf-8")
+    except Exception:
+        pass
+
+
+def _body_path(key):
+    import hashlib
+    return _BODIES / (hashlib.sha1(key.encode("utf-8", "replace")).hexdigest() + ".body")
+
+
+def _cond_ask(key, h):
+    rec = _cond_load().get(key) or {}
+    if rec.get("etag"):
+        h["If-None-Match"] = rec["etag"]
+    if rec.get("lm"):
+        h["If-Modified-Since"] = rec["lm"]
+    return bool(rec.get("etag") or rec.get("lm"))
+
+
+def _cond_store(key, resp):
+    if resp.status != 200 or not resp.text or len(resp.text) > 1_500_000:
+        return
+    hdr = {str(k).lower(): v for k, v in (resp.headers or {}).items()}
+    etag, lm = hdr.get("etag"), hdr.get("last-modified")
+    if not etag and not lm:
+        return
+    try:
+        _BODIES.mkdir(parents=True, exist_ok=True)
+        p = _body_path(key)
+        p.write_text(resp.text, encoding="utf-8")
+    except Exception:
+        return
+    _cond_load()[key] = {"etag": etag, "lm": lm, "body": p.name,
+                         "ts": int(time.time()), "size": len(resp.text)}
+    _cond_save()
+
+
+def _cond_hit(key):
+    rec = _cond_load().get(key)
+    if not rec or not rec.get("body"):
+        return None
+    try:
+        txt = (_BODIES / rec["body"]).read_text(encoding="utf-8")
+    except Exception:
+        return None
+    return Resp(key.split("|")[0], 200, txt, "cache-304", headers={"x-omni-cache": "304"})
+
+
 BLOCK_MARKS = ("just a moment", "cf-browser-verification", "attention required",
                "ddos-guard", "checking your browser", "enable javascript and cookies")
 
@@ -118,9 +190,11 @@ def post_json(url, payload, headers=None, timeout=None):
 
 
 def get(url, headers=None, timeout=None, json_mode=False, proxy=None):
-    """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.
+    """Лестница: httpx -> curl_cffi -> curl_cffi с прокси + условный GET (ETag/304).
 
     proxy=... — принудительный прокси для конкретного источника (напр. CF-форумы через DE).
+    Повторный запрос неизменного документа отдаётся из data/http_cache (304), это экономит
+    и трафик, и время прогона (RSS/HTML отдают ETag/Last-Modified почти всегда).
     """
     timeout = timeout or _cfg.get("timeout", 25)
     host = urllib.parse.urlsplit(url).netloc
@@ -135,23 +209,43 @@ def get(url, headers=None, timeout=None, json_mode=False, proxy=None):
     if ck:
         h["Cookie"] = ck
 
+    key = url + ("|" + proxy if proxy else "")
+    conditional = _cond_ask(key, h)
+
+    def finish(resp):
+        if conditional and resp.status in (304,):
+            hit = _cond_hit(key)
+            if hit:
+                return hit
+        if resp.status == 200:
+            _cond_store(key, resp)
+        return resp
+
     if proxy:
         forced = _try_curl(url, h, timeout, proxy)
         if forced.ok and not forced.blocked:
-            return forced
+            return finish(forced)
         alt = _try_httpx(url, h, timeout, proxy)
-        return alt if alt.status == 200 else forced
+        return finish(alt if alt.status == 200 else forced)
 
     first = _try_httpx(url, h, timeout, None)
+    if conditional and first.status == 304:
+        hit = _cond_hit(key)
+        if hit:
+            return hit
     if first.ok and not first.blocked:
-        return first
+        return finish(first)
     second = _try_curl(url, h, timeout, None)
+    if conditional and second.status == 304:
+        hit = _cond_hit(key)
+        if hit:
+            return hit
     if second.ok and not second.blocked:
-        return second
+        return finish(second)
     proxy = _pick_proxy()
     if proxy:
         third = _try_curl(url, h, timeout, proxy)
         if third.status == 200:
-            return third
-        return third
-    return second if second.status else first
+            return finish(third)
+        return finish(third)
+    return finish(second if second.status else first)