omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/config.py

@@ -0,0 +1,73 @@
+"""Конфиг: пути, лимиты, ключи из env/.env. Никаких секретов в коде."""
+import json
+import os
+import pathlib
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+DATA = pathlib.Path(os.environ.get("OMNI_DATA") or (ROOT / "data"))
+DB = pathlib.Path(os.environ.get("OMNI_DB") or (DATA / "omni.sqlite3"))
+THEMES = ROOT / "themes.txt"
+
+DEFAULTS = {
+    "timeout": 25,
+    "retries": 2,
+    "per_host_delay": 0.6,
+    "max_bytes": 4_000_000,
+    "limit": 20,
+    "lang": "ru",
+    "ua": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+           "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"),
+    "proxy_mode": "off",   # off | list | env
+    "proxies": [],
+}
+
+KEYS = {}          # имя -> значение
+CONFIG = {}        # прочие настройки
+
+
+def _load_env_file(path):
+    p = pathlib.Path(path)
+    if not p.exists():
+        return {}
+    out = {}
+    for line in p.read_text(encoding="utf-8", errors="ignore").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        out[k.strip()] = v.strip().strip('"').strip("'")
+    return out
+
+
+def load():
+    """Читает .env рядом с проектом и config.json (если есть), собирает ключи."""
+    global KEYS, CONFIG
+    envf = _load_env_file(ROOT / ".env")
+    for k, v in envf.items():
+        os.environ.setdefault(k, v)
+    cfg = dict(DEFAULTS)
+    cf = ROOT / "config.json"
+    if cf.exists():
+        try:
+            cfg.update(json.loads(cf.read_text(encoding="utf-8")))
+        except Exception:
+            pass
+    CONFIG = cfg
+    for name in ("TAVILY_API_KEY", "EXA_API_KEY", "YOU_API_KEY", "FIRECRAWL_API_KEY",
+                 "JINA_API_KEY", "SERPER_API_KEY", "BRAVE_API_KEY", "X_BEARER",
+                 "REDDIT_CLIENT_ID", "REDDIT_SECRET", "TREG_TOKEN", "GITHUB_TOKEN"):
+        v = os.environ.get(name)
+        if v:
+            KEYS[name] = v
+    DATA.mkdir(parents=True, exist_ok=True)
+    return cfg
+
+
+def proxies():
+    mode = CONFIG.get("proxy_mode", "off")
+    if mode == "env":
+        v = os.environ.get("OMNI_PROXIES", "")
+        return [x.strip() for x in v.split(",") if x.strip()]
+    if mode == "list":
+        return list(CONFIG.get("proxies") or [])
+    return []