feat: cookie-jar по хосту, CLI omni add для своих источников, README про ключи/прокси/куки/свой сервер
This commit is contained in:
25
omni/cli.py
25
omni/cli.py
@@ -241,6 +241,25 @@ def cmd_export(args):
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_add_src(a):
|
||||
"""Свой источник: пробуем url живым запросом, сохраняем только если отдал записи."""
|
||||
import re as _re
|
||||
import urllib.parse as _up
|
||||
from . import discover as _disc, source_state as _st
|
||||
url = (a.url or "").strip()
|
||||
if not url.startswith("http"):
|
||||
print("нужен url фида"); return 1
|
||||
ok, n, ad = _disc.probe(url, a.category, limit=6)
|
||||
if not ok:
|
||||
print("фид пустой или недоступен: %s" % ad); return 1
|
||||
name = "manual_" + _re.sub(r"[^a-z0-9]+", "_", _up.urlsplit(url).netloc.lower())[:32]
|
||||
kind = "rss" if ad == "rss" else "html"
|
||||
tags = [t.strip() for t in (a.tags or "").split(",") if t.strip()] or ["manual"]
|
||||
_st.add_custom(name, a.category, url, kind=kind, adapter=ad, tags=tags, limit=15)
|
||||
print("добавлен %s · %s · %d записей" % (name, ad, n))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
@@ -332,6 +351,12 @@ def main(argv=None):
|
||||
sp.add_argument("--limit", type=int, default=1000)
|
||||
sp.set_defaults(fn=cmd_export)
|
||||
|
||||
sp = sub.add_parser("add", help="добавить свой источник по url фида")
|
||||
sp.add_argument("url")
|
||||
sp.add_argument("--category", default="community")
|
||||
sp.add_argument("--tags", default="manual")
|
||||
sp.set_defaults(fn=cmd_add_src)
|
||||
|
||||
args = p.parse_args(argv)
|
||||
return args.fn(args)
|
||||
|
||||
|
||||
@@ -19,12 +19,28 @@ DEFAULTS = {
|
||||
"(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"),
|
||||
"proxy_mode": "off", # off | list | env
|
||||
"proxies": [],
|
||||
"cookies": {}, # host -> "a=b; c=d" (свои сессии к закрытым площадкам)
|
||||
}
|
||||
|
||||
KEYS = {} # имя -> значение
|
||||
CONFIG = {} # прочие настройки
|
||||
|
||||
|
||||
def cookies_for(host):
|
||||
"""Cookie-строка под хост: точное совпадение или суффикс (example.com -> www.example.com)."""
|
||||
ck = CONFIG.get("cookies") or {}
|
||||
if not ck:
|
||||
return ""
|
||||
host = (host or "").lower().split(":")[0]
|
||||
if host in ck:
|
||||
return ck[host]
|
||||
for k, v in ck.items():
|
||||
k = k.lower().lstrip(".")
|
||||
if host == k or host.endswith("." + k):
|
||||
return v
|
||||
return ""
|
||||
|
||||
|
||||
def _load_env_file(path):
|
||||
p = pathlib.Path(path)
|
||||
if not p.exists():
|
||||
@@ -53,6 +69,14 @@ def load():
|
||||
except Exception:
|
||||
pass
|
||||
CONFIG = cfg
|
||||
raw = os.environ.get("OMNI_COOKIES") or ""
|
||||
if raw:
|
||||
try:
|
||||
extra = json.loads(raw)
|
||||
if isinstance(extra, dict):
|
||||
cfg["cookies"] = {**(cfg.get("cookies") or {}), **extra}
|
||||
except Exception:
|
||||
pass
|
||||
for name in ("TAVILY_API_KEY", "EXA_API_KEY", "YOU_API_KEY", "FIRECRAWL_API_KEY",
|
||||
"JINA_API_KEY", "SERPER_API_KEY", "BRAVE_API_KEY", "X_BEARER",
|
||||
"REDDIT_CLIENT_ID", "REDDIT_SECRET", "TREG_TOKEN", "GITHUB_TOKEN"):
|
||||
|
||||
@@ -126,6 +126,9 @@ def get(url, headers=None, timeout=None, json_mode=False):
|
||||
h["Accept"] = "application/json"
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
ck = config.cookies_for(host)
|
||||
if ck:
|
||||
h["Cookie"] = ck
|
||||
|
||||
first = _try_httpx(url, h, timeout, None)
|
||||
if first.ok and not first.blocked:
|
||||
|
||||
Reference in New Issue
Block a user