tg-сессия в комбайне + панель: ключи/сессия в шапке + фикс пустого radar_sources
- omni/config.py: tg_cfg() читает data/tg.json (вне git), env приоритетнее
- omni/tgsess.py: enabled()/fetch() подтягивают креды из config
- tools/tg_pick.py: перебор пула сессий, проверка is_user_authorized, личные аккаунты исключены
- omni claw pump: 2 канала -> 27 постов через сессию (проверено)
- omni/serve.py: /api/stats += keys{totals}, tg{enabled}
- web/index.html: строка статуса в шапке (ключей / сервисов / TG сессия)
- omni/cli.py radar --apply: пишет полный набор, а не только новые (повторный прогон обнулял файл)
- omni/cli.py radar --import-posts: --since-days + скип известных url до simhash
This commit is contained in:
ENI
36
omni/cli.py
36
omni/cli.py
@@ -1,5 +1,6 @@
|
||||
"""CLI: providers / search / fanout / cards / top / find / stats / export."""
|
||||
import argparse
|
||||
import datetime as _dt
|
||||
import pathlib
|
||||
import concurrent.futures
|
||||
import json
|
||||
@@ -341,7 +342,15 @@ def cmd_radar(a):
|
||||
c = _sq.connect(db)
|
||||
doms = c.execute("SELECT domain, score, info FROM domains ORDER BY score DESC").fetchall()
|
||||
chans = c.execute("SELECT name, ok, fails FROM channels WHERE ok=1").fetchall()
|
||||
posts = c.execute("SELECT chan, mid, date, text FROM messages").fetchall() if a.import_posts else []
|
||||
if a.import_posts:
|
||||
if a.since_days:
|
||||
cutoff = _dt.datetime.utcnow() - _dt.timedelta(days=a.since_days)
|
||||
posts = c.execute("SELECT chan, mid, date, text FROM messages WHERE date >= ?",
|
||||
(cutoff.strftime("%Y-%m-%d"),)).fetchall()
|
||||
else:
|
||||
posts = c.execute("SELECT chan, mid, date, text FROM messages").fetchall()
|
||||
else:
|
||||
posts = []
|
||||
c.close()
|
||||
|
||||
if a.import_posts:
|
||||
@@ -353,12 +362,19 @@ def cmd_radar(a):
|
||||
return ((txt or "").strip().splitlines() or [""])[0][:150]
|
||||
|
||||
items, seen_u = [], set()
|
||||
known = set()
|
||||
try:
|
||||
with store.db() as kc:
|
||||
for (u,) in kc.execute("SELECT url FROM items WHERE url LIKE 'https://t.me/%'"):
|
||||
known.add(u)
|
||||
except Exception:
|
||||
known = set()
|
||||
for chan, mid, date, text in posts:
|
||||
text = (text or "").strip()
|
||||
if len(text) < 30:
|
||||
continue
|
||||
url_u = "https://t.me/%s/%s" % (chan, mid)
|
||||
if url_u in seen_u:
|
||||
if url_u in seen_u or url_u in known:
|
||||
continue
|
||||
seen_u.add(url_u)
|
||||
items.append(Item("rad_" + chan, "post", url=url_u, title=head_line(text), text=text,
|
||||
@@ -376,20 +392,23 @@ def cmd_radar(a):
|
||||
have = {s.url.rstrip('/').lower() for s in _src.all_sources()}
|
||||
have_names = {s.name for s in _src.all_sources()}
|
||||
|
||||
full_dom, full_chan = [], []
|
||||
new_dom, new_chan = [], []
|
||||
for dom, score, info in doms:
|
||||
if a.min_score and (score or 0) < a.min_score:
|
||||
continue
|
||||
url = "https://" + dom
|
||||
if url.lower() in have or ("rad_" + dom.replace('.', '_').replace('-', '_')) in have_names:
|
||||
continue
|
||||
title = ""
|
||||
try:
|
||||
title = (_json.loads(info or "{}") or {}).get("title") or ""
|
||||
except Exception:
|
||||
pass
|
||||
full_dom.append((dom, score, title))
|
||||
url = "https://" + dom
|
||||
if url.lower() in have or ("rad_" + dom.replace('.', '_').replace('-', '_')) in have_names:
|
||||
continue
|
||||
new_dom.append((dom, score, title))
|
||||
for name, ok, fails in chans:
|
||||
full_chan.append(name)
|
||||
if ("tg_" + name) in have_names:
|
||||
continue
|
||||
new_chan.append(name)
|
||||
@@ -408,9 +427,11 @@ def cmd_radar(a):
|
||||
return 0
|
||||
|
||||
out = _P("data") / "radar_sources.json"
|
||||
out.write_text(_json.dumps({"domains": new_dom, "channels": new_chan},
|
||||
out.write_text(_json.dumps({"domains": full_dom, "channels": full_chan,
|
||||
"new": len(new_dom)},
|
||||
ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print("записано: %s (доменов %d, каналов %d)" % (out, len(new_dom), len(new_chan)))
|
||||
print("записано: %s (доменов %d, каналов %d, из них новых %d)"
|
||||
% (out, len(full_dom), len(full_chan), len(new_dom)))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -531,6 +552,7 @@ def main(argv=None):
|
||||
sp.add_argument("--apply", action="store_true", help="записать в реестр omni")
|
||||
sp.add_argument("--min-score", type=int, default=20, help="минимальный score домена")
|
||||
sp.add_argument("--import-posts", action="store_true", help="залить посты каналов радара в базу")
|
||||
sp.add_argument("--since-days", type=int, default=0, help="только посты за N дней (0 = все)")
|
||||
sp.add_argument("--limit", type=int, default=0)
|
||||
sp.set_defaults(fn=cmd_radar)
|
||||
|
||||
|
||||
@@ -68,7 +68,6 @@ def load():
|
||||
cfg.update(json.loads(cf.read_text(encoding="utf-8")))
|
||||
except Exception:
|
||||
pass
|
||||
CONFIG = cfg
|
||||
# cookies, снятые браузером (tools/cf_cookies.py) — файл вне git
|
||||
ck_file = DATA / "cookies.json"
|
||||
if ck_file.exists():
|
||||
@@ -93,9 +92,38 @@ def load():
|
||||
if v:
|
||||
KEYS[name] = v
|
||||
DATA.mkdir(parents=True, exist_ok=True)
|
||||
CONFIG = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def tg_cfg():
|
||||
"""Креды Telegram-сессии: data/tg.json (вне git) или env. Возвращает dict.
|
||||
|
||||
Ключи: session (путь к .session без расширения), api_id, api_hash, limit.
|
||||
env всегда приоритетнее файла.
|
||||
"""
|
||||
import os
|
||||
out = {"session": "", "api_id": "", "api_hash": "", "limit": "100"}
|
||||
f = DATA / "tg.json"
|
||||
if f.exists():
|
||||
try:
|
||||
d = json.loads(f.read_text(encoding="utf-8"))
|
||||
for k in out:
|
||||
if d.get(k):
|
||||
out[k] = str(d[k])
|
||||
except Exception:
|
||||
pass
|
||||
for env, key in (("OMNI_TG_SESSION", "session"), ("OMNI_TG_API_ID", "api_id"),
|
||||
("OMNI_TG_API_HASH", "api_hash"), ("OMNI_TG_LIMIT", "limit")):
|
||||
if os.environ.get(env):
|
||||
out[key] = os.environ[env]
|
||||
for env, key in (("OMNI_TG_SESSION", "session"), ("OMNI_TG_API_ID", "api_id"),
|
||||
("OMNI_TG_API_HASH", "api_hash"), ("OMNI_TG_LIMIT", "limit")):
|
||||
if out[key]:
|
||||
os.environ.setdefault(env, out[key])
|
||||
return out
|
||||
|
||||
|
||||
def proxies():
|
||||
mode = CONFIG.get("proxy_mode", "off")
|
||||
if mode == "env":
|
||||
|
||||
@@ -219,8 +219,19 @@ class Handler(BaseHTTPRequestHandler):
|
||||
cats = _cat_health()
|
||||
for c in cats:
|
||||
c["stored"] = sum(per_source.get(x.name, 0) for x in all_s if x.category == c["name"])
|
||||
kv = {"services": 0, "keys": 0, "ready": 0}
|
||||
tgst = {"enabled": False}
|
||||
try:
|
||||
from . import vault as _vault, tgsess as _tg
|
||||
st_kv = _vault.stats()
|
||||
kv = {"services": len(st_kv), "keys": sum(x["keys"] for x in st_kv),
|
||||
"ready": sum(x["ready"] for x in st_kv)}
|
||||
tgst = {"enabled": bool(_tg.enabled())}
|
||||
except Exception:
|
||||
pass
|
||||
return self._json({
|
||||
"items": s["items"], "priced": s["priced"], "runs": s["runs"], "runs_ok": s["runs_ok"],
|
||||
"keys": kv, "tg": tgst,
|
||||
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"][:12]],
|
||||
"by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
|
||||
"sources": len(all_s),
|
||||
|
||||
@@ -23,6 +23,11 @@ SERVICE_RE = re.compile(
|
||||
|
||||
|
||||
def enabled():
|
||||
try:
|
||||
from . import config
|
||||
config.tg_cfg()
|
||||
except Exception:
|
||||
pass
|
||||
return bool(os.environ.get("OMNI_TG_SESSION") and os.environ.get("OMNI_TG_API_ID")
|
||||
and os.environ.get("OMNI_TG_API_HASH"))
|
||||
|
||||
@@ -75,6 +80,11 @@ def fetch(handles, limit=None):
|
||||
"""Список словарей {handle, title, id, posts[]} по каналам. Пусто, если нет env."""
|
||||
if not enabled():
|
||||
return []
|
||||
try:
|
||||
from . import config
|
||||
config.tg_cfg()
|
||||
except Exception:
|
||||
pass
|
||||
session = os.environ["OMNI_TG_SESSION"]
|
||||
api_id = int(os.environ["OMNI_TG_API_ID"])
|
||||
api_hash = os.environ["OMNI_TG_API_HASH"]
|
||||
|
||||
Reference in New Issue
Block a user