626 lines
27 KiB
Python
626 lines
27 KiB
Python
"""CLI: providers / search / fanout / cards / top / find / stats / export."""
|
||
import argparse
|
||
import datetime as _dt
|
||
import pathlib
|
||
import concurrent.futures
|
||
import json
|
||
import sys
|
||
|
||
from . import config, rank, store
|
||
from . import feeds, sources
|
||
from . import filter as flt
|
||
from .model import Item
|
||
from .providers import FUNCS, MODULES, get, status
|
||
|
||
ALL_FREE = ["reddit", "x", "forums", "ddg", "mojeek", "bing"]
|
||
ALL_KEYED = ["tavily", "exa", "you", "firecrawl"]
|
||
|
||
|
||
def _default_providers():
|
||
out = list(ALL_FREE)
|
||
for name in ALL_KEYED:
|
||
ok, _ = MODULES[name].available(name)
|
||
if ok:
|
||
out.append(name)
|
||
return out
|
||
|
||
|
||
def _run_one(name, query, limit, extra):
|
||
fn = get(name)
|
||
if not fn:
|
||
return name, [], "нет провайдера"
|
||
try:
|
||
if name == "cards":
|
||
items = fn(extra.get("urls") or [])
|
||
elif name == "x":
|
||
items = fn(query, limit=limit, handles=extra.get("handles"))
|
||
elif name == "reddit":
|
||
items = fn(query, limit=limit, subs=extra.get("subs"))
|
||
elif name == "forums":
|
||
items = fn(query, limit=limit, hosts=extra.get("hosts"), channels=extra.get("channels"))
|
||
else:
|
||
items = fn(query, limit=limit)
|
||
raw = items or []
|
||
if name == "cards":
|
||
return name, raw, "ok" if raw else "пусто"
|
||
kept = flt.apply(query, raw)
|
||
for it in kept:
|
||
it.extra["q"] = query
|
||
note = "ok" if kept else "пусто"
|
||
if len(kept) < len(raw):
|
||
note = f"ok {len(kept)}/{len(raw)}"
|
||
return name, kept, note
|
||
except Exception as e:
|
||
return name, [], f"{type(e).__name__}: {e}"[:120]
|
||
|
||
|
||
def cmd_providers(args):
|
||
for name, ok, note in status():
|
||
print(f"{'OK ' if ok else 'НЕТ'} {name:<10} {note}")
|
||
return 0
|
||
|
||
|
||
def cmd_search(args):
|
||
provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()]
|
||
extra = {"subs": args.subs.split(",") if args.subs else None,
|
||
"channels": args.channels.split(",") if args.channels else None,
|
||
"handles": args.handles.split(",") if args.handles else None}
|
||
results, report = [], []
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex:
|
||
futs = [ex.submit(_run_one, p, args.query, args.limit, extra) for p in provs]
|
||
for f in concurrent.futures.as_completed(futs):
|
||
name, items, st = f.result()
|
||
report.append((name, len(items), st))
|
||
results += items
|
||
store.log_run(name, "", args.query, len(items), 0, st)
|
||
results = rank.rank(results)
|
||
new, dup = store.add(results)
|
||
for name, got, st in sorted(report):
|
||
print(f" {name:<10} {got:>3} записей {st}")
|
||
print(f"\nсобрано {len(results)}, новых в базе {new}, дублей {dup}\n")
|
||
for it in results[:args.show]:
|
||
price = f" {it.price}{it.currency or ''}" if it.price else ""
|
||
print(f"[{it.score:5.2f}] {it.provider:<9} {it.kind:<8}{price} {it.title[:90]}")
|
||
print(f" {it.url[:110]}")
|
||
if args.json:
|
||
with open(args.json, "w", encoding="utf-8") as f:
|
||
for it in results:
|
||
f.write(json.dumps(it.as_dict(), ensure_ascii=False) + "\n")
|
||
print(f"\njsonl: {args.json}")
|
||
return 0
|
||
|
||
|
||
def cmd_fanout(args):
|
||
themes = [t.strip() for t in open(args.themes, encoding="utf-8") if t.strip() and not t.startswith("#")]
|
||
if args.limit_themes:
|
||
themes = themes[:args.limit_themes]
|
||
provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()]
|
||
total_new = total_dup = 0
|
||
for th in themes:
|
||
results, got = [], {}
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex:
|
||
futs = [ex.submit(_run_one, p, th, args.limit, {}) for p in provs]
|
||
for f in concurrent.futures.as_completed(futs):
|
||
name, items, st = f.result()
|
||
got[name] = len(items)
|
||
results += items
|
||
store.log_run(name, "", th, len(items), 0, st)
|
||
results = rank.rank(results)
|
||
new, dup = store.add(results)
|
||
total_new += new
|
||
total_dup += dup
|
||
print(f"{th[:48]:<50} собрано {len(results):>3} новых {new:>3} дублей {dup:>3}")
|
||
if args.verbose:
|
||
for it in results[:3]:
|
||
print(f" [{it.score:4.1f}] {it.provider} {it.title[:80]}")
|
||
print(f"\nитого новых {total_new}, дублей {total_dup}")
|
||
return 0
|
||
|
||
|
||
def cmd_cards(args):
|
||
urls = [u.strip() for u in args.urls.split(",") if u.strip()]
|
||
items = rank.rank(__import__("omni.providers.cards", fromlist=["extract_cards"]).extract_cards(urls, limit=len(urls)))
|
||
new, dup = store.add(items)
|
||
for it in items:
|
||
print(f"{it.title[:80]}")
|
||
print(f" цена: {it.price} {it.currency or ''} | {it.extra.get('availability', '')}")
|
||
print(f" {it.url[:110]}")
|
||
print(f"\nновых {new}, дублей {dup}")
|
||
return 0
|
||
|
||
|
||
def cmd_top(args):
|
||
for prov, kind, src, title, sc, price, cur, url in store.top(args.limit, args.provider):
|
||
p = f" {price}{cur or ''}" if price else ""
|
||
print(f"[{sc:5.2f}] {prov:<9} {kind:<8}{p} {title[:80]}")
|
||
return 0
|
||
|
||
|
||
def cmd_find(args):
|
||
for prov, kind, src, title, sc, url in store.search(args.query, args.limit):
|
||
print(f"[{sc:5.2f}] {prov:<9} {kind:<8} {title[:80]}\n {url[:110]}")
|
||
return 0
|
||
|
||
|
||
def cmd_stats(args):
|
||
s = store.stats()
|
||
print(f"записей: {s['items']} | с ценником: {s['priced']} | прогонов: {s['runs']} (ok {s['runs_ok']})")
|
||
print("по провайдерам:", ", ".join(f"{p}={n}" for p, n in s["by_provider"]))
|
||
print("по типам:", ", ".join(f"{k}={n}" for k, n in s["by_kind"]))
|
||
return 0
|
||
|
||
|
||
def cmd_sources(args):
|
||
from . import source_state
|
||
st = sources.stats()
|
||
fl = source_state.flags()
|
||
all_s = sources.all_sources()
|
||
print("источников:", len(all_s), "| включено:", sum(1 for s in all_s if fl.get(s.name, {}).get("enabled", True)),
|
||
"| по категориям:", ", ".join(f"{k}={v}" for k, v in st.items()))
|
||
if args.list:
|
||
for s in sources.pick(args.category, args.only, args.tags, respect_enabled=not args.all):
|
||
f = fl.get(s.name, {})
|
||
mark = "on " if f.get("enabled", True) else "off"
|
||
stt = f"{f.get('items')} зап." if (f.get("items") or 0) > 0 else (f.get("note") or "—")
|
||
print(f" {mark} {s.category:<12} {s.adapter:<18} {s.name:<26} {stt:<12} {s.url[:60]}")
|
||
return 0
|
||
|
||
|
||
def cmd_audit(args):
|
||
"""Сводка здоровья реестра: по категориям и список мёртвых с причиной."""
|
||
from . import source_state
|
||
fl = source_state.flags()
|
||
all_s = sources.all_sources()
|
||
agg = {}
|
||
dead = []
|
||
for s in all_s:
|
||
f = fl.get(s.name, {})
|
||
a = agg.setdefault(s.category, {"t": 0, "alive": 0, "dead": 0, "untested": 0})
|
||
a["t"] += 1
|
||
if not f.get("checked"):
|
||
a["untested"] += 1
|
||
elif (f.get("items") or 0) > 0:
|
||
a["alive"] += 1
|
||
else:
|
||
a["dead"] += 1
|
||
dead.append((s.category, s.name, f.get("note") or "empty"))
|
||
print("источников: %d | живых: %d | мёртвых: %d | не проверено: %d" % (
|
||
len(all_s), sum(a["alive"] for a in agg.values()),
|
||
sum(a["dead"] for a in agg.values()), sum(a["untested"] for a in agg.values())))
|
||
for cat, a in sorted(agg.items(), key=lambda x: -x[1]["t"]):
|
||
pct = round(a["alive"] / a["t"] * 100) if a["t"] else 0
|
||
print(" %-12s всего %3d живых %3d мёртвых %3d не проверено %3d (%d%%)"
|
||
% (cat, a["t"], a["alive"], a["dead"], a["untested"], pct))
|
||
if args.dead:
|
||
print("")
|
||
print("мёртвые:")
|
||
for cat, name, note in sorted(dead):
|
||
print(" %-12s %-28s %s" % (cat, name, note[:50]))
|
||
return 0
|
||
|
||
|
||
def cmd_feed(args):
|
||
picked = sources.pick(args.category, args.only, args.tags)
|
||
if not picked:
|
||
print("нет источников под фильтр")
|
||
return 1
|
||
print(f"обход {len(picked)} источников, лимит {args.limit} на источник")
|
||
items, report = feeds.fetch_many(picked, limit=args.limit, workers=args.workers)
|
||
items = rank.rank(items)
|
||
new, dup = store.add(items)
|
||
bad = [(n, c) for n, c in report if c == 0]
|
||
for name, cnt in sorted(report, key=lambda x: -x[1])[:20]:
|
||
print(f" {name:<28} {cnt}")
|
||
print(f"пусто: {len(bad)}/{len(report)} | собрано {len(items)}, новых {new}, дублей {dup}")
|
||
if bad and args.verbose:
|
||
print("без отдачи:", ", ".join(n for n, _ in bad))
|
||
return 0
|
||
|
||
|
||
def cmd_digest(args):
|
||
"""Сводка по базе: топ записей по категориям за прогон, в markdown."""
|
||
rows = store.top(limit=args.limit * 8)
|
||
by_cat = {}
|
||
for prov, kind, src, title, sc, price, cur, url in rows:
|
||
by_cat.setdefault(src or prov, []).append((sc, title, url))
|
||
out = [f"# дайджест omni · {__import__('datetime').date.today()}", ""]
|
||
for cat, entries in sorted(by_cat.items(), key=lambda x: -len(x[1])):
|
||
out.append(f"## {cat} · {len(entries)}")
|
||
for sc, title, url in entries[:args.limit]:
|
||
out.append(f"- [{title[:110]}]({url}) · {sc:.2f}")
|
||
out.append("")
|
||
text = chr(10).join(out)
|
||
path = args.out
|
||
pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||
pathlib.Path(path).write_text(text, encoding="utf-8")
|
||
print(f"дайджест: {len(by_cat)} категорий, {len(rows)} записей -> {path}")
|
||
return 0
|
||
|
||
|
||
def cmd_export(args):
|
||
n, path = store.export(args.out, args.fmt, args.limit)
|
||
print(f"выгружено {n} записей -> {path}")
|
||
return 0
|
||
|
||
|
||
def cmd_add_src(a):
|
||
"""Свой источник: пробуем url живым запросом, сохраняем только если отдал записи."""
|
||
import re as _re
|
||
import urllib.parse as _up
|
||
from . import discover as _disc, source_state as _st
|
||
url = (a.url or "").strip()
|
||
if not url.startswith("http"):
|
||
print("нужен url фида"); return 1
|
||
ok, n, ad = _disc.probe(url, a.category, limit=6)
|
||
if not ok:
|
||
print("фид пустой или недоступен: %s" % ad); return 1
|
||
name = "manual_" + _re.sub(r"[^a-z0-9]+", "_", _up.urlsplit(url).netloc.lower())[:32]
|
||
kind = "rss" if ad == "rss" else "html"
|
||
tags = [t.strip() for t in (a.tags or "").split(",") if t.strip()] or ["manual"]
|
||
_st.add_custom(name, a.category, url, kind=kind, adapter=ad, tags=tags, limit=15)
|
||
print("добавлен %s · %s · %d записей" % (name, ad, n))
|
||
return 0
|
||
|
||
|
||
def cmd_dedupe(a):
|
||
from . import store
|
||
groups, extra, killed = store.dedupe(apply=a.apply)
|
||
print(f"групп дублей: {groups} | лишних строк: {extra} | удалено: {killed}"
|
||
if a.apply else f"групп дублей: {groups} | лишних строк: {extra} (dry-run, добавь --apply)")
|
||
return 0
|
||
|
||
|
||
def cmd_pump(a):
|
||
"""История Telegram-каналов через свою сессию (нужны OMNI_TG_SESSION/API_ID/API_HASH)."""
|
||
from . import tgsess
|
||
if not tgsess.enabled():
|
||
print("нет сессии: задай OMNI_TG_SESSION, OMNI_TG_API_ID, OMNI_TG_API_HASH")
|
||
return 2
|
||
handles = [h.strip() for h in a.channels.split(",") if h.strip()]
|
||
recs = tgsess.fetch(handles, a.limit)
|
||
if not recs:
|
||
print("пусто")
|
||
return 1
|
||
items = []
|
||
for r in recs:
|
||
for post in r.get("posts") or []:
|
||
if len(post["text"]) < 12:
|
||
continue
|
||
items.append(Item("tg_" + r["handle"], "post", url=post["url"], title=post["title"],
|
||
text=post["text"], author="@" + r["handle"],
|
||
published=post["date"], source="tg_" + r["handle"],
|
||
extra={"views": post.get("views"), "via": "session"}))
|
||
print("каналов %d, постов %d" % (len(recs), len(items)))
|
||
if not a.apply:
|
||
print("dry-run: добавь --apply")
|
||
return 0
|
||
new, dup = store.add(items)
|
||
print("новых %d, дублей %d" % (new, dup))
|
||
return 0
|
||
|
||
|
||
def cmd_keys(a):
|
||
"""Сводка vault: сколько ключей на сервис, сколько готово, что остывает."""
|
||
from . import vault
|
||
st = vault.stats()
|
||
if a.service:
|
||
st = [x for x in st if x["service"] == a.service]
|
||
if not st:
|
||
print("vault пуст — собери: python tools/build_vault.py")
|
||
return 1
|
||
w = max(len(x["service"]) for x in st)
|
||
print("%-*s %8s %8s %9s %s" % (w, "сервис", "ключей", "готово", "остывают", "env"))
|
||
for x in st:
|
||
print("%-*s %8d %8d %9d %s" % (w, x["service"], x["keys"], x["ready"],
|
||
x["cooling"], "*" if x["env"] else ""))
|
||
if a.verbose:
|
||
print()
|
||
for x in st:
|
||
for r in (vault._load()[0].get(x["service"]) or []):
|
||
print(" %-*s %s %-7s %s" % (w, x["service"], vault.mask(r["key"]),
|
||
r.get("status") or "?", (r.get("note") or "")[:30]))
|
||
return 0
|
||
|
||
|
||
def cmd_radar(a):
|
||
"""Мост из abuse-radar: домены и Telegram-каналы радара -> реестр omni."""
|
||
import json as _json
|
||
import sqlite3 as _sq
|
||
from pathlib import Path as _P
|
||
from . import sources as _src
|
||
|
||
cand = [a.db] if a.db else [
|
||
str(_P("C:/Users/User/tmp/abuse_radar/radar_server.db")),
|
||
str(_P("C:/Users/User/tmp/abuse_radar/radar.db")),
|
||
"data/radar.db",
|
||
]
|
||
db = next((c for c in cand if _P(c).exists()), "")
|
||
if not db:
|
||
print("radar.db не найден, укажи --db")
|
||
return 2
|
||
|
||
c = _sq.connect(db)
|
||
doms = c.execute("SELECT domain, score, info FROM domains ORDER BY score DESC").fetchall()
|
||
chans = c.execute("SELECT name, ok, fails FROM channels WHERE ok=1").fetchall()
|
||
if a.import_posts:
|
||
if a.since_days:
|
||
cutoff = _dt.datetime.utcnow() - _dt.timedelta(days=a.since_days)
|
||
posts = c.execute("SELECT chan, mid, date, text FROM messages WHERE date >= ?",
|
||
(cutoff.strftime("%Y-%m-%d"),)).fetchall()
|
||
else:
|
||
posts = c.execute("SELECT chan, mid, date, text FROM messages").fetchall()
|
||
else:
|
||
posts = []
|
||
c.close()
|
||
|
||
if a.import_posts:
|
||
def head_line(txt):
|
||
for ln in (txt or "").splitlines():
|
||
ln = ln.strip()
|
||
if len(ln) >= 20 and sum(ch.isalpha() for ch in ln) >= 10:
|
||
return ln[:150]
|
||
return ((txt or "").strip().splitlines() or [""])[0][:150]
|
||
|
||
items, seen_u = [], set()
|
||
known = set()
|
||
try:
|
||
with store.db() as kc:
|
||
for (u,) in kc.execute("SELECT url FROM items WHERE url LIKE 'https://t.me/%'"):
|
||
known.add(u)
|
||
except Exception:
|
||
known = set()
|
||
for chan, mid, date, text in posts:
|
||
text = (text or "").strip()
|
||
if len(text) < 30:
|
||
continue
|
||
url_u = "https://t.me/%s/%s" % (chan, mid)
|
||
if url_u in seen_u or url_u in known:
|
||
continue
|
||
seen_u.add(url_u)
|
||
items.append(Item("rad_" + chan, "post", url=url_u, title=head_line(text), text=text,
|
||
author="@" + chan, published=(date or "")[:19],
|
||
source="rad_" + chan, extra={"via": "radar"}))
|
||
if a.limit:
|
||
items = items[:a.limit]
|
||
n_new = n_dup = 0
|
||
for i in range(0, len(items), 500):
|
||
n1, n2 = store.add(items[i:i + 500])
|
||
n_new += n1
|
||
n_dup += n2
|
||
print("постов радара: %d | новых %d, дублей %d" % (len(items), n_new, n_dup))
|
||
|
||
have = {s.url.rstrip('/').lower() for s in _src.all_sources()}
|
||
have_names = {s.name for s in _src.all_sources()}
|
||
|
||
full_dom, full_chan = [], []
|
||
new_dom, new_chan = [], []
|
||
for dom, score, info in doms:
|
||
if a.min_score and (score or 0) < a.min_score:
|
||
continue
|
||
title = ""
|
||
try:
|
||
title = (_json.loads(info or "{}") or {}).get("title") or ""
|
||
except Exception:
|
||
pass
|
||
full_dom.append((dom, score, title))
|
||
url = "https://" + dom
|
||
if url.lower() in have or ("rad_" + dom.replace('.', '_').replace('-', '_')) in have_names:
|
||
continue
|
||
new_dom.append((dom, score, title))
|
||
for name, ok, fails in chans:
|
||
full_chan.append(name)
|
||
if ("tg_" + name) in have_names:
|
||
continue
|
||
new_chan.append(name)
|
||
|
||
if a.limit:
|
||
new_dom, new_chan = new_dom[:a.limit], new_chan[:a.limit]
|
||
print("радар: %s" % db)
|
||
print("доменов всего %d, новых для omni %d | каналов живых %d, новых %d"
|
||
% (len(doms), len(new_dom), len(chans), len(new_chan)))
|
||
for dom, score, title in new_dom[:5]:
|
||
print(" %-28s score %-4s %s" % (dom, score, (title or "")[:44]))
|
||
if new_chan[:5]:
|
||
print(" каналы:", ", ".join(new_chan[:8]))
|
||
if not a.apply:
|
||
print("dry-run: добавь --apply, чтобы записать в data/radar_sources.json")
|
||
return 0
|
||
|
||
out = _P("data") / "radar_sources.json"
|
||
out.write_text(_json.dumps({"domains": full_dom, "channels": full_chan,
|
||
"new": len(new_dom)},
|
||
ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("записано: %s (доменов %d, каналов %d, из них новых %d)"
|
||
% (out, len(full_dom), len(full_chan), len(new_dom)))
|
||
return 0
|
||
|
||
|
||
def main(argv=None):
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||
except Exception:
|
||
pass
|
||
config.load()
|
||
p = argparse.ArgumentParser(prog="omni", description="универсальный парсер тем: соцсети, форумы, поиск, карточки")
|
||
sub = p.add_subparsers(dest="cmd", required=True)
|
||
|
||
sp = sub.add_parser("providers", help="что доступно и с какими ключами")
|
||
sp.set_defaults(fn=cmd_providers)
|
||
|
||
sp = sub.add_parser("search", help="один запрос по всем источникам")
|
||
sp.add_argument("query")
|
||
sp.add_argument("--providers", default="")
|
||
sp.add_argument("--limit", type=int, default=15)
|
||
sp.add_argument("--show", type=int, default=15)
|
||
sp.add_argument("--json", default="")
|
||
sp.add_argument("--subs", default="")
|
||
sp.add_argument("--channels", default="")
|
||
sp.add_argument("--handles", default="")
|
||
sp.set_defaults(fn=cmd_search)
|
||
|
||
sp = sub.add_parser("fanout", help="список тем -> полный обход")
|
||
sp.add_argument("--themes", default="themes.txt")
|
||
sp.add_argument("--providers", default="")
|
||
sp.add_argument("--limit", type=int, default=12)
|
||
sp.add_argument("--limit-themes", type=int, default=0)
|
||
sp.add_argument("--verbose", action="store_true")
|
||
sp.set_defaults(fn=cmd_fanout)
|
||
|
||
sp = sub.add_parser("cards", help="карточки товаров по URL")
|
||
sp.add_argument("urls")
|
||
sp.set_defaults(fn=cmd_cards)
|
||
|
||
sp = sub.add_parser("top", help="топ базы по скору")
|
||
sp.add_argument("--limit", type=int, default=20)
|
||
sp.add_argument("--provider", default=None)
|
||
sp.set_defaults(fn=cmd_top)
|
||
|
||
sp = sub.add_parser("find", help="поиск по своей базе")
|
||
sp.add_argument("query")
|
||
sp.add_argument("--limit", type=int, default=20)
|
||
sp.set_defaults(fn=cmd_find)
|
||
|
||
sp = sub.add_parser("stats", help="сводка базы")
|
||
sp.set_defaults(fn=cmd_stats)
|
||
|
||
sp = sub.add_parser("sources", help="реестр источников")
|
||
sp.add_argument("--category", default=None)
|
||
sp.add_argument("--only", default=None)
|
||
sp.add_argument("--tags", default=None)
|
||
sp.add_argument("--list", action="store_true")
|
||
sp.add_argument("--all", action="store_true", help="показать и выключенные")
|
||
sp.set_defaults(fn=cmd_sources)
|
||
|
||
sp = sub.add_parser("audit", help="здоровье реестра источников")
|
||
sp.add_argument("--dead", action="store_true", help="показать мёртвые с причиной")
|
||
sp.set_defaults(fn=cmd_audit)
|
||
|
||
sp = sub.add_parser("feed", help="обход реестра источников без запроса")
|
||
sp.add_argument("--category", default=None)
|
||
sp.add_argument("--only", default=None)
|
||
sp.add_argument("--tags", default=None)
|
||
sp.add_argument("--limit", type=int, default=15)
|
||
sp.add_argument("--workers", type=int, default=8)
|
||
sp.add_argument("--verbose", action="store_true")
|
||
sp.set_defaults(fn=cmd_feed)
|
||
|
||
sp = sub.add_parser("doctor", help="проверка источников, --fix подбирает рабочие фиды")
|
||
sp.add_argument("--only", default=None)
|
||
sp.add_argument("--fix", action="store_true")
|
||
sp.add_argument("--limit", type=int, default=3)
|
||
sp.set_defaults(fn=lambda a: (__import__("omni.doctor", fromlist=["run"]).run(a.only, a.fix, limit=a.limit), 0)[1])
|
||
|
||
sp = sub.add_parser("serve", help="веб-панель + JSON API")
|
||
sp.add_argument("--port", type=int, default=8077)
|
||
sp.set_defaults(fn=lambda a: __import__("omni.serve", fromlist=["serve"]).serve(a.port))
|
||
|
||
sp = sub.add_parser("digest", help="markdown-дайджест из базы")
|
||
sp.add_argument("--limit", type=int, default=8)
|
||
sp.add_argument("--out", default="data/digest.md")
|
||
sp.set_defaults(fn=cmd_digest)
|
||
|
||
sp = sub.add_parser("export", help="выгрузка jsonl/csv")
|
||
sp.add_argument("--out", default="data/top.jsonl")
|
||
sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"])
|
||
sp.add_argument("--limit", type=int, default=1000)
|
||
sp.set_defaults(fn=cmd_export)
|
||
|
||
sp = sub.add_parser("add", help="добавить свой источник по url фида")
|
||
sp.add_argument("url")
|
||
sp.add_argument("--category", default="community")
|
||
sp.add_argument("--tags", default="manual")
|
||
sp.set_defaults(fn=cmd_add_src)
|
||
|
||
sp = sub.add_parser("dedupe", help="схлопнуть дубли записей (по url или source+title)")
|
||
sp.add_argument("--apply", action="store_true", help="реально удалить лишние строки")
|
||
sp.set_defaults(fn=cmd_dedupe)
|
||
|
||
sp = sub.add_parser("pump", help="история Telegram-каналов через свою сессию")
|
||
sp.add_argument("channels", help="каналы через запятую")
|
||
sp.add_argument("--limit", type=int, default=100)
|
||
sp.add_argument("--apply", action="store_true")
|
||
sp.set_defaults(fn=cmd_pump)
|
||
|
||
sp = sub.add_parser("tgsearch", help="найти Telegram-каналы по ключевым словам через свою сессию")
|
||
sp.add_argument("queries", help="запросы через запятую")
|
||
sp.add_argument("--limit", type=int, default=40, help="каналов на запрос")
|
||
sp.add_argument("--min-members", type=int, default=0, help="отсеять мелкие каналы")
|
||
sp.add_argument("--apply", action="store_true", help="дописать найденные каналы в data/radar_sources.json")
|
||
sp.set_defaults(fn=cmd_tgsearch)
|
||
|
||
sp = sub.add_parser("keys", help="ключи парсеров из vault: сервисы, счётчики, остывание")
|
||
sp.add_argument("--service", default="", help="только один сервис")
|
||
sp.add_argument("-v", "--verbose", action="store_true", help="по ключам, масками")
|
||
sp.set_defaults(fn=cmd_keys)
|
||
|
||
sp = sub.add_parser("radar", help="забрать источники из abuse-radar (домены + каналы)")
|
||
sp.add_argument("--db", default="", help="путь к radar.db (по умолчанию ищет рядом)")
|
||
sp.add_argument("--apply", action="store_true", help="записать в реестр omni")
|
||
sp.add_argument("--min-score", type=int, default=20, help="минимальный score домена")
|
||
sp.add_argument("--import-posts", action="store_true", help="залить посты каналов радара в базу")
|
||
sp.add_argument("--since-days", type=int, default=0, help="только посты за N дней (0 = все)")
|
||
sp.add_argument("--limit", type=int, default=0)
|
||
sp.set_defaults(fn=cmd_radar)
|
||
|
||
args = p.parse_args(argv)
|
||
return args.fn(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|
||
|
||
|
||
def cmd_tgsearch(a):
|
||
"""Поиск каналов по ключевым словам -> пополнение пула источников."""
|
||
from . import tgsess
|
||
if not tgsess.enabled():
|
||
print("сессия не настроена: data/tg.json или OMNI_TG_* в env")
|
||
return 1
|
||
queries = [q.strip() for q in str(a.queries).split(",") if q.strip()]
|
||
found = {}
|
||
for q in queries:
|
||
try:
|
||
rows = tgsess.search(q, limit=a.limit)
|
||
except Exception as e:
|
||
print(" %-28s ошибка %s" % (q, type(e).__name__))
|
||
continue
|
||
for r in rows:
|
||
if a.min_members and (r.get("participants") or 0) < a.min_members:
|
||
continue
|
||
prev = found.get(r["username"])
|
||
if prev:
|
||
prev["hits"] = prev.get("hits", 1) + 1
|
||
continue
|
||
r["hits"] = 1
|
||
found[r["username"]] = r
|
||
print(" %-28s каналов %d (всего уникальных %d)" % (q, len(rows), len(found)))
|
||
rows = sorted(found.values(), key=lambda x: (-(x.get("hits") or 0), -(x.get("participants") or 0)))
|
||
for r in rows[:20]:
|
||
print(" @%-28s %7s участн. hits=%d %s" % (r["username"], r.get("participants") or 0,
|
||
r["hits"], (r.get("title") or "")[:40]))
|
||
if not a.apply:
|
||
print("найдено %d каналов; добавь --apply чтобы дописать в пул" % len(rows))
|
||
return 0
|
||
import json
|
||
from pathlib import Path
|
||
p = Path(__file__).resolve().parents[1] / "data" / "radar_sources.json"
|
||
try:
|
||
d = json.loads(p.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
d = {"domains": [], "channels": []}
|
||
chans = list(d.get("channels") or [])
|
||
have = {str(c).lstrip("@").lower() for c in chans}
|
||
added = 0
|
||
for r in rows:
|
||
u = str(r["username"]).lstrip("@")
|
||
if u.lower() in have:
|
||
continue
|
||
chans.append(u)
|
||
have.add(u.lower())
|
||
added += 1
|
||
d["channels"] = chans
|
||
p.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("дописано каналов: %d | всего каналов в пуле: %d" % (added, len(chans)))
|
||
return 0
|