omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности
This commit is contained in:
dev
215
omni/cli.py
Normal file
215
omni/cli.py
Normal file
@@ -0,0 +1,215 @@
|
||||
"""CLI: providers / search / fanout / cards / top / find / stats / export."""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import sys
|
||||
|
||||
from . import config, rank, store
|
||||
from . import filter as flt
|
||||
from .providers import FUNCS, MODULES, get, status
|
||||
|
||||
ALL_FREE = ["reddit", "x", "forums", "ddg", "mojeek", "bing"]
|
||||
ALL_KEYED = ["tavily", "exa", "you", "firecrawl"]
|
||||
|
||||
|
||||
def _default_providers():
|
||||
out = list(ALL_FREE)
|
||||
for name in ALL_KEYED:
|
||||
ok, _ = MODULES[name].available(name)
|
||||
if ok:
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def _run_one(name, query, limit, extra):
|
||||
fn = get(name)
|
||||
if not fn:
|
||||
return name, [], "нет провайдера"
|
||||
try:
|
||||
if name == "cards":
|
||||
items = fn(extra.get("urls") or [])
|
||||
elif name == "x":
|
||||
items = fn(query, limit=limit, handles=extra.get("handles"))
|
||||
elif name == "reddit":
|
||||
items = fn(query, limit=limit, subs=extra.get("subs"))
|
||||
elif name == "forums":
|
||||
items = fn(query, limit=limit, hosts=extra.get("hosts"), channels=extra.get("channels"))
|
||||
else:
|
||||
items = fn(query, limit=limit)
|
||||
raw = items or []
|
||||
if name == "cards":
|
||||
return name, raw, "ok" if raw else "пусто"
|
||||
kept = flt.apply(query, raw)
|
||||
for it in kept:
|
||||
it.extra["q"] = query
|
||||
note = "ok" if kept else "пусто"
|
||||
if len(kept) < len(raw):
|
||||
note = f"ok {len(kept)}/{len(raw)}"
|
||||
return name, kept, note
|
||||
except Exception as e:
|
||||
return name, [], f"{type(e).__name__}: {e}"[:120]
|
||||
|
||||
|
||||
def cmd_providers(args):
|
||||
for name, ok, note in status():
|
||||
print(f"{'OK ' if ok else 'НЕТ'} {name:<10} {note}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_search(args):
|
||||
provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()]
|
||||
extra = {"subs": args.subs.split(",") if args.subs else None,
|
||||
"channels": args.channels.split(",") if args.channels else None,
|
||||
"handles": args.handles.split(",") if args.handles else None}
|
||||
results, report = [], []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex:
|
||||
futs = [ex.submit(_run_one, p, args.query, args.limit, extra) for p in provs]
|
||||
for f in concurrent.futures.as_completed(futs):
|
||||
name, items, st = f.result()
|
||||
report.append((name, len(items), st))
|
||||
results += items
|
||||
store.log_run(name, "", args.query, len(items), 0, st)
|
||||
results = rank.rank(results)
|
||||
new, dup = store.add(results)
|
||||
for name, got, st in sorted(report):
|
||||
print(f" {name:<10} {got:>3} записей {st}")
|
||||
print(f"\nсобрано {len(results)}, новых в базе {new}, дублей {dup}\n")
|
||||
for it in results[:args.show]:
|
||||
price = f" {it.price}{it.currency or ''}" if it.price else ""
|
||||
print(f"[{it.score:5.2f}] {it.provider:<9} {it.kind:<8}{price} {it.title[:90]}")
|
||||
print(f" {it.url[:110]}")
|
||||
if args.json:
|
||||
with open(args.json, "w", encoding="utf-8") as f:
|
||||
for it in results:
|
||||
f.write(json.dumps(it.as_dict(), ensure_ascii=False) + "\n")
|
||||
print(f"\njsonl: {args.json}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_fanout(args):
|
||||
themes = [t.strip() for t in open(args.themes, encoding="utf-8") if t.strip() and not t.startswith("#")]
|
||||
if args.limit_themes:
|
||||
themes = themes[:args.limit_themes]
|
||||
provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()]
|
||||
total_new = total_dup = 0
|
||||
for th in themes:
|
||||
results, got = [], {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex:
|
||||
futs = [ex.submit(_run_one, p, th, args.limit, {}) for p in provs]
|
||||
for f in concurrent.futures.as_completed(futs):
|
||||
name, items, st = f.result()
|
||||
got[name] = len(items)
|
||||
results += items
|
||||
store.log_run(name, "", th, len(items), 0, st)
|
||||
results = rank.rank(results)
|
||||
new, dup = store.add(results)
|
||||
total_new += new
|
||||
total_dup += dup
|
||||
print(f"{th[:48]:<50} собрано {len(results):>3} новых {new:>3} дублей {dup:>3}")
|
||||
if args.verbose:
|
||||
for it in results[:3]:
|
||||
print(f" [{it.score:4.1f}] {it.provider} {it.title[:80]}")
|
||||
print(f"\nитого новых {total_new}, дублей {total_dup}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_cards(args):
|
||||
urls = [u.strip() for u in args.urls.split(",") if u.strip()]
|
||||
items = rank.rank(__import__("omni.providers.cards", fromlist=["extract_cards"]).extract_cards(urls, limit=len(urls)))
|
||||
new, dup = store.add(items)
|
||||
for it in items:
|
||||
print(f"{it.title[:80]}")
|
||||
print(f" цена: {it.price} {it.currency or ''} | {it.extra.get('availability', '')}")
|
||||
print(f" {it.url[:110]}")
|
||||
print(f"\nновых {new}, дублей {dup}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_top(args):
|
||||
for prov, kind, src, title, sc, price, cur, url in store.top(args.limit, args.provider):
|
||||
p = f" {price}{cur or ''}" if price else ""
|
||||
print(f"[{sc:5.2f}] {prov:<9} {kind:<8}{p} {title[:80]}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_find(args):
|
||||
for prov, kind, src, title, sc, url in store.search(args.query, args.limit):
|
||||
print(f"[{sc:5.2f}] {prov:<9} {kind:<8} {title[:80]}\n {url[:110]}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_stats(args):
|
||||
s = store.stats()
|
||||
print(f"записей: {s['items']} | с ценником: {s['priced']} | прогонов: {s['runs']} (ok {s['runs_ok']})")
|
||||
print("по провайдерам:", ", ".join(f"{p}={n}" for p, n in s["by_provider"]))
|
||||
print("по типам:", ", ".join(f"{k}={n}" for k, n in s["by_kind"]))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_export(args):
|
||||
n, path = store.export(args.out, args.fmt, args.limit)
|
||||
print(f"выгружено {n} записей -> {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
config.load()
|
||||
p = argparse.ArgumentParser(prog="omni", description="универсальный парсер тем: соцсети, форумы, поиск, карточки")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sp = sub.add_parser("providers", help="что доступно и с какими ключами")
|
||||
sp.set_defaults(fn=cmd_providers)
|
||||
|
||||
sp = sub.add_parser("search", help="один запрос по всем источникам")
|
||||
sp.add_argument("query")
|
||||
sp.add_argument("--providers", default="")
|
||||
sp.add_argument("--limit", type=int, default=15)
|
||||
sp.add_argument("--show", type=int, default=15)
|
||||
sp.add_argument("--json", default="")
|
||||
sp.add_argument("--subs", default="")
|
||||
sp.add_argument("--channels", default="")
|
||||
sp.add_argument("--handles", default="")
|
||||
sp.set_defaults(fn=cmd_search)
|
||||
|
||||
sp = sub.add_parser("fanout", help="список тем -> полный обход")
|
||||
sp.add_argument("--themes", default="themes.txt")
|
||||
sp.add_argument("--providers", default="")
|
||||
sp.add_argument("--limit", type=int, default=12)
|
||||
sp.add_argument("--limit-themes", type=int, default=0)
|
||||
sp.add_argument("--verbose", action="store_true")
|
||||
sp.set_defaults(fn=cmd_fanout)
|
||||
|
||||
sp = sub.add_parser("cards", help="карточки товаров по URL")
|
||||
sp.add_argument("urls")
|
||||
sp.set_defaults(fn=cmd_cards)
|
||||
|
||||
sp = sub.add_parser("top", help="топ базы по скору")
|
||||
sp.add_argument("--limit", type=int, default=20)
|
||||
sp.add_argument("--provider", default=None)
|
||||
sp.set_defaults(fn=cmd_top)
|
||||
|
||||
sp = sub.add_parser("find", help="поиск по своей базе")
|
||||
sp.add_argument("query")
|
||||
sp.add_argument("--limit", type=int, default=20)
|
||||
sp.set_defaults(fn=cmd_find)
|
||||
|
||||
sp = sub.add_parser("stats", help="сводка базы")
|
||||
sp.set_defaults(fn=cmd_stats)
|
||||
|
||||
sp = sub.add_parser("export", help="выгрузка jsonl/csv")
|
||||
sp.add_argument("--out", default="data/top.jsonl")
|
||||
sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"])
|
||||
sp.add_argument("--limit", type=int, default=1000)
|
||||
sp.set_defaults(fn=cmd_export)
|
||||
|
||||
args = p.parse_args(argv)
|
||||
return args.fn(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user