reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

omni: реестр 113 источников (papers/company/cn/kr/community/security/forum/models/benchmark) + команды sources и feed

Browse Source
This commit is contained in:
dev
2026-10-25 18:54:33 +03:00
parent 02816cacfb
commit c4d188520f
4 changed files with 683 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
omni/cli.py
View File

@@ -5,6 +5,7 @@ import json
import sys
from . import config, rank, store
from . import feeds, sources
from . import filter as flt
from .providers import FUNCS, MODULES, get, status
@@ -146,6 +147,34 @@ def cmd_stats(args):
return 0
def cmd_sources(args):
st = sources.stats()
print("источников:", len(sources.SOURCES), "| по категориям:",
", ".join(f"{k}={v}" for k, v in st.items()))
if args.list:
for s in sources.pick(args.category, args.only, args.tags):
print(f" {s.category:<12} {s.adapter:<18} {s.name:<26} {s.url[:70]}")
return 0
def cmd_feed(args):
picked = sources.pick(args.category, args.only, args.tags)
if not picked:
print("нет источников под фильтр")
return 1
print(f"обход {len(picked)} источников, лимит {args.limit} на источник")
items, report = feeds.fetch_many(picked, limit=args.limit, workers=args.workers)
items = rank.rank(items)
new, dup = store.add(items)
bad = [(n, c) for n, c in report if c == 0]
for name, cnt in sorted(report, key=lambda x: -x[1])[:20]:
print(f" {name:<28} {cnt}")
print(f"пусто: {len(bad)}/{len(report)} | собрано {len(items)}, новых {new}, дублей {dup}")
if bad and args.verbose:
print("без отдачи:", ", ".join(n for n, _ in bad))
return 0
def cmd_export(args):
n, path = store.export(args.out, args.fmt, args.limit)
print(f"выгружено {n} записей -> {path}")
@@ -201,6 +230,22 @@ def main(argv=None):
sp = sub.add_parser("stats", help="сводка базы")
sp.set_defaults(fn=cmd_stats)
sp = sub.add_parser("sources", help="реестр источников")
sp.add_argument("--category", default=None)
sp.add_argument("--only", default=None)
sp.add_argument("--tags", default=None)
sp.add_argument("--list", action="store_true")
sp.set_defaults(fn=cmd_sources)
sp = sub.add_parser("feed", help="обход реестра источников без запроса")
sp.add_argument("--category", default=None)
sp.add_argument("--only", default=None)
sp.add_argument("--tags", default=None)
sp.add_argument("--limit", type=int, default=15)
sp.add_argument("--workers", type=int, default=8)
sp.add_argument("--verbose", action="store_true")
sp.set_defaults(fn=cmd_feed)
sp = sub.add_parser("export", help="выгрузка jsonl/csv")
sp.add_argument("--out", default="data/top.jsonl")
sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"])