65 lines
2.5 KiB
Python
65 lines
2.5 KiB
Python
"""Прогон свежей волны источников: фактическая отдача и сколько легло в базу.
|
|
|
|
python tools/run_fresh.py [--limit 10] [--workers 8]
|
|
"""
|
|
import argparse
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
|
|
from omni import feeds, rank, sources, source_state, store
|
|
from omni.sources_fresh import fresh as _fresh
|
|
|
|
|
|
class _Fake:
|
|
def __init__(self, *a, **k):
|
|
self.name = a[0]
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=10)
|
|
ap.add_argument("--workers", type=int, default=8)
|
|
ap.add_argument("--wave", choices=("fresh", "quality"), default="fresh")
|
|
ap.add_argument("--out", default="")
|
|
a = ap.parse_args()
|
|
|
|
if a.wave == "quality":
|
|
from omni.sources_quality import quality_sources
|
|
names = {f.name for f in quality_sources(_Fake)}
|
|
else:
|
|
names = {f.name for f in _fresh(_Fake)}
|
|
output = a.out or "data/%s_run.json" % a.wave
|
|
picked = [s for s in sources.all_sources() if s.name in names]
|
|
print("источников свежей волны:", len(picked), "| лимит", a.limit)
|
|
t0 = time.time()
|
|
items, report = feeds.fetch_many(picked, limit=a.limit, workers=a.workers)
|
|
items = rank.rank(items)
|
|
new, dup = store.add(items)
|
|
rep = dict(report)
|
|
ok = {k: v for k, v in rep.items() if v}
|
|
bad = sorted(k for k, v in rep.items() if not v)
|
|
by_src = {}
|
|
for it in items:
|
|
by_src[it.source] = by_src.get(it.source, 0) + 1
|
|
# состояние реестра: что реально отдало записи
|
|
for name, n in rep.items():
|
|
source_state.record_check(name, 200 if n else 0, n, "ok" if n else "пусто при проверке")
|
|
out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked),
|
|
"alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup,
|
|
"by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad}
|
|
with open(output, "w", encoding="utf-8") as f:
|
|
json.dump(out, f, ensure_ascii=False, indent=1)
|
|
print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad))
|
|
print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup)
|
|
print("топ:", list(out["by_source"].items())[:12])
|
|
if bad:
|
|
print("пустые:", ", ".join(bad))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|