prod: чистая лента (дедуп перепечаток, анти-шум в score, категорийный вес), счётчик живых/мёртвых источников, таймауты net

tools/rescore.py

@@ -8,25 +8,28 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
-from omni import config, rank, store  # noqa: E402
+from omni import config, rank, sources, store  # noqa: E402
+
+CAT = {s.name: (s.category or "") for s in sources.SOURCES}
 
 
 class _It:
-    __slots__ = ("provider", "kind", "title", "text", "published", "price", "score")
+    __slots__ = ("provider", "kind", "title", "text", "published", "price", "score", "category")
 
-    def __init__(self, provider, kind, title, text, published, price):
+    def __init__(self, provider, kind, title, text, published, price, category=""):
         self.provider, self.kind = provider, kind
         self.title, self.text = title or "", text or ""
         self.published, self.price, self.score = published or "", price, 0.0
+        self.category = category or ""
 
 
 def main():
     apply = "--apply" in sys.argv
     c = store.conn()
-    rows = c.execute("SELECT id, provider, kind, title, text, published, price, score FROM items").fetchall()
+    rows = c.execute("SELECT id, provider, kind, title, text, published, price, score, source FROM items").fetchall()
     upd, top = [], []
-    for iid, prov, kind, title, text, pub, price, old in rows:
-        it = rank.score(_It(prov, kind, title, text, pub, price))
+    for iid, prov, kind, title, text, pub, price, old, src in rows:
+        it = rank.score(_It(prov, kind, title, text, pub, price, CAT.get(src, "")))
         upd.append((it.score, iid))
         top.append((it.score, old, prov, title or ""))
     top.sort(reverse=True)