prod: чистая лента (дедуп перепечаток, анти-шум в score, категорийный вес), счётчик живых/мёртвых источников, таймауты net

omni/store.py

@@ -232,11 +232,25 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, ty
              "price": "ORDER BY CAST(price AS REAL) DESC, score DESC",
              "old": "ORDER BY published ASC, id ASC",
              "quality": "ORDER BY score DESC, id DESC"}.get(sort, "ORDER BY score DESC, id DESC")
+    pull, skip = limit, offset
+    if sort == "quality":
+        # Перепечатки одного и того же поста из разных источников схлопываем:
+        # берём с запасом и оставляем по одной записи на нормализованный заголовок.
+        pull, skip = min(600, limit * 4), 0
     with db() as c:
         total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
         rows = c.execute("SELECT id,provider,kind,source,title,text,author,published,score,price,currency,url,types "
                          "FROM items " + where + order + " LIMIT ? OFFSET ?",
-                         args + [limit, offset]).fetchall()
+                         args + [pull, skip]).fetchall()
+    if sort == "quality":
+        seen, uniq = set(), []
+        for r in rows:
+            t = "".join(ch for ch in (r[4] or "").lower() if ch.isalnum())[:60] or str(r[0])
+            if t in seen:
+                continue
+            seen.add(t)
+            uniq.append(r)
+        rows = uniq[offset:offset + limit]
     keys = ("id", "provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
     return [dict(zip(keys, r)) for r in rows], total