omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/rank.py

@@ -0,0 +1,41 @@
+"""Скор записи: вес провайдера + вовлечённость + свежесть + ценник + объём текста."""
+import re
+import time
+
+WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0,
+          "firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8}
+
+KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "result": 0.1, "post": 0.1}
+
+
+def _fresh(published):
+    """published: unix-строка или ISO. Возвращает 0..1."""
+    if not published:
+        return 0.3
+    try:
+        if published.isdigit():
+            ts = int(published)
+            if ts > 10**12:
+                ts //= 1000
+        else:
+            ts = time.mktime(time.strptime(published[:19], "%Y-%m-%dT%H:%M:%S"))
+        days = max(0.0, (time.time() - ts) / 86400)
+        return max(0.0, 1.0 - days / 30.0)
+    except Exception:
+        return 0.3
+
+
+def score(it):
+    base = it.score if it.score else 0.0
+    w = WEIGHT.get(it.provider, 0.7)
+    fresh = _fresh(it.published)
+    priced = 0.35 if it.price else 0.0
+    body = min(1.0, len(it.text) / 1200.0)
+    eng = min(3.0, base ** 0.5) if base > 1 else base
+    total = (w * (1.0 + eng) + fresh * 1.2 + priced + body + KIND_BONUS.get(it.kind, 0.0))
+    it.score = round(total, 4)
+    return it
+
+
+def rank(items):
+    return sorted((score(i) for i in items), key=lambda x: -x.score)