chore(auto): самоаудит 2026-10-26T16:19:01Z — selfcheck_exit=0

omni/store.py

@@ -86,6 +86,10 @@ def add(items):
             if _near_dup(c, h):
                 dup += 1
                 continue
+            u = (getattr(it, "url", "") or "").strip()
+            if u and c.execute("SELECT 1 FROM items WHERE url = ? LIMIT 1", (u,)).fetchone():
+                dup += 1
+                continue
             try:
                 cur = c.execute(
                     "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
@@ -243,4 +247,25 @@ def export(path, fmt="jsonl", limit=1000):
         with path.open("w", encoding="utf-8") as f:
             for r in rows:
                 f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n")
-    return len(rows), str(path)
+    return len(rows), str(path)
+
+def dedupe(apply: bool = False):
+    """Схлопывает записи с одинаковым адресом (или source+title при пустом адресе).
+
+    Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено).
+    """
+    key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END"
+    with conn() as c:
+        groups = c.execute(
+            "SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0]
+        extra = c.execute(
+            "SELECT coalesce(sum(n - 1), 0) FROM (SELECT count(*) n FROM items GROUP BY " + key + " HAVING n > 1)").fetchone()[0]
+        killed = 0
+        if apply and extra:
+            ids = [r[0] for r in c.execute(
+                "SELECT id FROM items WHERE id NOT IN (SELECT max(id) FROM items GROUP BY " + key + ")")]
+            for chunk in [ids[i:i + 400] for i in range(0, len(ids), 400)]:
+                q = "DELETE FROM items WHERE id IN (" + ",".join("?" * len(chunk)) + ")"
+                c.execute(q, chunk)
+                killed += len(chunk)
+    return groups, extra, killed