chore(auto): самоаудит 2026-10-26T16:19:01Z — selfcheck_exit=0

omni/cli.py

@@ -260,6 +260,14 @@ def cmd_add_src(a):
     return 0
 
 
+def cmd_dedupe(a):
+    from . import store
+    groups, extra, killed = store.dedupe(apply=a.apply)
+    print(f"групп дублей: {groups} | лишних строк: {extra} | удалено: {killed}"
+          if a.apply else f"групп дублей: {groups} | лишних строк: {extra} (dry-run, добавь --apply)")
+    return 0
+
+
 def main(argv=None):
     try:
         sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -357,6 +365,10 @@ def main(argv=None):
     sp.add_argument("--tags", default="manual")
     sp.set_defaults(fn=cmd_add_src)
 
+    sp = sub.add_parser("dedupe", help="схлопнуть дубли записей (по url или source+title)")
+    sp.add_argument("--apply", action="store_true", help="реально удалить лишние строки")
+    sp.set_defaults(fn=cmd_dedupe)
+
     args = p.parse_args(argv)
     return args.fn(args)
 

omni/store.py

@@ -86,6 +86,10 @@ def add(items):
             if _near_dup(c, h):
                 dup += 1
                 continue
+            u = (getattr(it, "url", "") or "").strip()
+            if u and c.execute("SELECT 1 FROM items WHERE url = ? LIMIT 1", (u,)).fetchone():
+                dup += 1
+                continue
             try:
                 cur = c.execute(
                     "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
@@ -243,4 +247,25 @@ def export(path, fmt="jsonl", limit=1000):
         with path.open("w", encoding="utf-8") as f:
             for r in rows:
                 f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n")
-    return len(rows), str(path)
+    return len(rows), str(path)
+
+def dedupe(apply: bool = False):
+    """Схлопывает записи с одинаковым адресом (или source+title при пустом адресе).
+
+    Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено).
+    """
+    key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END"
+    with conn() as c:
+        groups = c.execute(
+            "SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0]
+        extra = c.execute(
+            "SELECT coalesce(sum(n - 1), 0) FROM (SELECT count(*) n FROM items GROUP BY " + key + " HAVING n > 1)").fetchone()[0]
+        killed = 0
+        if apply and extra:
+            ids = [r[0] for r in c.execute(
+                "SELECT id FROM items WHERE id NOT IN (SELECT max(id) FROM items GROUP BY " + key + ")")]
+            for chunk in [ids[i:i + 400] for i in range(0, len(ids), 400)]:
+                q = "DELETE FROM items WHERE id IN (" + ",".join("?" * len(chunk)) + ")"
+                c.execute(q, chunk)
+                killed += len(chunk)
+    return groups, extra, killed