chore(auto): самоаудит 2026-10-26T16:19:01Z — selfcheck_exit=0

REPORT.md

@@ -0,0 +1,34 @@
+# omni-parse · автоотчёт
+
+<!-- STATS:START -->
+_обновлено 2026-10-26T16:19:01+00:00_
+
+| метрика | значение |
+|---|---|
+| источников | 635 |
+| категорий | 11 |
+| записей | 5281 |
+| с ценой | 209 |
+| типизировано | 5281 |
+| прогонов | 120 |
+| пустых источников | 200 |
+
+типы: новость 3516 · софт 495 · релиз 347 · уязвимость 342 · модель 322 · вопрос 245 · цена 192 · гайд 190
+
+| источник | записей |
+|---|---|
+| arxiv_cs_ai | 280 |
+| community | 215 |
+| reddit_machinelearning | 208 |
+| hf_trending_models | 156 |
+| ab_niflheim_world | 123 |
+| ab_cpamafia_pro | 121 |
+| ab_hardforum_com | 120 |
+| ab_carders_biz | 120 |
+| ab_zhyk_org | 119 |
+| ab_ascarding_net | 119 |
+| ab_altenens_is | 119 |
+| ab_afflift_com | 119 |
+
+самоаудит: дубли url: 1
+<!-- STATS:END -->

omni/cli.py

@@ -260,6 +260,14 @@ def cmd_add_src(a):
     return 0
 
 
+def cmd_dedupe(a):
+    from . import store
+    groups, extra, killed = store.dedupe(apply=a.apply)
+    print(f"групп дублей: {groups} | лишних строк: {extra} | удалено: {killed}"
+          if a.apply else f"групп дублей: {groups} | лишних строк: {extra} (dry-run, добавь --apply)")
+    return 0
+
+
 def main(argv=None):
     try:
         sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -357,6 +365,10 @@ def main(argv=None):
     sp.add_argument("--tags", default="manual")
     sp.set_defaults(fn=cmd_add_src)
 
+    sp = sub.add_parser("dedupe", help="схлопнуть дубли записей (по url или source+title)")
+    sp.add_argument("--apply", action="store_true", help="реально удалить лишние строки")
+    sp.set_defaults(fn=cmd_dedupe)
+
     args = p.parse_args(argv)
     return args.fn(args)
 

omni/store.py

@@ -86,6 +86,10 @@ def add(items):
             if _near_dup(c, h):
                 dup += 1
                 continue
+            u = (getattr(it, "url", "") or "").strip()
+            if u and c.execute("SELECT 1 FROM items WHERE url = ? LIMIT 1", (u,)).fetchone():
+                dup += 1
+                continue
             try:
                 cur = c.execute(
                     "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
@@ -244,3 +248,24 @@ def export(path, fmt="jsonl", limit=1000):
             for r in rows:
                 f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n")
     return len(rows), str(path)
+
+def dedupe(apply: bool = False):
+    """Схлопывает записи с одинаковым адресом (или source+title при пустом адресе).
+
+    Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено).
+    """
+    key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END"
+    with conn() as c:
+        groups = c.execute(
+            "SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0]
+        extra = c.execute(
+            "SELECT coalesce(sum(n - 1), 0) FROM (SELECT count(*) n FROM items GROUP BY " + key + " HAVING n > 1)").fetchone()[0]
+        killed = 0
+        if apply and extra:
+            ids = [r[0] for r in c.execute(
+                "SELECT id FROM items WHERE id NOT IN (SELECT max(id) FROM items GROUP BY " + key + ")")]
+            for chunk in [ids[i:i + 400] for i in range(0, len(ids), 400)]:
+                q = "DELETE FROM items WHERE id IN (" + ",".join("?" * len(chunk)) + ")"
+                c.execute(q, chunk)
+                killed += len(chunk)
+    return groups, extra, killed

tools/autocommit.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Автоцикл: самоаудит -> (опц.) подкачка источников -> коммит изменений -> пуш в Gitea и GitHub.
+# Использование: bash tools/autocommit.sh [--feed] [--quiet]
+set -uo pipefail
+cd "$(dirname "$0")/.." || exit 1
+
+PY="/c/Users/User/AppData/Local/Programs/Python/Python311/python.exe"
+FEED=0
+for a in "$@"; do [ "$a" = "--feed" ] && FEED=1; done
+
+export PYTHONIOENCODING=utf-8
+unset PYTHONPATH
+
+STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+LOG="data/autocommit.log"
+mkdir -p data
+
+echo "== $STAMP ==" >> "$LOG"
+
+# 1) подкачка новых записей (необязательно)
+if [ "$FEED" = "1" ]; then
+  "$PY" -m omni feed --limit 8 --workers 6 >> "$LOG" 2>&1
+  echo "feed_exit=$?" >> "$LOG"
+fi
+
+# 2) самоаудит + REPORT.md
+"$PY" tools/selfcheck.py >> "$LOG" 2>&1
+echo "selfcheck_exit=$?" >> "$LOG"
+
+# 3) коммит, только если есть реальные изменения кода/отчёта
+git add -A
+if git diff --cached --quiet; then
+  echo "no_changes" >> "$LOG"
+  exit 0
+fi
+
+STATS="$(tail -1 "$LOG" | sed 's/^/ /')"
+git commit -q -m "chore(auto): самоаудит $STAMP —$(printf '%s' "$STATS" | cut -c1-160)"
+COMMIT="$(git rev-parse --short HEAD)"
+echo "commit=$COMMIT" >> "$LOG"
+
+# 4) пуш в оба ремоута
+if git push -q gitea main >> "$LOG" 2>&1; then echo "gitea=OK" >> "$LOG"; else echo "gitea=FAIL" >> "$LOG"; fi
+if [ -x /c/Users/User/tmp/push_omni_gh.sh ] || [ -f /c/Users/User/tmp/push_omni_gh.sh ]; then
+  if bash /c/Users/User/tmp/push_omni_gh.sh >> "$LOG" 2>&1; then echo "github=OK" >> "$LOG"; else echo "github=FAIL" >> "$LOG"; fi
+fi
+
+echo "done $COMMIT" >> "$LOG"
+echo "autocommit $COMMIT"

tools/selfcheck.py

@@ -0,0 +1,172 @@
+"""Непрерывный самоаудит omni-parse: собирает метрики, правит REPORT.md, копит аномалии.
+
+Запуск: python tools/selfcheck.py [--json]
+Ничего не меняет в базе, только читает и пишет REPORT.md + data/selfcheck.jsonl.
+"""
+from __future__ import annotations
+
+import json
+import os
+import py_compile
+import sqlite3
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+os.environ.setdefault("PYTHONIOENCODING", "utf-8")
+
+DB = ROOT / "data" / "omni.sqlite3"
+REPORT = ROOT / "REPORT.md"
+HIST = ROOT / "data" / "selfcheck.jsonl"
+
+
+def q(con, sql, args=()):
+    try:
+        return con.execute(sql, args).fetchall()
+    except sqlite3.Error:
+        return []
+
+
+def one(con, sql, args=(), default=0):
+    r = q(con, sql, args)
+    return r[0][0] if r and r[0][0] is not None else default
+
+
+def compile_all():
+    bad = []
+    for p in sorted((ROOT / "omni").glob("*.py")):
+        try:
+            py_compile.compile(str(p), doraise=True)
+        except py_compile.PyCompileError as e:
+            bad.append(f"{p.name}: {str(e).splitlines()[0][:120]}")
+    return bad
+
+
+def collect():
+    st: dict = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds")}
+    st["compile_errors"] = compile_all()
+
+    from omni import sources, taxonomy  # noqa: E402
+
+    st["sources"] = len(sources.all_sources())
+    st["categories"] = len(sources.CATEGORIES)
+
+    if not DB.exists():
+        st["error"] = "нет базы"
+        return st, None
+
+    con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=20)
+    try:
+        st["items"] = one(con, "SELECT count(*) FROM items")
+        st["priced"] = one(con, "SELECT count(*) FROM items WHERE price IS NOT NULL AND price > 0")
+        st["typed"] = one(con, "SELECT count(*) FROM items WHERE types IS NOT NULL AND types != ''")
+        st["runs"] = one(con, "SELECT count(*) FROM runs")
+        cols = [r[1] for r in q(con, "PRAGMA table_info(items)")]
+        st["has_types_col"] = "types" in cols
+        st["last_run"] = (q(con, "SELECT max(started_at) FROM runs") or [[None]])[0][0]
+
+        st["types"] = {k: v for k, v in q(
+            con, "SELECT types, count(*) FROM items WHERE types != '' GROUP BY types ORDER BY 2 DESC")}
+        flat = {}
+        for combo, n in st["types"].items():
+            for t in str(combo).split(","):
+                t = t.strip()
+                if t:
+                    flat[t] = flat.get(t, 0) + n
+        st["type_flat"] = dict(sorted(flat.items(), key=lambda kv: -kv[1]))
+        st["type_labels"] = {k: taxonomy.LABELS.get(k, k) for k in st["type_flat"]}
+
+        st["by_source_top"] = [{"name": r[0], "n": r[1]} for r in q(
+            con, "SELECT source, count(*) FROM items GROUP BY 1 ORDER BY 2 DESC LIMIT 12")]
+        dupes = q(con, "SELECT url, count(*) c FROM items GROUP BY url HAVING c > 1 LIMIT 5")
+        st["dupe_urls_sample"] = len(dupes)
+        st["empty_sources"] = st["sources"] - one(con, "SELECT count(DISTINCT source) FROM items")
+        st["abuse_stored"] = one(con, "SELECT count(*) FROM items WHERE category = 'abuse'") \
+            if "category" in cols else None
+    finally:
+        con.close()
+
+    problems = []
+    if st["compile_errors"]:
+        problems.append(f"compile: {len(st['compile_errors'])} файлов")
+    if not st["has_types_col"]:
+        problems.append("нет колонки items.types")
+    if st["items"] and st["typed"] < st["items"] * 0.95:
+        problems.append(f"не типизировано {st['items'] - st['typed']} записей")
+    if st["dupe_urls_sample"]:
+        problems.append(f"дубли url: {st['dupe_urls_sample']}")
+    if st["sources"] and st["empty_sources"] > st["sources"] * 0.5:
+        problems.append(f"пустых источников {st['empty_sources']}/{st['sources']}")
+    st["problems"] = problems
+    return st, con
+
+
+def render(st):
+    t = st.get("type_flat") or {}
+    labels = st.get("type_labels") or {}
+    types_line = " · ".join(f"{labels.get(k, k)} {v}" for k, v in list(t.items())[:8]) or "—"
+    top = st.get("by_source_top") or []
+    rows = "\n".join(f"| {r['name']} | {r['n']} |" for r in top) or "| — | 0 |"
+    problems = st.get("problems") or []
+    block = f"""<!-- STATS:START -->
+_обновлено {st['ts']}_
+
+| метрика | значение |
+|---|---|
+| источников | {st.get('sources', 0)} |
+| категорий | {st.get('categories', 0)} |
+| записей | {st.get('items', 0)} |
+| с ценой | {st.get('priced', 0)} |
+| типизировано | {st.get('typed', 0)} |
+| прогонов | {st.get('runs', 0)} |
+| пустых источников | {st.get('empty_sources', 0)} |
+
+типы: {types_line}
+
+| источник | записей |
+|---|---|
+{rows}
+
+самоаудит: {'OK' if not problems else '; '.join(problems)}
+<!-- STATS:END -->"""
+    return block
+
+
+def main():
+    st, _ = collect()
+    hist: list = []
+    HIST.parent.mkdir(parents=True, exist_ok=True)
+    if HIST.exists():
+        try:
+            hist = [json.loads(x) for x in HIST.read_text(encoding="utf-8").splitlines() if x.strip()]
+        except Exception:
+            hist = []
+    hist.append({k: st.get(k) for k in ("ts", "items", "priced", "typed", "runs", "sources", "problems")})
+    HIST.write_text("\n".join(json.dumps(x, ensure_ascii=False) for x in hist[-500:]) + "\n", encoding="utf-8")
+
+    block = render(st)
+    if REPORT.exists():
+        old = REPORT.read_text(encoding="utf-8")
+        if "<!-- STATS:START -->" in old and "<!-- STATS:END -->" in old:
+            head = old.split("<!-- STATS:START -->")[0]
+            tail = old.split("<!-- STATS:END -->")[1]
+            new = head + block + tail
+        else:
+            new = old.rstrip() + "\n\n" + block + "\n"
+    else:
+        new = "# omni-parse · автоотчёт\n\n" + block + "\n"
+    REPORT.write_text(new, encoding="utf-8")
+
+    if "--json" in sys.argv:
+        print(json.dumps(st, ensure_ascii=False, indent=1))
+    else:
+        print(f"items={st.get('items')} typed={st.get('typed')} sources={st.get('sources')} "
+              f"priced={st.get('priced')} runs={st.get('runs')} problems={st.get('problems') or 'OK'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())