diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 0000000..b6b0263 --- /dev/null +++ b/REPORT.md @@ -0,0 +1,34 @@ +# omni-parse · автоотчёт + + +_обновлено 2026-10-26T16:19:01+00:00_ + +| метрика | значение | +|---|---| +| источников | 635 | +| категорий | 11 | +| записей | 5281 | +| с ценой | 209 | +| типизировано | 5281 | +| прогонов | 120 | +| пустых источников | 200 | + +типы: новость 3516 · софт 495 · релиз 347 · уязвимость 342 · модель 322 · вопрос 245 · цена 192 · гайд 190 + +| источник | записей | +|---|---| +| arxiv_cs_ai | 280 | +| community | 215 | +| reddit_machinelearning | 208 | +| hf_trending_models | 156 | +| ab_niflheim_world | 123 | +| ab_cpamafia_pro | 121 | +| ab_hardforum_com | 120 | +| ab_carders_biz | 120 | +| ab_zhyk_org | 119 | +| ab_ascarding_net | 119 | +| ab_altenens_is | 119 | +| ab_afflift_com | 119 | + +самоаудит: дубли url: 1 + diff --git a/omni/cli.py b/omni/cli.py index 1bc4463..cc0fba2 100644 --- a/omni/cli.py +++ b/omni/cli.py @@ -260,6 +260,14 @@ def cmd_add_src(a): return 0 +def cmd_dedupe(a): + from . import store + groups, extra, killed = store.dedupe(apply=a.apply) + print(f"групп дублей: {groups} | лишних строк: {extra} | удалено: {killed}" + if a.apply else f"групп дублей: {groups} | лишних строк: {extra} (dry-run, добавь --apply)") + return 0 + + def main(argv=None): try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") @@ -357,6 +365,10 @@ def main(argv=None): sp.add_argument("--tags", default="manual") sp.set_defaults(fn=cmd_add_src) + sp = sub.add_parser("dedupe", help="схлопнуть дубли записей (по url или source+title)") + sp.add_argument("--apply", action="store_true", help="реально удалить лишние строки") + sp.set_defaults(fn=cmd_dedupe) + args = p.parse_args(argv) return args.fn(args) diff --git a/omni/store.py b/omni/store.py index f348a2a..d7a6e1b 100644 --- a/omni/store.py +++ b/omni/store.py @@ -86,6 +86,10 @@ def add(items): if _near_dup(c, h): dup += 1 continue + u = (getattr(it, "url", "") or "").strip() + if u and c.execute("SELECT 1 FROM items WHERE url = ? LIMIT 1", (u,)).fetchone(): + dup += 1 + continue try: cur = c.execute( "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang," @@ -243,4 +247,25 @@ def export(path, fmt="jsonl", limit=1000): with path.open("w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n") - return len(rows), str(path) \ No newline at end of file + return len(rows), str(path) + +def dedupe(apply: bool = False): + """Схлопывает записи с одинаковым адресом (или source+title при пустом адресе). + + Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено). + """ + key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END" + with conn() as c: + groups = c.execute( + "SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0] + extra = c.execute( + "SELECT coalesce(sum(n - 1), 0) FROM (SELECT count(*) n FROM items GROUP BY " + key + " HAVING n > 1)").fetchone()[0] + killed = 0 + if apply and extra: + ids = [r[0] for r in c.execute( + "SELECT id FROM items WHERE id NOT IN (SELECT max(id) FROM items GROUP BY " + key + ")")] + for chunk in [ids[i:i + 400] for i in range(0, len(ids), 400)]: + q = "DELETE FROM items WHERE id IN (" + ",".join("?" * len(chunk)) + ")" + c.execute(q, chunk) + killed += len(chunk) + return groups, extra, killed diff --git a/tools/autocommit.sh b/tools/autocommit.sh new file mode 100644 index 0000000..fdaa0c3 --- /dev/null +++ b/tools/autocommit.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Автоцикл: самоаудит -> (опц.) подкачка источников -> коммит изменений -> пуш в Gitea и GitHub. +# Использование: bash tools/autocommit.sh [--feed] [--quiet] +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PY="/c/Users/User/AppData/Local/Programs/Python/Python311/python.exe" +FEED=0 +for a in "$@"; do [ "$a" = "--feed" ] && FEED=1; done + +export PYTHONIOENCODING=utf-8 +unset PYTHONPATH + +STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +LOG="data/autocommit.log" +mkdir -p data + +echo "== $STAMP ==" >> "$LOG" + +# 1) подкачка новых записей (необязательно) +if [ "$FEED" = "1" ]; then + "$PY" -m omni feed --limit 8 --workers 6 >> "$LOG" 2>&1 + echo "feed_exit=$?" >> "$LOG" +fi + +# 2) самоаудит + REPORT.md +"$PY" tools/selfcheck.py >> "$LOG" 2>&1 +echo "selfcheck_exit=$?" >> "$LOG" + +# 3) коммит, только если есть реальные изменения кода/отчёта +git add -A +if git diff --cached --quiet; then + echo "no_changes" >> "$LOG" + exit 0 +fi + +STATS="$(tail -1 "$LOG" | sed 's/^/ /')" +git commit -q -m "chore(auto): самоаудит $STAMP —$(printf '%s' "$STATS" | cut -c1-160)" +COMMIT="$(git rev-parse --short HEAD)" +echo "commit=$COMMIT" >> "$LOG" + +# 4) пуш в оба ремоута +if git push -q gitea main >> "$LOG" 2>&1; then echo "gitea=OK" >> "$LOG"; else echo "gitea=FAIL" >> "$LOG"; fi +if [ -x /c/Users/User/tmp/push_omni_gh.sh ] || [ -f /c/Users/User/tmp/push_omni_gh.sh ]; then + if bash /c/Users/User/tmp/push_omni_gh.sh >> "$LOG" 2>&1; then echo "github=OK" >> "$LOG"; else echo "github=FAIL" >> "$LOG"; fi +fi + +echo "done $COMMIT" >> "$LOG" +echo "autocommit $COMMIT" \ No newline at end of file diff --git a/tools/selfcheck.py b/tools/selfcheck.py new file mode 100644 index 0000000..2e54eca --- /dev/null +++ b/tools/selfcheck.py @@ -0,0 +1,172 @@ +"""Непрерывный самоаудит omni-parse: собирает метрики, правит REPORT.md, копит аномалии. + +Запуск: python tools/selfcheck.py [--json] +Ничего не меняет в базе, только читает и пишет REPORT.md + data/selfcheck.jsonl. +""" +from __future__ import annotations + +import json +import os +import py_compile +import sqlite3 +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) +os.environ.setdefault("PYTHONIOENCODING", "utf-8") + +DB = ROOT / "data" / "omni.sqlite3" +REPORT = ROOT / "REPORT.md" +HIST = ROOT / "data" / "selfcheck.jsonl" + + +def q(con, sql, args=()): + try: + return con.execute(sql, args).fetchall() + except sqlite3.Error: + return [] + + +def one(con, sql, args=(), default=0): + r = q(con, sql, args) + return r[0][0] if r and r[0][0] is not None else default + + +def compile_all(): + bad = [] + for p in sorted((ROOT / "omni").glob("*.py")): + try: + py_compile.compile(str(p), doraise=True) + except py_compile.PyCompileError as e: + bad.append(f"{p.name}: {str(e).splitlines()[0][:120]}") + return bad + + +def collect(): + st: dict = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds")} + st["compile_errors"] = compile_all() + + from omni import sources, taxonomy # noqa: E402 + + st["sources"] = len(sources.all_sources()) + st["categories"] = len(sources.CATEGORIES) + + if not DB.exists(): + st["error"] = "нет базы" + return st, None + + con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=20) + try: + st["items"] = one(con, "SELECT count(*) FROM items") + st["priced"] = one(con, "SELECT count(*) FROM items WHERE price IS NOT NULL AND price > 0") + st["typed"] = one(con, "SELECT count(*) FROM items WHERE types IS NOT NULL AND types != ''") + st["runs"] = one(con, "SELECT count(*) FROM runs") + cols = [r[1] for r in q(con, "PRAGMA table_info(items)")] + st["has_types_col"] = "types" in cols + st["last_run"] = (q(con, "SELECT max(started_at) FROM runs") or [[None]])[0][0] + + st["types"] = {k: v for k, v in q( + con, "SELECT types, count(*) FROM items WHERE types != '' GROUP BY types ORDER BY 2 DESC")} + flat = {} + for combo, n in st["types"].items(): + for t in str(combo).split(","): + t = t.strip() + if t: + flat[t] = flat.get(t, 0) + n + st["type_flat"] = dict(sorted(flat.items(), key=lambda kv: -kv[1])) + st["type_labels"] = {k: taxonomy.LABELS.get(k, k) for k in st["type_flat"]} + + st["by_source_top"] = [{"name": r[0], "n": r[1]} for r in q( + con, "SELECT source, count(*) FROM items GROUP BY 1 ORDER BY 2 DESC LIMIT 12")] + dupes = q(con, "SELECT url, count(*) c FROM items GROUP BY url HAVING c > 1 LIMIT 5") + st["dupe_urls_sample"] = len(dupes) + st["empty_sources"] = st["sources"] - one(con, "SELECT count(DISTINCT source) FROM items") + st["abuse_stored"] = one(con, "SELECT count(*) FROM items WHERE category = 'abuse'") \ + if "category" in cols else None + finally: + con.close() + + problems = [] + if st["compile_errors"]: + problems.append(f"compile: {len(st['compile_errors'])} файлов") + if not st["has_types_col"]: + problems.append("нет колонки items.types") + if st["items"] and st["typed"] < st["items"] * 0.95: + problems.append(f"не типизировано {st['items'] - st['typed']} записей") + if st["dupe_urls_sample"]: + problems.append(f"дубли url: {st['dupe_urls_sample']}") + if st["sources"] and st["empty_sources"] > st["sources"] * 0.5: + problems.append(f"пустых источников {st['empty_sources']}/{st['sources']}") + st["problems"] = problems + return st, con + + +def render(st): + t = st.get("type_flat") or {} + labels = st.get("type_labels") or {} + types_line = " · ".join(f"{labels.get(k, k)} {v}" for k, v in list(t.items())[:8]) or "—" + top = st.get("by_source_top") or [] + rows = "\n".join(f"| {r['name']} | {r['n']} |" for r in top) or "| — | 0 |" + problems = st.get("problems") or [] + block = f""" +_обновлено {st['ts']}_ + +| метрика | значение | +|---|---| +| источников | {st.get('sources', 0)} | +| категорий | {st.get('categories', 0)} | +| записей | {st.get('items', 0)} | +| с ценой | {st.get('priced', 0)} | +| типизировано | {st.get('typed', 0)} | +| прогонов | {st.get('runs', 0)} | +| пустых источников | {st.get('empty_sources', 0)} | + +типы: {types_line} + +| источник | записей | +|---|---| +{rows} + +самоаудит: {'OK' if not problems else '; '.join(problems)} +""" + return block + + +def main(): + st, _ = collect() + hist: list = [] + HIST.parent.mkdir(parents=True, exist_ok=True) + if HIST.exists(): + try: + hist = [json.loads(x) for x in HIST.read_text(encoding="utf-8").splitlines() if x.strip()] + except Exception: + hist = [] + hist.append({k: st.get(k) for k in ("ts", "items", "priced", "typed", "runs", "sources", "problems")}) + HIST.write_text("\n".join(json.dumps(x, ensure_ascii=False) for x in hist[-500:]) + "\n", encoding="utf-8") + + block = render(st) + if REPORT.exists(): + old = REPORT.read_text(encoding="utf-8") + if "" in old and "" in old: + head = old.split("")[0] + tail = old.split("")[1] + new = head + block + tail + else: + new = old.rstrip() + "\n\n" + block + "\n" + else: + new = "# omni-parse · автоотчёт\n\n" + block + "\n" + REPORT.write_text(new, encoding="utf-8") + + if "--json" in sys.argv: + print(json.dumps(st, ensure_ascii=False, indent=1)) + else: + print(f"items={st.get('items')} typed={st.get('typed')} sources={st.get('sources')} " + f"priced={st.get('priced')} runs={st.get('runs')} problems={st.get('problems') or 'OK'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file