chore(auto): самоаудит 2026-10-26T16:19:01Z — selfcheck_exit=0
This commit is contained in:
Vlad Gothbreach
34
REPORT.md
Normal file
34
REPORT.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# omni-parse · автоотчёт
|
||||
|
||||
<!-- STATS:START -->
|
||||
_обновлено 2026-10-26T16:19:01+00:00_
|
||||
|
||||
| метрика | значение |
|
||||
|---|---|
|
||||
| источников | 635 |
|
||||
| категорий | 11 |
|
||||
| записей | 5281 |
|
||||
| с ценой | 209 |
|
||||
| типизировано | 5281 |
|
||||
| прогонов | 120 |
|
||||
| пустых источников | 200 |
|
||||
|
||||
типы: новость 3516 · софт 495 · релиз 347 · уязвимость 342 · модель 322 · вопрос 245 · цена 192 · гайд 190
|
||||
|
||||
| источник | записей |
|
||||
|---|---|
|
||||
| arxiv_cs_ai | 280 |
|
||||
| community | 215 |
|
||||
| reddit_machinelearning | 208 |
|
||||
| hf_trending_models | 156 |
|
||||
| ab_niflheim_world | 123 |
|
||||
| ab_cpamafia_pro | 121 |
|
||||
| ab_hardforum_com | 120 |
|
||||
| ab_carders_biz | 120 |
|
||||
| ab_zhyk_org | 119 |
|
||||
| ab_ascarding_net | 119 |
|
||||
| ab_altenens_is | 119 |
|
||||
| ab_afflift_com | 119 |
|
||||
|
||||
самоаудит: дубли url: 1
|
||||
<!-- STATS:END -->
|
||||
12
omni/cli.py
12
omni/cli.py
@@ -260,6 +260,14 @@ def cmd_add_src(a):
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_dedupe(a):
|
||||
from . import store
|
||||
groups, extra, killed = store.dedupe(apply=a.apply)
|
||||
print(f"групп дублей: {groups} | лишних строк: {extra} | удалено: {killed}"
|
||||
if a.apply else f"групп дублей: {groups} | лишних строк: {extra} (dry-run, добавь --apply)")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
@@ -357,6 +365,10 @@ def main(argv=None):
|
||||
sp.add_argument("--tags", default="manual")
|
||||
sp.set_defaults(fn=cmd_add_src)
|
||||
|
||||
sp = sub.add_parser("dedupe", help="схлопнуть дубли записей (по url или source+title)")
|
||||
sp.add_argument("--apply", action="store_true", help="реально удалить лишние строки")
|
||||
sp.set_defaults(fn=cmd_dedupe)
|
||||
|
||||
args = p.parse_args(argv)
|
||||
return args.fn(args)
|
||||
|
||||
|
||||
@@ -86,6 +86,10 @@ def add(items):
|
||||
if _near_dup(c, h):
|
||||
dup += 1
|
||||
continue
|
||||
u = (getattr(it, "url", "") or "").strip()
|
||||
if u and c.execute("SELECT 1 FROM items WHERE url = ? LIMIT 1", (u,)).fetchone():
|
||||
dup += 1
|
||||
continue
|
||||
try:
|
||||
cur = c.execute(
|
||||
"INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
|
||||
@@ -244,3 +248,24 @@ def export(path, fmt="jsonl", limit=1000):
|
||||
for r in rows:
|
||||
f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n")
|
||||
return len(rows), str(path)
|
||||
|
||||
def dedupe(apply: bool = False):
|
||||
"""Схлопывает записи с одинаковым адресом (или source+title при пустом адресе).
|
||||
|
||||
Оставляет самую свежую (max id). Возвращает (групп, лишних_строк, удалено).
|
||||
"""
|
||||
key = "CASE WHEN url IS NULL OR url = '' THEN source || '|' || title ELSE url END"
|
||||
with conn() as c:
|
||||
groups = c.execute(
|
||||
"SELECT count(*) FROM (SELECT " + key + " k FROM items GROUP BY k HAVING count(*) > 1)").fetchone()[0]
|
||||
extra = c.execute(
|
||||
"SELECT coalesce(sum(n - 1), 0) FROM (SELECT count(*) n FROM items GROUP BY " + key + " HAVING n > 1)").fetchone()[0]
|
||||
killed = 0
|
||||
if apply and extra:
|
||||
ids = [r[0] for r in c.execute(
|
||||
"SELECT id FROM items WHERE id NOT IN (SELECT max(id) FROM items GROUP BY " + key + ")")]
|
||||
for chunk in [ids[i:i + 400] for i in range(0, len(ids), 400)]:
|
||||
q = "DELETE FROM items WHERE id IN (" + ",".join("?" * len(chunk)) + ")"
|
||||
c.execute(q, chunk)
|
||||
killed += len(chunk)
|
||||
return groups, extra, killed
|
||||
|
||||
49
tools/autocommit.sh
Normal file
49
tools/autocommit.sh
Normal file
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# Автоцикл: самоаудит -> (опц.) подкачка источников -> коммит изменений -> пуш в Gitea и GitHub.
|
||||
# Использование: bash tools/autocommit.sh [--feed] [--quiet]
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.." || exit 1
|
||||
|
||||
PY="/c/Users/User/AppData/Local/Programs/Python/Python311/python.exe"
|
||||
FEED=0
|
||||
for a in "$@"; do [ "$a" = "--feed" ] && FEED=1; done
|
||||
|
||||
export PYTHONIOENCODING=utf-8
|
||||
unset PYTHONPATH
|
||||
|
||||
STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
LOG="data/autocommit.log"
|
||||
mkdir -p data
|
||||
|
||||
echo "== $STAMP ==" >> "$LOG"
|
||||
|
||||
# 1) подкачка новых записей (необязательно)
|
||||
if [ "$FEED" = "1" ]; then
|
||||
"$PY" -m omni feed --limit 8 --workers 6 >> "$LOG" 2>&1
|
||||
echo "feed_exit=$?" >> "$LOG"
|
||||
fi
|
||||
|
||||
# 2) самоаудит + REPORT.md
|
||||
"$PY" tools/selfcheck.py >> "$LOG" 2>&1
|
||||
echo "selfcheck_exit=$?" >> "$LOG"
|
||||
|
||||
# 3) коммит, только если есть реальные изменения кода/отчёта
|
||||
git add -A
|
||||
if git diff --cached --quiet; then
|
||||
echo "no_changes" >> "$LOG"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
STATS="$(tail -1 "$LOG" | sed 's/^/ /')"
|
||||
git commit -q -m "chore(auto): самоаудит $STAMP —$(printf '%s' "$STATS" | cut -c1-160)"
|
||||
COMMIT="$(git rev-parse --short HEAD)"
|
||||
echo "commit=$COMMIT" >> "$LOG"
|
||||
|
||||
# 4) пуш в оба ремоута
|
||||
if git push -q gitea main >> "$LOG" 2>&1; then echo "gitea=OK" >> "$LOG"; else echo "gitea=FAIL" >> "$LOG"; fi
|
||||
if [ -x /c/Users/User/tmp/push_omni_gh.sh ] || [ -f /c/Users/User/tmp/push_omni_gh.sh ]; then
|
||||
if bash /c/Users/User/tmp/push_omni_gh.sh >> "$LOG" 2>&1; then echo "github=OK" >> "$LOG"; else echo "github=FAIL" >> "$LOG"; fi
|
||||
fi
|
||||
|
||||
echo "done $COMMIT" >> "$LOG"
|
||||
echo "autocommit $COMMIT"
|
||||
172
tools/selfcheck.py
Normal file
172
tools/selfcheck.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""Непрерывный самоаудит omni-parse: собирает метрики, правит REPORT.md, копит аномалии.
|
||||
|
||||
Запуск: python tools/selfcheck.py [--json]
|
||||
Ничего не меняет в базе, только читает и пишет REPORT.md + data/selfcheck.jsonl.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import py_compile
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
|
||||
|
||||
DB = ROOT / "data" / "omni.sqlite3"
|
||||
REPORT = ROOT / "REPORT.md"
|
||||
HIST = ROOT / "data" / "selfcheck.jsonl"
|
||||
|
||||
|
||||
def q(con, sql, args=()):
|
||||
try:
|
||||
return con.execute(sql, args).fetchall()
|
||||
except sqlite3.Error:
|
||||
return []
|
||||
|
||||
|
||||
def one(con, sql, args=(), default=0):
|
||||
r = q(con, sql, args)
|
||||
return r[0][0] if r and r[0][0] is not None else default
|
||||
|
||||
|
||||
def compile_all():
|
||||
bad = []
|
||||
for p in sorted((ROOT / "omni").glob("*.py")):
|
||||
try:
|
||||
py_compile.compile(str(p), doraise=True)
|
||||
except py_compile.PyCompileError as e:
|
||||
bad.append(f"{p.name}: {str(e).splitlines()[0][:120]}")
|
||||
return bad
|
||||
|
||||
|
||||
def collect():
|
||||
st: dict = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds")}
|
||||
st["compile_errors"] = compile_all()
|
||||
|
||||
from omni import sources, taxonomy # noqa: E402
|
||||
|
||||
st["sources"] = len(sources.all_sources())
|
||||
st["categories"] = len(sources.CATEGORIES)
|
||||
|
||||
if not DB.exists():
|
||||
st["error"] = "нет базы"
|
||||
return st, None
|
||||
|
||||
con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=20)
|
||||
try:
|
||||
st["items"] = one(con, "SELECT count(*) FROM items")
|
||||
st["priced"] = one(con, "SELECT count(*) FROM items WHERE price IS NOT NULL AND price > 0")
|
||||
st["typed"] = one(con, "SELECT count(*) FROM items WHERE types IS NOT NULL AND types != ''")
|
||||
st["runs"] = one(con, "SELECT count(*) FROM runs")
|
||||
cols = [r[1] for r in q(con, "PRAGMA table_info(items)")]
|
||||
st["has_types_col"] = "types" in cols
|
||||
st["last_run"] = (q(con, "SELECT max(started_at) FROM runs") or [[None]])[0][0]
|
||||
|
||||
st["types"] = {k: v for k, v in q(
|
||||
con, "SELECT types, count(*) FROM items WHERE types != '' GROUP BY types ORDER BY 2 DESC")}
|
||||
flat = {}
|
||||
for combo, n in st["types"].items():
|
||||
for t in str(combo).split(","):
|
||||
t = t.strip()
|
||||
if t:
|
||||
flat[t] = flat.get(t, 0) + n
|
||||
st["type_flat"] = dict(sorted(flat.items(), key=lambda kv: -kv[1]))
|
||||
st["type_labels"] = {k: taxonomy.LABELS.get(k, k) for k in st["type_flat"]}
|
||||
|
||||
st["by_source_top"] = [{"name": r[0], "n": r[1]} for r in q(
|
||||
con, "SELECT source, count(*) FROM items GROUP BY 1 ORDER BY 2 DESC LIMIT 12")]
|
||||
dupes = q(con, "SELECT url, count(*) c FROM items GROUP BY url HAVING c > 1 LIMIT 5")
|
||||
st["dupe_urls_sample"] = len(dupes)
|
||||
st["empty_sources"] = st["sources"] - one(con, "SELECT count(DISTINCT source) FROM items")
|
||||
st["abuse_stored"] = one(con, "SELECT count(*) FROM items WHERE category = 'abuse'") \
|
||||
if "category" in cols else None
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
problems = []
|
||||
if st["compile_errors"]:
|
||||
problems.append(f"compile: {len(st['compile_errors'])} файлов")
|
||||
if not st["has_types_col"]:
|
||||
problems.append("нет колонки items.types")
|
||||
if st["items"] and st["typed"] < st["items"] * 0.95:
|
||||
problems.append(f"не типизировано {st['items'] - st['typed']} записей")
|
||||
if st["dupe_urls_sample"]:
|
||||
problems.append(f"дубли url: {st['dupe_urls_sample']}")
|
||||
if st["sources"] and st["empty_sources"] > st["sources"] * 0.5:
|
||||
problems.append(f"пустых источников {st['empty_sources']}/{st['sources']}")
|
||||
st["problems"] = problems
|
||||
return st, con
|
||||
|
||||
|
||||
def render(st):
|
||||
t = st.get("type_flat") or {}
|
||||
labels = st.get("type_labels") or {}
|
||||
types_line = " · ".join(f"{labels.get(k, k)} {v}" for k, v in list(t.items())[:8]) or "—"
|
||||
top = st.get("by_source_top") or []
|
||||
rows = "\n".join(f"| {r['name']} | {r['n']} |" for r in top) or "| — | 0 |"
|
||||
problems = st.get("problems") or []
|
||||
block = f"""<!-- STATS:START -->
|
||||
_обновлено {st['ts']}_
|
||||
|
||||
| метрика | значение |
|
||||
|---|---|
|
||||
| источников | {st.get('sources', 0)} |
|
||||
| категорий | {st.get('categories', 0)} |
|
||||
| записей | {st.get('items', 0)} |
|
||||
| с ценой | {st.get('priced', 0)} |
|
||||
| типизировано | {st.get('typed', 0)} |
|
||||
| прогонов | {st.get('runs', 0)} |
|
||||
| пустых источников | {st.get('empty_sources', 0)} |
|
||||
|
||||
типы: {types_line}
|
||||
|
||||
| источник | записей |
|
||||
|---|---|
|
||||
{rows}
|
||||
|
||||
самоаудит: {'OK' if not problems else '; '.join(problems)}
|
||||
<!-- STATS:END -->"""
|
||||
return block
|
||||
|
||||
|
||||
def main():
|
||||
st, _ = collect()
|
||||
hist: list = []
|
||||
HIST.parent.mkdir(parents=True, exist_ok=True)
|
||||
if HIST.exists():
|
||||
try:
|
||||
hist = [json.loads(x) for x in HIST.read_text(encoding="utf-8").splitlines() if x.strip()]
|
||||
except Exception:
|
||||
hist = []
|
||||
hist.append({k: st.get(k) for k in ("ts", "items", "priced", "typed", "runs", "sources", "problems")})
|
||||
HIST.write_text("\n".join(json.dumps(x, ensure_ascii=False) for x in hist[-500:]) + "\n", encoding="utf-8")
|
||||
|
||||
block = render(st)
|
||||
if REPORT.exists():
|
||||
old = REPORT.read_text(encoding="utf-8")
|
||||
if "<!-- STATS:START -->" in old and "<!-- STATS:END -->" in old:
|
||||
head = old.split("<!-- STATS:START -->")[0]
|
||||
tail = old.split("<!-- STATS:END -->")[1]
|
||||
new = head + block + tail
|
||||
else:
|
||||
new = old.rstrip() + "\n\n" + block + "\n"
|
||||
else:
|
||||
new = "# omni-parse · автоотчёт\n\n" + block + "\n"
|
||||
REPORT.write_text(new, encoding="utf-8")
|
||||
|
||||
if "--json" in sys.argv:
|
||||
print(json.dumps(st, ensure_ascii=False, indent=1))
|
||||
else:
|
||||
print(f"items={st.get('items')} typed={st.get('typed')} sources={st.get('sources')} "
|
||||
f"priced={st.get('priced')} runs={st.get('runs')} problems={st.get('problems') or 'OK'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user