sources: авто-расширение из публичных API без ключей (46 живых), universal generic_json, bsky/mastodon, --only по префиксу, auto_cycle

tools/auto_cycle.py

@@ -0,0 +1,90 @@
+"""Один цикл автоулучшения: проверить кандидатов → пересобрать sources_auto.py →
+докачать новое → пересобрать abuse-индекс → коммит/пуш. Запускать по крону (6ч).
+
+    python tools/auto_cycle.py            # полный цикл
+    python tools/auto_cycle.py --dry      # без коммита и без сети (только отчёт)
+
+Лог: data/auto_cycle.log (append).
+"""
+import argparse
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+PY = sys.executable
+LOG = ROOT / "data" / "auto_cycle.log"
+ENV = dict(os.environ, PYTHONPATH="", PYTHONIOENCODING="utf-8", PYTHONUTF8="1")
+
+
+def log(msg):
+    line = "%s %s" % (datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M"), msg)
+    print(line)
+    with LOG.open("a", encoding="utf-8") as fh:
+        fh.write(line + "\n")
+
+
+def run(args, timeout=1800):
+    t = time.time()
+    p = subprocess.run([PY] + args, cwd=str(ROOT), env=ENV, capture_output=True, text=True,
+                       encoding="utf-8", errors="replace", timeout=timeout)
+    out = (p.stdout or "") + (p.stderr or "")
+    return p.returncode, out.strip(), round(time.time() - t, 1)
+
+
+def git(*args, timeout=180):
+    p = subprocess.run(["git"] + list(args), cwd=str(ROOT), capture_output=True, text=True,
+                       encoding="utf-8", errors="replace", timeout=timeout)
+    return p.returncode, ((p.stdout or "") + (p.stderr or "")).strip()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dry", action="store_true")
+    ap.add_argument("--pause", type=float, default=6.0)
+    a = ap.parse_args()
+    log("=== cycle start ===")
+
+    if not a.dry:
+        code, out, sec = run(["tools/auto_expand.py", "--probe", "--write", "--pause", str(a.pause)])
+        tail = [l for l in out.splitlines() if "LIVE" in l or "wrote" in l or "skip (" in l]
+        log("probe/write rc=%s %ss | %s" % (code, sec, " / ".join(tail[-3:]) or out[-200:]))
+
+    changed = git("status", "--porcelain", "--", "omni/sources_auto.py")[1]
+    live = 0
+    try:
+        import re
+        txt = (ROOT / "omni" / "sources_auto.py").read_text(encoding="utf-8")
+        live = len(re.findall(r"^    A\(", txt, re.M))
+    except Exception:
+        pass
+    log("sources_auto: live=%d changed=%s" % (live, bool(changed)))
+
+    if changed and not a.dry:
+        code, out, sec = run(["-m", "omni", "feed", "--only", "aa_", "--limit", "15"])
+        log("feed rc=%s %ss | %s" % (code, sec, out.splitlines()[-1] if out else ""))
+
+    if not a.dry:
+        code, out, sec = run(["-c", "import sys;sys.path.insert(0,'.');"
+                              "from omni import abuse;print('rebuilt', abuse.rebuild())"], timeout=2400)
+        log("abuse rc=%s %ss | %s" % (code, sec, (out.splitlines() or [""])[-1]))
+
+    if changed and not a.dry:
+        git("add", "--", "omni/sources_auto.py")
+        code, out = git("commit", "-m", "auto: источники %d live (%s)" %
+                        (live, datetime.now(timezone.utc).strftime("%d.%m %H:%M UTC")))
+        log("commit rc=%s | %s" % (code, out.splitlines()[0] if out else ""))
+        for remote in ("gitea", "github"):
+            if not git("remote")[1].find(remote) >= 0:
+                continue
+            rc, o = git("push", remote, "main")
+            log("push %s rc=%s | %s" % (remote, rc, o.splitlines()[-1] if o else ""))
+    log("=== cycle end ===")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())