sources: авто-расширение из публичных API без ключей (46 живых), universal generic_json, bsky/mastodon, --only по префиксу, auto_cycle

tools/auto_cycle.py

@@ -0,0 +1,90 @@
+"""Один цикл автоулучшения: проверить кандидатов → пересобрать sources_auto.py →
+докачать новое → пересобрать abuse-индекс → коммит/пуш. Запускать по крону (6ч).
+
+    python tools/auto_cycle.py            # полный цикл
+    python tools/auto_cycle.py --dry      # без коммита и без сети (только отчёт)
+
+Лог: data/auto_cycle.log (append).
+"""
+import argparse
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+PY = sys.executable
+LOG = ROOT / "data" / "auto_cycle.log"
+ENV = dict(os.environ, PYTHONPATH="", PYTHONIOENCODING="utf-8", PYTHONUTF8="1")
+
+
+def log(msg):
+    line = "%s %s" % (datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M"), msg)
+    print(line)
+    with LOG.open("a", encoding="utf-8") as fh:
+        fh.write(line + "\n")
+
+
+def run(args, timeout=1800):
+    t = time.time()
+    p = subprocess.run([PY] + args, cwd=str(ROOT), env=ENV, capture_output=True, text=True,
+                       encoding="utf-8", errors="replace", timeout=timeout)
+    out = (p.stdout or "") + (p.stderr or "")
+    return p.returncode, out.strip(), round(time.time() - t, 1)
+
+
+def git(*args, timeout=180):
+    p = subprocess.run(["git"] + list(args), cwd=str(ROOT), capture_output=True, text=True,
+                       encoding="utf-8", errors="replace", timeout=timeout)
+    return p.returncode, ((p.stdout or "") + (p.stderr or "")).strip()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dry", action="store_true")
+    ap.add_argument("--pause", type=float, default=6.0)
+    a = ap.parse_args()
+    log("=== cycle start ===")
+
+    if not a.dry:
+        code, out, sec = run(["tools/auto_expand.py", "--probe", "--write", "--pause", str(a.pause)])
+        tail = [l for l in out.splitlines() if "LIVE" in l or "wrote" in l or "skip (" in l]
+        log("probe/write rc=%s %ss | %s" % (code, sec, " / ".join(tail[-3:]) or out[-200:]))
+
+    changed = git("status", "--porcelain", "--", "omni/sources_auto.py")[1]
+    live = 0
+    try:
+        import re
+        txt = (ROOT / "omni" / "sources_auto.py").read_text(encoding="utf-8")
+        live = len(re.findall(r"^    A\(", txt, re.M))
+    except Exception:
+        pass
+    log("sources_auto: live=%d changed=%s" % (live, bool(changed)))
+
+    if changed and not a.dry:
+        code, out, sec = run(["-m", "omni", "feed", "--only", "aa_", "--limit", "15"])
+        log("feed rc=%s %ss | %s" % (code, sec, out.splitlines()[-1] if out else ""))
+
+    if not a.dry:
+        code, out, sec = run(["-c", "import sys;sys.path.insert(0,'.');"
+                              "from omni import abuse;print('rebuilt', abuse.rebuild())"], timeout=2400)
+        log("abuse rc=%s %ss | %s" % (code, sec, (out.splitlines() or [""])[-1]))
+
+    if changed and not a.dry:
+        git("add", "--", "omni/sources_auto.py")
+        code, out = git("commit", "-m", "auto: источники %d live (%s)" %
+                        (live, datetime.now(timezone.utc).strftime("%d.%m %H:%M UTC")))
+        log("commit rc=%s | %s" % (code, out.splitlines()[0] if out else ""))
+        for remote in ("gitea", "github"):
+            if not git("remote")[1].find(remote) >= 0:
+                continue
+            rc, o = git("push", remote, "main")
+            log("push %s rc=%s | %s" % (remote, rc, o.splitlines()[-1] if o else ""))
+    log("=== cycle end ===")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/auto_expand.py

@@ -0,0 +1,218 @@
+"""Авто-расширение источников: тест кандидатов → генерация omni/sources_auto.py.
+
+    python tools/auto_expand.py --probe         # только проверка, без записи
+    python tools/auto_expand.py --write         # живые кандидаты → sources_auto.py
+    python tools/auto_expand.py --discover      # добрать кандидатов из public-apis
+    python tools/auto_expand.py --add NAME URL --adapter rss
+
+Кандидаты: data/source_candidates.json (list of {name,url,adapter,kind,extra,tags,limit}).
+Здоровье: data/source_health.json — сколько раз источник отдал items и когда.
+Мёртвым считается кандидат с 0 items в 3 прогонах подряд → исключается из генерации.
+"""
+import argparse
+import json
+import sys
+import time
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from omni import feeds  # noqa: E402
+from omni.sources import Source  # noqa: E402
+
+CAND = ROOT / "data" / "source_candidates.json"
+HEALTH = ROOT / "data" / "source_health.json"
+OUT = ROOT / "omni" / "sources_auto.py"
+PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md"
+
+HEADER = '''"""Авто-расширение: источники на публичных API без ключа.
+
+Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
+(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.
+
+Обновлено: {ts} · живых {live} из {total}
+"""
+
+
+def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None):
+    return {{"name": name, "category": "abuse", "kind": kind, "url": url,
+            "adapter": adapter, "tags": tuple(tags), "limit": limit,
+            "detail": False, "extra": dict(extra or {{}})}}
+
+
+AUTO = [
+{body}]
+'''
+
+GROUP_RU = {
+    "json": "# ---------- открытые JSON API без ключа ----------",
+    "reddit": "# ---------- Reddit через pullpush.io (без ключа) ----------",
+    "social": "# ---------- Bluesky / Mastodon ----------",
+    "discourse": "# ---------- Discourse-сообщества ----------",
+    "rss": "# ---------- RSS/Atom ----------",
+    "tg": "# ---------- Telegram-превью ----------",
+}
+
+
+def load(path, default):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return default
+
+
+def save(path, data):
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
+
+
+def to_source(c):
+    return Source(name=c["name"], category=c.get("category", "abuse"), kind=c.get("kind", "json"),
+                  url=c["url"], adapter=c["adapter"], tags=tuple(c.get("tags") or ("abuse",)),
+                  limit=int(c.get("limit", 20)), detail=False, extra=dict(c.get("extra") or {}))
+
+
+def probe(cands, pause=2.0):
+    health = load(HEALTH, {})
+    res = []
+    last_host = None
+    for c in cands:
+        host = c["url"].split("/")[2] if "//" in c["url"] else ""
+        if host == last_host and pause:
+            time.sleep(pause)
+        last_host = host
+        try:
+            items = feeds.fetch(to_source(c), limit=min(20, int(c.get("limit", 20))))
+            n = len(items)
+            sample = [i.title[:70] for i in items[:2]]
+            err = ""
+        except Exception as exc:
+            n, sample, err = 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:90])
+        h = health.setdefault(c["name"], {"ok": 0, "fail": 0, "items": 0})
+        h["items"] = n
+        h["ok"] = h.get("ok", 0) + 1 if n else 0
+        h["fail"] = 0 if n else h.get("fail", 0) + 1
+        h["last"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M")
+        h["url"] = c["url"]
+        if err:
+            h["error"] = err
+        else:
+            h.pop("error", None)
+        res.append((c, n, sample, err))
+        print("%-26s %-4s %3d  %s" % (c["name"], "LIVE" if n else "DEAD", n, err or " | ".join(sample)))
+    save(HEALTH, health)
+    return res
+
+
+def write_module(cands, results):
+    health = load(HEALTH, {})
+    order = ["json", "reddit", "social", "discourse", "rss", "tg"]
+    buckets = {k: [] for k in order}
+    for c, n, _, _ in results:
+        if not n:
+            continue
+        if (health.get(c["name"]) or {}).get("fail", 0) >= 3:
+            print("skip (3 fails подряд):", c["name"])
+            continue
+        g = c.get("group") or ("tg" if c["adapter"] == "tg_web" else
+                               "discourse" if c["adapter"] == "discourse" else
+                               "reddit" if "pullpush" in c["url"] or "reddit" in c["url"] else
+                               "social" if c["adapter"] in ("bsky_search", "mastodon_tag") else
+                               "rss" if c["adapter"] == "rss" else "json")
+        buckets.setdefault(g, []).append(c)
+    body = []
+    live = 0
+    for g in order + [k for k in buckets if k not in order]:
+        rows = buckets.get(g) or []
+        if not rows:
+            continue
+        body.append("    " + GROUP_RU.get(g, "# ---------- прочее ----------"))
+        for c in rows:
+            live += 1
+            extra = ""
+            if c.get("extra"):
+                extra = ", extra=%s" % json.dumps(c["extra"], ensure_ascii=False)
+            tags = ", ".join('"%s"' % t for t in (c.get("tags") or ("abuse",)))
+            body.append('    A("%s", "%s", "%s", "%s", (%s)%s),' % (
+                c["name"], c.get("kind", "json"), c["url"], c["adapter"], tags, extra))
+        body.append("")
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    OUT.write_text(HEADER.format(ts=ts, live=live, total=len(cands), body="\n".join(body) + "\n"),
+                   encoding="utf-8")
+    print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands)))
+
+
+def discover(limit=40):
+    """Добрать бесплатные (Auth=No) API из public-apis."""
+    req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"})
+    md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace")
+    rows = []
+    for line in md.splitlines():
+        if not line.startswith("|") or "http" not in line:
+            continue
+        cells = [c.strip() for c in line.strip("|").split("|")]
+        if len(cells) < 6:
+            continue
+        url, auth, https = cells[2], cells[4], cells[5]
+        if "No" not in auth or "http" not in url:
+            continue
+        if any(b in url for b in ("github.com/public-apis", "localhost")):
+            continue
+        rows.append({"name": "dis_" + url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_")[:24],
+                     "url": url, "adapter": "generic_json", "kind": "json",
+                     "tags": ["abuse", "public_api"], "group": "json", "limit": 15})
+    uniq = {}
+    for r in rows:
+        uniq.setdefault(r["name"], r)
+    cands = load(CAND, [])
+    have = {c["name"] for c in cands} | {c["url"] for c in cands}
+    new = [r for r in uniq.values() if r["name"] not in have and r["url"] not in have][:limit]
+    cands.extend(new)
+    save(CAND, cands)
+    print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands)))
+    return new
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--probe", action="store_true")
+    ap.add_argument("--write", action="store_true")
+    ap.add_argument("--write-only", action="store_true", help="сгенерировать из data/source_health.json без сети")
+    ap.add_argument("--discover", action="store_true")
+    ap.add_argument("--add", nargs=2, metavar=("NAME", "URL"))
+    ap.add_argument("--adapter", default="generic_json")
+    ap.add_argument("--pause", type=float, default=2.0)
+    a = ap.parse_args()
+
+    if a.add:
+        cands = load(CAND, [])
+        cands.append({"name": a.add[0], "url": a.add[1], "adapter": a.adapter,
+                      "kind": "json", "tags": ["abuse"], "limit": 20})
+        save(CAND, cands)
+        print("added", a.add[0])
+        return 0
+    if a.discover:
+        discover()
+        return 0
+
+    cands = load(CAND, [])
+    if a.write_only:
+        health = load(HEALTH, {})
+        results = [(c, (health.get(c["name"]) or {}).get("items", 0), [], "") for c in cands]
+        write_module(cands, results)
+        return 0
+    if not cands:
+        print("нет кандидатов в", CAND)
+        return 1
+    results = probe(cands, pause=a.pause)
+    live = sum(1 for _, n, _, _ in results if n)
+    print("\nLIVE %d / %d, items %d" % (live, len(results), sum(n for _, n, _, _ in results)))
+    if a.write:
+        write_module(cands, results)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/check_yield.py

@@ -0,0 +1,31 @@
+"""Проверка yield каждого источника: имя | items | первых 2 заголовка."""
+import sys
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+
+sys.path.insert(0, ".")
+
+from omni import feeds, sources  # noqa: E402
+
+prefix = sys.argv[1] if len(sys.argv) > 1 else "aa_"
+picked = [s for s in sources.SOURCES if s.name.startswith(prefix)]
+print("sources:", len(picked))
+
+
+def run(src):
+    try:
+        items = feeds.fetch(src, limit=min(20, src.limit or 20))
+        return src.name, len(items), [i.title[:60] for i in items[:2]], ""
+    except Exception as exc:
+        return src.name, 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:70])
+
+
+with ThreadPoolExecutor(max_workers=10) as ex:
+    rows = list(ex.map(run, picked))
+
+live = 0
+for name, n, titles, err in sorted(rows, key=lambda r: -r[1]):
+    flag = "LIVE" if n else "DEAD"
+    live += bool(n)
+    print("%-26s %-4s %3d  %s" % (name, flag, n, err or " | ".join(titles)))
+print("LIVE %d / %d, items %d" % (live, len(rows), sum(r[1] for r in rows)))

tools/probe_candidates.py

@@ -0,0 +1,89 @@
+"""Живость кандидатов: статус, размер, число элементов в JSON/ленте."""
+import json
+import sys
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+
+UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
+
+CAND = [
+    ("devto", "https://dev.to/api/articles?per_page=25&top=7"),
+    ("lobsters", "https://lobste.rs/hottest.json"),
+    ("npm", "https://registry.npmjs.org/-/v1/search?text=llm&size=25"),
+    ("crates", "https://crates.io/api/v1/crates?q=llm&per_page=25"),
+    ("dockerhub", "https://hub.docker.com/v2/repositories/library/?page_size=25"),
+    ("stackoverflow", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=openai&site=stackoverflow&pagesize=25"),
+    ("gitlab", "https://gitlab.com/api/v4/projects?sort=desc&order_by=created_at&per_page=25"),
+    ("devpost", "https://devpost.com/api/hackathons?status[]=open"),
+    ("hf_spaces", "https://huggingface.co/api/spaces?sort=likes&limit=25"),
+    ("mastodon", "https://mastodon.social/api/v1/timelines/tag/llm?limit=25"),
+    ("bsky", "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20api&limit=25"),
+    ("reddit_localllama", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25"),
+    ("reddit_chatgptproxies", "https://www.reddit.com/r/ChatGPTProxies/.json?limit=25"),
+    ("reddit_jailbreak", "https://www.reddit.com/r/jailbreak/.json?limit=25"),
+    ("reddit_cybersecurity", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
+    ("reddit_selfhosted", "https://www.reddit.com/r/selfhosted/.json?limit=25"),
+    ("reddit_piracy", "https://www.reddit.com/r/Piracy/.json?limit=25"),
+    ("dis_openai", "https://community.openai.com/latest.json"),
+    ("dis_hf", "https://discuss.huggingface.co/latest.json"),
+    ("dis_cursor", "https://forum.cursor.com/latest.json"),
+    ("dis_python", "https://discuss.python.org/latest.json"),
+    ("dis_fly", "https://community.fly.io/latest.json"),
+    ("dis_obsidian", "https://forum.obsidian.md/latest.json"),
+    ("rss_habr", "https://habr.com/ru/rss/best/daily/?fl=ru"),
+    ("rss_decoder", "https://the-decoder.com/feed/"),
+    ("rss_marktechpost", "https://www.marktechpost.com/feed/"),
+    ("rss_openai", "https://openai.com/news/rss.xml"),
+    ("rss_simonw", "https://simonwillison.net/atom/everything/"),
+    ("rss_hf", "https://huggingface.co/blog/feed.xml"),
+    ("rss_tc_ai", "https://techcrunch.com/category/artificial-intelligence/feed/"),
+    ("tg_ai_newz", "https://t.me/s/ai_newz"),
+    ("tg_seeallo", "https://t.me/s/seeallo"),
+    ("tg_data_secrets", "https://t.me/s/data_secrets"),
+    ("tg_gptblog", "https://t.me/s/gptblog"),
+    ("tg_denissexy", "https://t.me/s/denissexy"),
+    ("gh_trending", "https://github.com/trending?since=daily"),
+]
+
+
+def probe(item):
+    name, url = item
+    req = urllib.request.Request(url, headers={"User-Agent": UA,
+                                               "Accept": "application/json, text/html, application/xml;q=0.9"})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as r:
+            raw = r.read(400000)
+            code = r.status
+    except Exception as exc:
+        return name, str(exc)[:60], 0, 0
+    n = 0
+    try:
+        d = json.loads(raw.decode("utf-8", "replace"))
+        if isinstance(d, list):
+            n = len(d)
+        elif isinstance(d, dict):
+            for k in ("items", "posts", "results", "objects", "crates", "hackathons", "data", "articles", "children"):
+                v = d.get(k)
+                if isinstance(v, list):
+                    n = len(v)
+                    break
+            if not n:
+                n = len([v for v in d.values() if isinstance(v, list)])
+    except Exception:
+        try:
+            text = raw.decode("utf-8", "replace")
+            n = text.count("<item") or text.count("<entry") or text.count("message-text")
+            if not n and "t.me" in url:
+                n = text.count("tgme_widget_message_text")
+        except Exception:
+            n = 0
+    return name, "ok", code, n
+
+
+with ThreadPoolExecutor(max_workers=12) as ex:
+    res = list(ex.map(probe, CAND))
+
+for name, status, code, n in res:
+    print("%-24s %-6s %s n=%s" % (name, code, status, n))
+print("LIVE:", sum(1 for _, s, _, n in res if s == "ok" and n), "of", len(res))