sources: авто-расширение из публичных API без ключей (46 живых), universal generic_json, bsky/mastodon, --only по префиксу, auto_cycle

tools/auto_expand.py

@@ -0,0 +1,218 @@
+"""Авто-расширение источников: тест кандидатов → генерация omni/sources_auto.py.
+
+    python tools/auto_expand.py --probe         # только проверка, без записи
+    python tools/auto_expand.py --write         # живые кандидаты → sources_auto.py
+    python tools/auto_expand.py --discover      # добрать кандидатов из public-apis
+    python tools/auto_expand.py --add NAME URL --adapter rss
+
+Кандидаты: data/source_candidates.json (list of {name,url,adapter,kind,extra,tags,limit}).
+Здоровье: data/source_health.json — сколько раз источник отдал items и когда.
+Мёртвым считается кандидат с 0 items в 3 прогонах подряд → исключается из генерации.
+"""
+import argparse
+import json
+import sys
+import time
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from omni import feeds  # noqa: E402
+from omni.sources import Source  # noqa: E402
+
+CAND = ROOT / "data" / "source_candidates.json"
+HEALTH = ROOT / "data" / "source_health.json"
+OUT = ROOT / "omni" / "sources_auto.py"
+PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md"
+
+HEADER = '''"""Авто-расширение: источники на публичных API без ключа.
+
+Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
+(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.
+
+Обновлено: {ts} · живых {live} из {total}
+"""
+
+
+def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None):
+    return {{"name": name, "category": "abuse", "kind": kind, "url": url,
+            "adapter": adapter, "tags": tuple(tags), "limit": limit,
+            "detail": False, "extra": dict(extra or {{}})}}
+
+
+AUTO = [
+{body}]
+'''
+
+GROUP_RU = {
+    "json": "# ---------- открытые JSON API без ключа ----------",
+    "reddit": "# ---------- Reddit через pullpush.io (без ключа) ----------",
+    "social": "# ---------- Bluesky / Mastodon ----------",
+    "discourse": "# ---------- Discourse-сообщества ----------",
+    "rss": "# ---------- RSS/Atom ----------",
+    "tg": "# ---------- Telegram-превью ----------",
+}
+
+
+def load(path, default):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return default
+
+
+def save(path, data):
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
+
+
+def to_source(c):
+    return Source(name=c["name"], category=c.get("category", "abuse"), kind=c.get("kind", "json"),
+                  url=c["url"], adapter=c["adapter"], tags=tuple(c.get("tags") or ("abuse",)),
+                  limit=int(c.get("limit", 20)), detail=False, extra=dict(c.get("extra") or {}))
+
+
+def probe(cands, pause=2.0):
+    health = load(HEALTH, {})
+    res = []
+    last_host = None
+    for c in cands:
+        host = c["url"].split("/")[2] if "//" in c["url"] else ""
+        if host == last_host and pause:
+            time.sleep(pause)
+        last_host = host
+        try:
+            items = feeds.fetch(to_source(c), limit=min(20, int(c.get("limit", 20))))
+            n = len(items)
+            sample = [i.title[:70] for i in items[:2]]
+            err = ""
+        except Exception as exc:
+            n, sample, err = 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:90])
+        h = health.setdefault(c["name"], {"ok": 0, "fail": 0, "items": 0})
+        h["items"] = n
+        h["ok"] = h.get("ok", 0) + 1 if n else 0
+        h["fail"] = 0 if n else h.get("fail", 0) + 1
+        h["last"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M")
+        h["url"] = c["url"]
+        if err:
+            h["error"] = err
+        else:
+            h.pop("error", None)
+        res.append((c, n, sample, err))
+        print("%-26s %-4s %3d  %s" % (c["name"], "LIVE" if n else "DEAD", n, err or " | ".join(sample)))
+    save(HEALTH, health)
+    return res
+
+
+def write_module(cands, results):
+    health = load(HEALTH, {})
+    order = ["json", "reddit", "social", "discourse", "rss", "tg"]
+    buckets = {k: [] for k in order}
+    for c, n, _, _ in results:
+        if not n:
+            continue
+        if (health.get(c["name"]) or {}).get("fail", 0) >= 3:
+            print("skip (3 fails подряд):", c["name"])
+            continue
+        g = c.get("group") or ("tg" if c["adapter"] == "tg_web" else
+                               "discourse" if c["adapter"] == "discourse" else
+                               "reddit" if "pullpush" in c["url"] or "reddit" in c["url"] else
+                               "social" if c["adapter"] in ("bsky_search", "mastodon_tag") else
+                               "rss" if c["adapter"] == "rss" else "json")
+        buckets.setdefault(g, []).append(c)
+    body = []
+    live = 0
+    for g in order + [k for k in buckets if k not in order]:
+        rows = buckets.get(g) or []
+        if not rows:
+            continue
+        body.append("    " + GROUP_RU.get(g, "# ---------- прочее ----------"))
+        for c in rows:
+            live += 1
+            extra = ""
+            if c.get("extra"):
+                extra = ", extra=%s" % json.dumps(c["extra"], ensure_ascii=False)
+            tags = ", ".join('"%s"' % t for t in (c.get("tags") or ("abuse",)))
+            body.append('    A("%s", "%s", "%s", "%s", (%s)%s),' % (
+                c["name"], c.get("kind", "json"), c["url"], c["adapter"], tags, extra))
+        body.append("")
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    OUT.write_text(HEADER.format(ts=ts, live=live, total=len(cands), body="\n".join(body) + "\n"),
+                   encoding="utf-8")
+    print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands)))
+
+
+def discover(limit=40):
+    """Добрать бесплатные (Auth=No) API из public-apis."""
+    req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"})
+    md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace")
+    rows = []
+    for line in md.splitlines():
+        if not line.startswith("|") or "http" not in line:
+            continue
+        cells = [c.strip() for c in line.strip("|").split("|")]
+        if len(cells) < 6:
+            continue
+        url, auth, https = cells[2], cells[4], cells[5]
+        if "No" not in auth or "http" not in url:
+            continue
+        if any(b in url for b in ("github.com/public-apis", "localhost")):
+            continue
+        rows.append({"name": "dis_" + url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_")[:24],
+                     "url": url, "adapter": "generic_json", "kind": "json",
+                     "tags": ["abuse", "public_api"], "group": "json", "limit": 15})
+    uniq = {}
+    for r in rows:
+        uniq.setdefault(r["name"], r)
+    cands = load(CAND, [])
+    have = {c["name"] for c in cands} | {c["url"] for c in cands}
+    new = [r for r in uniq.values() if r["name"] not in have and r["url"] not in have][:limit]
+    cands.extend(new)
+    save(CAND, cands)
+    print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands)))
+    return new
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--probe", action="store_true")
+    ap.add_argument("--write", action="store_true")
+    ap.add_argument("--write-only", action="store_true", help="сгенерировать из data/source_health.json без сети")
+    ap.add_argument("--discover", action="store_true")
+    ap.add_argument("--add", nargs=2, metavar=("NAME", "URL"))
+    ap.add_argument("--adapter", default="generic_json")
+    ap.add_argument("--pause", type=float, default=2.0)
+    a = ap.parse_args()
+
+    if a.add:
+        cands = load(CAND, [])
+        cands.append({"name": a.add[0], "url": a.add[1], "adapter": a.adapter,
+                      "kind": "json", "tags": ["abuse"], "limit": 20})
+        save(CAND, cands)
+        print("added", a.add[0])
+        return 0
+    if a.discover:
+        discover()
+        return 0
+
+    cands = load(CAND, [])
+    if a.write_only:
+        health = load(HEALTH, {})
+        results = [(c, (health.get(c["name"]) or {}).get("items", 0), [], "") for c in cands]
+        write_module(cands, results)
+        return 0
+    if not cands:
+        print("нет кандидатов в", CAND)
+        return 1
+    results = probe(cands, pause=a.pause)
+    live = sum(1 for _, n, _, _ in results if n)
+    print("\nLIVE %d / %d, items %d" % (live, len(results), sum(n for _, n, _, _ in results)))
+    if a.write:
+        write_module(cands, results)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())