sources: авто-расширение из публичных API без ключей (46 живых), universal generic_json, bsky/mastodon, --only по префиксу, auto_cycle

omni/feeds.py

@@ -484,41 +484,123 @@ def _json_of(url, timeout=35):
 
 
 def _generic_json(src, limit):
-    """Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам."""
+    """Любой JSON-эндпоинт: список объектов (+extra: rows/title_key/url_key/text_key/date_key/base/slug_key/deref)."""
     d = _json_of(src.url)
+    ex = src.extra or {}
     rows = []
     if isinstance(d, list):
         rows = d
     elif isinstance(d, dict):
-        for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"):
-            v = d.get(key)
-            if isinstance(v, list):
-                rows = v
-                break
+        path = ex.get("rows")
+        if path:
+            cur = d
+            for part in str(path).split("."):
+                cur = cur.get(part) if isinstance(cur, dict) else None
+            if isinstance(cur, list):
+                rows = cur
+        if not rows:
+            for key in ("data", "items", "results", "papers", "models", "hits", "notes",
+                        "posts", "records", "objects", "crates", "hackathons"):
+                v = d.get(key)
+                if isinstance(v, list):
+                    rows = v
+                    break
         if not rows:
             for v in d.values():
                 if isinstance(v, list) and v and isinstance(v[0], dict):
                     rows = v
                     break
     out = []
+    deref = ex.get("deref")
+    base = str(ex.get("base") or "")
+    title_keys = ([ex["title_key"]] if ex.get("title_key") else []) + \
+                 ["title", "name", "paper_title", "full_name", "id"]
+    url_keys = ([ex["url_key"]] if ex.get("url_key") else []) + \
+               ["url", "link", "html_url", "paper_url", "web_url"]
+    text_keys = ([ex["text_key"]] if ex.get("text_key") else []) + \
+                ["abstract", "summary", "description", "body", "content"]
+    date_keys = ([ex["date_key"]] if ex.get("date_key") else []) + \
+                ["published", "created", "created_at", "date", "published_at"]
     for r in rows[:limit]:
         if not isinstance(r, dict):
             continue
-        title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id")
-        url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id")
+        if deref and isinstance(r.get(deref), dict):
+            r = r[deref]
+        title = ""
+        for k in title_keys:
+            v = r.get(k)
+            if isinstance(v, str) and v.strip():
+                title = v.strip()
+                break
+        url = ""
+        for k in url_keys:
+            v = r.get(k)
+            if isinstance(v, str) and v.strip():
+                url = v.strip()
+                break
+        if base and (not url or not url.startswith("http")):
+            slug = ""
+            if ex.get("slug_key"):
+                slug = str(r.get(ex["slug_key"]) or "")
+            url = base.rstrip("/") + "/" + (url or slug).lstrip("/")
         if isinstance(url, str) and url and not url.startswith("http"):
             if "github.com" in src.url:
                 url = "https://github.com/" + url.lstrip("/")
             elif "huggingface.co" in src.url:
                 url = "https://huggingface.co/" + url.lstrip("/")
-        text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or ""
-        if isinstance(text, dict):
-            text = json.dumps(text, ensure_ascii=False)[:400]
-        published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or ""
+        text = ""
+        for k in text_keys:
+            v = r.get(k)
+            if isinstance(v, str) and v.strip():
+                text = v
+                break
+            if isinstance(v, (dict, list)):
+                text = json.dumps(v, ensure_ascii=False)[:400]
+                break
+        published = ""
+        for k in date_keys:
+            v = r.get(k)
+            if isinstance(v, str) and v.strip():
+                published = v.strip()
+                break
         if not title:
             continue
-        out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800],
-                         str(r.get("author") or r.get("owner") or ""), str(published)[:24]))
+        out.append(_item(src, "entry", str(url or ""), _txt(title)[:180], _txt(text)[:800],
+                         str(r.get("author") or r.get("owner") or r.get("by") or ""),
+                         str(published)[:24]))
+    return out
+
+
+def _bsky_search(src, limit):
+    """Bluesky: публичный поиск постов без ключа (api.bsky.app)."""
+    d = _json_of(src.url)
+    out = []
+    for p in (d.get("posts") or [])[:limit]:
+        if not isinstance(p, dict):
+            continue
+        rec = p.get("record") or {}
+        txt = _txt(str(rec.get("text") or ""))
+        author = str((p.get("author") or {}).get("handle") or "")
+        uri = str(p.get("uri") or "")
+        rkey = uri.rsplit("/", 1)[-1] if uri else ""
+        url = "https://bsky.app/profile/%s/post/%s" % (author, rkey) if author and rkey else uri
+        out.append(_item(src, "post", url, txt[:150], txt[:800], author,
+                         str(rec.get("createdAt") or "")[:24]))
+    return out
+
+
+def _mastodon_tag(src, limit):
+    """Mastodon: публичная лента по тегу, без токена."""
+    d = _json_of(src.url)
+    rows = d if isinstance(d, list) else []
+    out = []
+    for r in rows[:limit]:
+        if not isinstance(r, dict):
+            continue
+        txt = _txt(str(r.get("content") or ""))
+        out.append(_item(src, "post", str(r.get("url") or ""), txt[:150], txt[:800],
+                         str((r.get("account") or {}).get("acct") or ""),
+                         str(r.get("created_at") or "")[:24]))
     return out
 
 
@@ -840,6 +922,9 @@ def _github_search(src, limit):
            + "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50))))
     hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"}
     tok = ((config.CONFIG.get("keys") or {}).get("github") or [])
+    if not tok:
+        single = config.KEYS.get("GITHUB_TOKEN")
+        tok = [single] if single else []
     if isinstance(tok, list) and tok:
         hdr["Authorization"] = "Bearer " + str(tok[0])
     r = net.get(url, headers=hdr)
@@ -917,6 +1002,8 @@ ADAPTERS = {
     "newapi_pricing": _newapi_pricing,
     "github_search": _github_search,
     "hn_search": _hn_search,
+    "bsky_search": _bsky_search,
+    "mastodon_tag": _mastodon_tag,
 
 }
 

omni/sources.py

@@ -21,6 +21,7 @@ from .sources_ext5 import EXTRA6
 from .sources_fresh import fresh as _fresh_sources
 from .sources_quality import quality_sources as _quality_sources
 from .sources_x import X_SOURCES
+from .sources_auto import AUTO as AUTO_SOURCES
 
 
 @dataclass
@@ -204,7 +205,7 @@ def _dedup(rows):
     return out
 
 
-SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES])
+SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES] + [Source(**d) for d in AUTO_SOURCES])
 
 BY_NAME = {s.name: s for s in SOURCES}
 CATEGORIES = sorted({s.category for s in SOURCES})
@@ -365,7 +366,18 @@ def pick(category=None, only=None, tags=None, respect_enabled=True):
         out = [s for s in out if s.category in cats]
     if only:
         names = {n.strip() for n in only.split(",") if n.strip()}
-        out = [s for s in out if s.name in names]
+
+        def _hit(src):
+            if src.name in names:
+                return True
+            for n in names:
+                if n.endswith("*") and src.name.startswith(n[:-1]):
+                    return True
+                if n.endswith("_") and src.name.startswith(n):
+                    return True
+            return False
+
+        out = [x for x in out if _hit(x)]
     if tags:
         wanted = {t.strip() for t in tags.split(",") if t.strip()}
         out = [s for s in out if wanted & set(s.tags)]

omni/sources_auto.py

@@ -0,0 +1,75 @@
+"""Авто-расширение: источники на публичных API без ключа.
+
+Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
+(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.
+
+Обновлено: 2026-10-27 16:41 UTC · живых 46 из 52
+"""
+
+
+def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None):
+    return {"name": name, "category": "abuse", "kind": kind, "url": url,
+            "adapter": adapter, "tags": tuple(tags), "limit": limit,
+            "detail": False, "extra": dict(extra or {})}
+
+
+AUTO = [
+    # ---------- открытые JSON API без ключа ----------
+    A("aa_devto_ai", "json", "https://dev.to/api/articles?per_page=25&top=7&tag=ai", "generic_json", ("abuse", "devto"), extra={"date_key": "published_at"}),
+    A("aa_lobsters", "json", "https://lobste.rs/hottest.json", "generic_json", ("abuse")),
+    A("aa_npm_llm", "json", "https://registry.npmjs.org/-/v1/search?text=llm%20api&size=25", "generic_json", ("abuse", "npm"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}),
+    A("aa_npm_mcp", "json", "https://registry.npmjs.org/-/v1/search?text=mcp%20server&size=25", "generic_json", ("abuse", "npm"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}),
+    A("aa_crates_llm", "json", "https://crates.io/api/v1/crates?q=llm&per_page=25", "generic_json", ("abuse", "crates"), extra={"rows": "crates", "base": "https://crates.io/crates/", "slug_key": "name"}),
+    A("aa_dockerhub_llm", "json", "https://hub.docker.com/v2/repositories/library/?page_size=25", "generic_json", ("abuse", "docker"), extra={"rows": "results", "base": "https://hub.docker.com/r/library/", "slug_key": "name"}),
+    A("aa_stack_openai", "json", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=openai&site=stackoverflow&pagesize=25", "generic_json", ("abuse", "stack"), extra={"rows": "items", "url_key": "link", "date_key": "creation_date"}),
+    A("aa_stack_llm", "json", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=llm&site=stackoverflow&pagesize=25", "generic_json", ("abuse", "stack"), extra={"rows": "items", "url_key": "link", "date_key": "creation_date"}),
+    A("aa_gitlab_new", "json", "https://gitlab.com/api/v4/projects?sort=desc&order_by=created_at&per_page=25", "generic_json", ("abuse", "gitlab"), extra={"url_key": "web_url"}),
+    A("aa_devpost", "json", "https://devpost.com/api/hackathons?status[]=open", "generic_json", ("abuse", "hackathon"), extra={"rows": "hackathons", "url_key": "url"}),
+    A("aa_hf_spaces_likes", "json", "https://huggingface.co/api/spaces?sort=likes&limit=25", "generic_json", ("abuse", "hf"), extra={"base": "https://huggingface.co/spaces/", "slug_key": "id", "title_key": "id"}),
+    A("aa_hf_spaces_trending", "json", "https://huggingface.co/api/spaces?sort=trendingScore&limit=25", "generic_json", ("abuse", "hf"), extra={"base": "https://huggingface.co/spaces/", "slug_key": "id", "title_key": "id"}),
+    A("aa_hf_models_trending", "json", "https://huggingface.co/api/models?sort=trendingScore&limit=25", "generic_json", ("abuse"), extra={"base": "https://huggingface.co/", "slug_key": "id", "title_key": "id"}),
+    A("aa_gitlab_llm", "json", "https://gitlab.com/api/v4/projects?search=llm&sort=desc&order_by=created_at&per_page=25", "generic_json", ("abuse"), extra={"url_key": "web_url"}),
+    A("aa_npm_free_api", "json", "https://registry.npmjs.org/-/v1/search?text=free%20api%20keys&size=25", "generic_json", ("abuse"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}),
+    A("aa_crates_proxy", "json", "https://crates.io/api/v1/crates?q=openai%20proxy&per_page=20", "generic_json", ("abuse"), extra={"rows": "crates", "base": "https://crates.io/crates/", "slug_key": "name"}),
+
+    # ---------- Reddit через pullpush.io (без ключа) ----------
+    A("aa_pp_freeai", "json", "https://api.pullpush.io/reddit/search/submission/?subreddit=FreeAI&size=25", "generic_json", ("abuse"), extra={"rows": "data", "url_key": "url", "title_key": "title", "text_key": "selftext", "date_key": "created_utc"}),
+    A("aa_pp_openaiapi", "json", "https://api.pullpush.io/reddit/search/submission/?subreddit=OpenAIAPI&size=25", "generic_json", ("abuse"), extra={"rows": "data", "url_key": "url", "title_key": "title", "text_key": "selftext", "date_key": "created_utc"}),
+
+    # ---------- Bluesky / Mastodon ----------
+    A("aa_bsky_free_api", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20api%20key&limit=25", "bsky_search", ("abuse", "bsky")),
+    A("aa_bsky_claude_api", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=claude%20api&limit=25", "bsky_search", ("abuse", "bsky")),
+    A("aa_bsky_jailbreak", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=jailbreak%20llm&limit=25", "bsky_search", ("abuse", "bsky", "jb")),
+    A("aa_masto_llm", "json", "https://mastodon.social/api/v1/timelines/tag/llm?limit=25", "mastodon_tag", ("abuse", "mastodon")),
+    A("aa_masto_jailbreak", "json", "https://mastodon.social/api/v1/timelines/tag/jailbreak?limit=25", "mastodon_tag", ("abuse", "mastodon", "jb")),
+    A("aa_masto_ai", "json", "https://mastodon.social/api/v1/timelines/tag/openai?limit=25", "mastodon_tag", ("abuse")),
+    A("aa_bsky_free_tokens", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20tokens%20api&limit=25", "bsky_search", ("abuse")),
+    A("aa_bsky_openrouter", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=openrouter&limit=25", "bsky_search", ("abuse")),
+
+    # ---------- Discourse-сообщества ----------
+    A("aa_dis_openai", "json", "https://community.openai.com/latest.json", "discourse", ("abuse")),
+    A("aa_dis_hf", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("abuse")),
+    A("aa_dis_cursor", "json", "https://forum.cursor.com/latest.json", "discourse", ("abuse")),
+    A("aa_dis_python", "json", "https://discuss.python.org/latest.json", "discourse", ("abuse")),
+    A("aa_dis_fly", "json", "https://community.fly.io/latest.json", "discourse", ("abuse")),
+    A("aa_dis_obsidian", "json", "https://forum.obsidian.md/latest.json", "discourse", ("abuse")),
+
+    # ---------- RSS/Atom ----------
+    A("aa_rss_habr", "rss", "https://habr.com/ru/rss/best/daily/?fl=ru", "rss", ("abuse")),
+    A("aa_rss_decoder", "rss", "https://the-decoder.com/feed/", "rss", ("abuse")),
+    A("aa_rss_marktechpost", "rss", "https://www.marktechpost.com/feed/", "rss", ("abuse")),
+    A("aa_rss_openai_news", "rss", "https://openai.com/news/rss.xml", "rss", ("abuse")),
+    A("aa_rss_simonw", "rss", "https://simonwillison.net/atom/everything/", "rss", ("abuse")),
+    A("aa_rss_hf_blog", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("abuse")),
+    A("aa_rss_tc_ai", "rss", "https://techcrunch.com/category/artificial-intelligence/feed/", "rss", ("abuse")),
+    A("aa_rss_52pojie", "rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", ("abuse")),
+
+    # ---------- Telegram-превью ----------
+    A("aa_tg_ai_newz", "html", "https://t.me/s/ai_newz", "tg_web", ("abuse", "tg")),
+    A("aa_tg_data_secrets", "html", "https://t.me/s/data_secrets", "tg_web", ("abuse", "tg")),
+    A("aa_tg_denissexy", "html", "https://t.me/s/denissexy", "tg_web", ("abuse", "tg")),
+    A("aa_tg_neurohive", "json", "https://t.me/s/neurohive", "tg_web", ("abuse")),
+    A("aa_tg_gpt4telegram", "json", "https://t.me/s/gpt4telegram", "tg_web", ("abuse")),
+    A("aa_tg_dataleak", "json", "https://t.me/s/dataleak", "tg_web", ("abuse")),
+
+]