sources: авто-расширение из публичных API без ключей (46 живых), universal generic_json, bsky/mastodon, --only по префиксу, auto_cycle

tools/probe_candidates.py

@@ -0,0 +1,89 @@
+"""Живость кандидатов: статус, размер, число элементов в JSON/ленте."""
+import json
+import sys
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+
+UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
+
+CAND = [
+    ("devto", "https://dev.to/api/articles?per_page=25&top=7"),
+    ("lobsters", "https://lobste.rs/hottest.json"),
+    ("npm", "https://registry.npmjs.org/-/v1/search?text=llm&size=25"),
+    ("crates", "https://crates.io/api/v1/crates?q=llm&per_page=25"),
+    ("dockerhub", "https://hub.docker.com/v2/repositories/library/?page_size=25"),
+    ("stackoverflow", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=openai&site=stackoverflow&pagesize=25"),
+    ("gitlab", "https://gitlab.com/api/v4/projects?sort=desc&order_by=created_at&per_page=25"),
+    ("devpost", "https://devpost.com/api/hackathons?status[]=open"),
+    ("hf_spaces", "https://huggingface.co/api/spaces?sort=likes&limit=25"),
+    ("mastodon", "https://mastodon.social/api/v1/timelines/tag/llm?limit=25"),
+    ("bsky", "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20api&limit=25"),
+    ("reddit_localllama", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25"),
+    ("reddit_chatgptproxies", "https://www.reddit.com/r/ChatGPTProxies/.json?limit=25"),
+    ("reddit_jailbreak", "https://www.reddit.com/r/jailbreak/.json?limit=25"),
+    ("reddit_cybersecurity", "https://www.reddit.com/r/cybersecurity/.json?limit=25"),
+    ("reddit_selfhosted", "https://www.reddit.com/r/selfhosted/.json?limit=25"),
+    ("reddit_piracy", "https://www.reddit.com/r/Piracy/.json?limit=25"),
+    ("dis_openai", "https://community.openai.com/latest.json"),
+    ("dis_hf", "https://discuss.huggingface.co/latest.json"),
+    ("dis_cursor", "https://forum.cursor.com/latest.json"),
+    ("dis_python", "https://discuss.python.org/latest.json"),
+    ("dis_fly", "https://community.fly.io/latest.json"),
+    ("dis_obsidian", "https://forum.obsidian.md/latest.json"),
+    ("rss_habr", "https://habr.com/ru/rss/best/daily/?fl=ru"),
+    ("rss_decoder", "https://the-decoder.com/feed/"),
+    ("rss_marktechpost", "https://www.marktechpost.com/feed/"),
+    ("rss_openai", "https://openai.com/news/rss.xml"),
+    ("rss_simonw", "https://simonwillison.net/atom/everything/"),
+    ("rss_hf", "https://huggingface.co/blog/feed.xml"),
+    ("rss_tc_ai", "https://techcrunch.com/category/artificial-intelligence/feed/"),
+    ("tg_ai_newz", "https://t.me/s/ai_newz"),
+    ("tg_seeallo", "https://t.me/s/seeallo"),
+    ("tg_data_secrets", "https://t.me/s/data_secrets"),
+    ("tg_gptblog", "https://t.me/s/gptblog"),
+    ("tg_denissexy", "https://t.me/s/denissexy"),
+    ("gh_trending", "https://github.com/trending?since=daily"),
+]
+
+
+def probe(item):
+    name, url = item
+    req = urllib.request.Request(url, headers={"User-Agent": UA,
+                                               "Accept": "application/json, text/html, application/xml;q=0.9"})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as r:
+            raw = r.read(400000)
+            code = r.status
+    except Exception as exc:
+        return name, str(exc)[:60], 0, 0
+    n = 0
+    try:
+        d = json.loads(raw.decode("utf-8", "replace"))
+        if isinstance(d, list):
+            n = len(d)
+        elif isinstance(d, dict):
+            for k in ("items", "posts", "results", "objects", "crates", "hackathons", "data", "articles", "children"):
+                v = d.get(k)
+                if isinstance(v, list):
+                    n = len(v)
+                    break
+            if not n:
+                n = len([v for v in d.values() if isinstance(v, list)])
+    except Exception:
+        try:
+            text = raw.decode("utf-8", "replace")
+            n = text.count("<item") or text.count("<entry") or text.count("message-text")
+            if not n and "t.me" in url:
+                n = text.count("tgme_widget_message_text")
+        except Exception:
+            n = 0
+    return name, "ok", code, n
+
+
+with ThreadPoolExecutor(max_workers=12) as ex:
+    res = list(ex.map(probe, CAND))
+
+for name, status, code, n in res:
+    print("%-24s %-6s %s n=%s" % (name, code, status, n))
+print("LIVE:", sum(1 for _, s, _, n in res if s == "ok" and n), "of", len(res))