diff --git a/README.md b/README.md index 7fe1e2a..1ccd1ee 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ python -m omni export --format jsonl --out data/dump.jsonl ## Реестр источников -`omni/sources.py` + `omni/sources_ext.py` — **589 источников в 9 категориях**, у каждого свой адаптер и флаг вкл/выкл: +`omni/sources.py` + `omni/sources_ext.py` + `omni/sources_auto.py` — **1072 источника в 11 категориях**, у каждого свой адаптер и флаг вкл/выкл. Из них **467 abuse** (включая 46 авто-подключённых `aa_`) и **33 в активной авто-группе `aa_`** (остальные 13 авто-кандидатов совпали по url с уже существующими источниками и были схлопнуты дедупом): - **papers 49** — arxiv по 25+ разделам (cs.AI/LG/CL/CV/RO/IR/CR/SE/NE/MA/DC/DB/HC/SD/CY, stat.ML, eess.AS/SP/IV, math.OC, q-bio.NC, cs.GR, cs.AR, cs.PF), HF daily papers, OpenAlex, Semantic Scholar, Papers with Code, Connected Papers - **company 95** — ленты OpenAI, Google AI/Research/DeepMind, Microsoft Research, NVIDIA, Apple ML, Amazon Science, HF, Meta AI, Mistral, Cohere, Together, Perplexity, Groq, Cerebras, Stability, Databricks, Anyscale, Modal, W&B, LangChain, LlamaIndex, vLLM, Replicate, Fireworks, SambaNova, Scale, xAI, Salesforce, AWS ML, Azure AI, Anthropic, AI2, EleutherAI, LAION, Runway, ElevenLabs, Deepgram, AssemblyAI, Pinecone, Weaviate, Qdrant, Chroma, Unsloth, Axolotl @@ -212,6 +212,37 @@ python -m omni export --format jsonl --out data/dump.jsonl Адаптеры: `rss`, `generic_json`, `or_models`, `cisa_kev`, `nvd`, `json` (HF, HN, OpenAlex, S2, Discourse), `html` (список + детальная), `github_org`. Медленные хосты (reddit, github, HF, arxiv, openrouter) обходятся последовательно с паузой, остальные — в 8 потоков. Для arxiv RSS, который часто отдаёт пустое, есть fallback на официальный API (`export.arxiv.org/api/query`). +## Авто-расширение источников (публичные API без ключей) + +`omni/sources_auto.py` **генерируется** `tools/auto_expand.py` — руками не править, правки перезапишутся. Пул кандидатов лежит в `data/source_candidates.json`, здоровье — в `data/source_health.json`. + +``` +python tools/auto_expand.py --probe # проверить всех: имя | LIVE/DEAD | items | заголовки +python tools/auto_expand.py --write # probe + пересобрать omni/sources_auto.py (живые, items>0) +python tools/auto_expand.py --write-only # пересобрать из последнего health (без сети) +python tools/auto_expand.py --discover # добрать бесплатные (Auth=No) эндпоинты из public-apis +python tools/auto_expand.py --add NAME URL --adapter rss # новый кандидат одной строкой +python tools/auto_cycle.py # полный цикл: probe → write → feed → индекс → коммит/пуш +``` + +Правила: кандидат попадает в реестр только если адаптер реально вернул `items > 0`; после 3 прогонов подряд с нулём источник выкидывается из генерации автоматически. Группы в сгенерированном файле: `json`, `reddit`, `social`, `discourse`, `rss`, `tg`. + +Универсальный `generic_json` понимает описание в `extra`: `rows` (путь к списку через точку), `title_key`, `url_key`, `text_key`, `date_key`, `base` + `slug_key` (собрать ссылку из слага), `deref` (спуститься в подобъект). Поэтому большинство новых JSON-источников — одна строка в `data/source_candidates.json`, без нового кода. Отдельные адаптеры только там, где схема нестандартная: `bsky_search` (Bluesky public API, без токена), `mastodon_tag`, `tg_web` (`t.me/s/`), `discourse` (`latest.json`). + +Подключено в этом апдейте (46 живых из 52 проверенных, `пусто 0/33 | собрано 458, новых 408`): + +- **JSON без ключа** — dev.to, Lobsters, npm search (llm / mcp / free api keys), crates.io, Docker Hub, Stack Overflow (openai / prompt-injection), GitLab (new/llm), Hugging Face spaces + models trending +- **Reddit** — через `api.pullpush.io` (архив Reddit, без OAuth): LocalLLaMA, FreeAI, OpenAIAPI, ChatGPT, selfhosted +- **Bluesky** — `public.api.bsky.app` searchPosts: free tokens api, openrouter, free api, ai keys, llm api +- **Mastodon** — теги ai / openai / llm +- **Discourse** — community.openai, discuss.huggingface, forum.cursor, discuss.python, discuss.fly, forum.obsidian +- **RSS** — Habr, The Decoder, MarkTechPost, OpenAI news, Simon Willison, HF blog, TechCrunch AI, 52pojie +- **Telegram-превью** — ai_newz, data_secrets, neurohive, gpt4telegram, dataleak, denissexy + +`--only` в `omni feed` понимает префикс: `python -m omni feed --only aa_` берёт всю авто-группу. Для `github_search` поднимается лимит, если в `.env` есть `GITHUB_TOKEN` (анонимно 10 запросов/мин, с токеном 30). + +Крон (каждые 6 часов, `tools/auto_cycle.py`): проверка кандидатов → пересборка `sources_auto.py` → докачка только при изменениях → пересборка abuse-индекса → коммит `auto: источники N live` и пуш в gitea/github. Лог — `data/auto_cycle.log`. + ## Панель и команды ``` diff --git a/omni/feeds.py b/omni/feeds.py index bcbc7ae..0817c48 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -484,41 +484,123 @@ def _json_of(url, timeout=35): def _generic_json(src, limit): - """Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам.""" + """Любой JSON-эндпоинт: список объектов (+extra: rows/title_key/url_key/text_key/date_key/base/slug_key/deref).""" d = _json_of(src.url) + ex = src.extra or {} rows = [] if isinstance(d, list): rows = d elif isinstance(d, dict): - for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"): - v = d.get(key) - if isinstance(v, list): - rows = v - break + path = ex.get("rows") + if path: + cur = d + for part in str(path).split("."): + cur = cur.get(part) if isinstance(cur, dict) else None + if isinstance(cur, list): + rows = cur + if not rows: + for key in ("data", "items", "results", "papers", "models", "hits", "notes", + "posts", "records", "objects", "crates", "hackathons"): + v = d.get(key) + if isinstance(v, list): + rows = v + break if not rows: for v in d.values(): if isinstance(v, list) and v and isinstance(v[0], dict): rows = v break out = [] + deref = ex.get("deref") + base = str(ex.get("base") or "") + title_keys = ([ex["title_key"]] if ex.get("title_key") else []) + \ + ["title", "name", "paper_title", "full_name", "id"] + url_keys = ([ex["url_key"]] if ex.get("url_key") else []) + \ + ["url", "link", "html_url", "paper_url", "web_url"] + text_keys = ([ex["text_key"]] if ex.get("text_key") else []) + \ + ["abstract", "summary", "description", "body", "content"] + date_keys = ([ex["date_key"]] if ex.get("date_key") else []) + \ + ["published", "created", "created_at", "date", "published_at"] for r in rows[:limit]: if not isinstance(r, dict): continue - title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id") - url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id") + if deref and isinstance(r.get(deref), dict): + r = r[deref] + title = "" + for k in title_keys: + v = r.get(k) + if isinstance(v, str) and v.strip(): + title = v.strip() + break + url = "" + for k in url_keys: + v = r.get(k) + if isinstance(v, str) and v.strip(): + url = v.strip() + break + if base and (not url or not url.startswith("http")): + slug = "" + if ex.get("slug_key"): + slug = str(r.get(ex["slug_key"]) or "") + url = base.rstrip("/") + "/" + (url or slug).lstrip("/") if isinstance(url, str) and url and not url.startswith("http"): if "github.com" in src.url: url = "https://github.com/" + url.lstrip("/") elif "huggingface.co" in src.url: url = "https://huggingface.co/" + url.lstrip("/") - text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or "" - if isinstance(text, dict): - text = json.dumps(text, ensure_ascii=False)[:400] - published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or "" + text = "" + for k in text_keys: + v = r.get(k) + if isinstance(v, str) and v.strip(): + text = v + break + if isinstance(v, (dict, list)): + text = json.dumps(v, ensure_ascii=False)[:400] + break + published = "" + for k in date_keys: + v = r.get(k) + if isinstance(v, str) and v.strip(): + published = v.strip() + break if not title: continue - out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800], - str(r.get("author") or r.get("owner") or ""), str(published)[:24])) + out.append(_item(src, "entry", str(url or ""), _txt(title)[:180], _txt(text)[:800], + str(r.get("author") or r.get("owner") or r.get("by") or ""), + str(published)[:24])) + return out + + +def _bsky_search(src, limit): + """Bluesky: публичный поиск постов без ключа (api.bsky.app).""" + d = _json_of(src.url) + out = [] + for p in (d.get("posts") or [])[:limit]: + if not isinstance(p, dict): + continue + rec = p.get("record") or {} + txt = _txt(str(rec.get("text") or "")) + author = str((p.get("author") or {}).get("handle") or "") + uri = str(p.get("uri") or "") + rkey = uri.rsplit("/", 1)[-1] if uri else "" + url = "https://bsky.app/profile/%s/post/%s" % (author, rkey) if author and rkey else uri + out.append(_item(src, "post", url, txt[:150], txt[:800], author, + str(rec.get("createdAt") or "")[:24])) + return out + + +def _mastodon_tag(src, limit): + """Mastodon: публичная лента по тегу, без токена.""" + d = _json_of(src.url) + rows = d if isinstance(d, list) else [] + out = [] + for r in rows[:limit]: + if not isinstance(r, dict): + continue + txt = _txt(str(r.get("content") or "")) + out.append(_item(src, "post", str(r.get("url") or ""), txt[:150], txt[:800], + str((r.get("account") or {}).get("acct") or ""), + str(r.get("created_at") or "")[:24])) return out @@ -840,6 +922,9 @@ def _github_search(src, limit): + "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50)))) hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"} tok = ((config.CONFIG.get("keys") or {}).get("github") or []) + if not tok: + single = config.KEYS.get("GITHUB_TOKEN") + tok = [single] if single else [] if isinstance(tok, list) and tok: hdr["Authorization"] = "Bearer " + str(tok[0]) r = net.get(url, headers=hdr) @@ -917,6 +1002,8 @@ ADAPTERS = { "newapi_pricing": _newapi_pricing, "github_search": _github_search, "hn_search": _hn_search, + "bsky_search": _bsky_search, + "mastodon_tag": _mastodon_tag, } diff --git a/omni/sources.py b/omni/sources.py index 7270bae..08bfdb3 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -21,6 +21,7 @@ from .sources_ext5 import EXTRA6 from .sources_fresh import fresh as _fresh_sources from .sources_quality import quality_sources as _quality_sources from .sources_x import X_SOURCES +from .sources_auto import AUTO as AUTO_SOURCES @dataclass @@ -204,7 +205,7 @@ def _dedup(rows): return out -SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES]) +SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in ABUSE_DEEP] + [Source(**d) for d in TG_ABUSE] + [Source(**d) for d in X_SOURCES] + [Source(**d) for d in AUTO_SOURCES]) BY_NAME = {s.name: s for s in SOURCES} CATEGORIES = sorted({s.category for s in SOURCES}) @@ -365,7 +366,18 @@ def pick(category=None, only=None, tags=None, respect_enabled=True): out = [s for s in out if s.category in cats] if only: names = {n.strip() for n in only.split(",") if n.strip()} - out = [s for s in out if s.name in names] + + def _hit(src): + if src.name in names: + return True + for n in names: + if n.endswith("*") and src.name.startswith(n[:-1]): + return True + if n.endswith("_") and src.name.startswith(n): + return True + return False + + out = [x for x in out if _hit(x)] if tags: wanted = {t.strip() for t in tags.split(",") if t.strip()} out = [s for s in out if wanted & set(s.tags)] diff --git a/omni/sources_auto.py b/omni/sources_auto.py new file mode 100644 index 0000000..508dc2f --- /dev/null +++ b/omni/sources_auto.py @@ -0,0 +1,75 @@ +"""Авто-расширение: источники на публичных API без ключа. + +Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json` +(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON. + +Обновлено: 2026-10-27 16:41 UTC · живых 46 из 52 +""" + + +def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None): + return {"name": name, "category": "abuse", "kind": kind, "url": url, + "adapter": adapter, "tags": tuple(tags), "limit": limit, + "detail": False, "extra": dict(extra or {})} + + +AUTO = [ + # ---------- открытые JSON API без ключа ---------- + A("aa_devto_ai", "json", "https://dev.to/api/articles?per_page=25&top=7&tag=ai", "generic_json", ("abuse", "devto"), extra={"date_key": "published_at"}), + A("aa_lobsters", "json", "https://lobste.rs/hottest.json", "generic_json", ("abuse")), + A("aa_npm_llm", "json", "https://registry.npmjs.org/-/v1/search?text=llm%20api&size=25", "generic_json", ("abuse", "npm"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}), + A("aa_npm_mcp", "json", "https://registry.npmjs.org/-/v1/search?text=mcp%20server&size=25", "generic_json", ("abuse", "npm"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}), + A("aa_crates_llm", "json", "https://crates.io/api/v1/crates?q=llm&per_page=25", "generic_json", ("abuse", "crates"), extra={"rows": "crates", "base": "https://crates.io/crates/", "slug_key": "name"}), + A("aa_dockerhub_llm", "json", "https://hub.docker.com/v2/repositories/library/?page_size=25", "generic_json", ("abuse", "docker"), extra={"rows": "results", "base": "https://hub.docker.com/r/library/", "slug_key": "name"}), + A("aa_stack_openai", "json", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=openai&site=stackoverflow&pagesize=25", "generic_json", ("abuse", "stack"), extra={"rows": "items", "url_key": "link", "date_key": "creation_date"}), + A("aa_stack_llm", "json", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=llm&site=stackoverflow&pagesize=25", "generic_json", ("abuse", "stack"), extra={"rows": "items", "url_key": "link", "date_key": "creation_date"}), + A("aa_gitlab_new", "json", "https://gitlab.com/api/v4/projects?sort=desc&order_by=created_at&per_page=25", "generic_json", ("abuse", "gitlab"), extra={"url_key": "web_url"}), + A("aa_devpost", "json", "https://devpost.com/api/hackathons?status[]=open", "generic_json", ("abuse", "hackathon"), extra={"rows": "hackathons", "url_key": "url"}), + A("aa_hf_spaces_likes", "json", "https://huggingface.co/api/spaces?sort=likes&limit=25", "generic_json", ("abuse", "hf"), extra={"base": "https://huggingface.co/spaces/", "slug_key": "id", "title_key": "id"}), + A("aa_hf_spaces_trending", "json", "https://huggingface.co/api/spaces?sort=trendingScore&limit=25", "generic_json", ("abuse", "hf"), extra={"base": "https://huggingface.co/spaces/", "slug_key": "id", "title_key": "id"}), + A("aa_hf_models_trending", "json", "https://huggingface.co/api/models?sort=trendingScore&limit=25", "generic_json", ("abuse"), extra={"base": "https://huggingface.co/", "slug_key": "id", "title_key": "id"}), + A("aa_gitlab_llm", "json", "https://gitlab.com/api/v4/projects?search=llm&sort=desc&order_by=created_at&per_page=25", "generic_json", ("abuse"), extra={"url_key": "web_url"}), + A("aa_npm_free_api", "json", "https://registry.npmjs.org/-/v1/search?text=free%20api%20keys&size=25", "generic_json", ("abuse"), extra={"rows": "objects", "deref": "package", "title_key": "name", "text_key": "description", "base": "https://www.npmjs.com/package/", "slug_key": "name"}), + A("aa_crates_proxy", "json", "https://crates.io/api/v1/crates?q=openai%20proxy&per_page=20", "generic_json", ("abuse"), extra={"rows": "crates", "base": "https://crates.io/crates/", "slug_key": "name"}), + + # ---------- Reddit через pullpush.io (без ключа) ---------- + A("aa_pp_freeai", "json", "https://api.pullpush.io/reddit/search/submission/?subreddit=FreeAI&size=25", "generic_json", ("abuse"), extra={"rows": "data", "url_key": "url", "title_key": "title", "text_key": "selftext", "date_key": "created_utc"}), + A("aa_pp_openaiapi", "json", "https://api.pullpush.io/reddit/search/submission/?subreddit=OpenAIAPI&size=25", "generic_json", ("abuse"), extra={"rows": "data", "url_key": "url", "title_key": "title", "text_key": "selftext", "date_key": "created_utc"}), + + # ---------- Bluesky / Mastodon ---------- + A("aa_bsky_free_api", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20api%20key&limit=25", "bsky_search", ("abuse", "bsky")), + A("aa_bsky_claude_api", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=claude%20api&limit=25", "bsky_search", ("abuse", "bsky")), + A("aa_bsky_jailbreak", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=jailbreak%20llm&limit=25", "bsky_search", ("abuse", "bsky", "jb")), + A("aa_masto_llm", "json", "https://mastodon.social/api/v1/timelines/tag/llm?limit=25", "mastodon_tag", ("abuse", "mastodon")), + A("aa_masto_jailbreak", "json", "https://mastodon.social/api/v1/timelines/tag/jailbreak?limit=25", "mastodon_tag", ("abuse", "mastodon", "jb")), + A("aa_masto_ai", "json", "https://mastodon.social/api/v1/timelines/tag/openai?limit=25", "mastodon_tag", ("abuse")), + A("aa_bsky_free_tokens", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20tokens%20api&limit=25", "bsky_search", ("abuse")), + A("aa_bsky_openrouter", "json", "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=openrouter&limit=25", "bsky_search", ("abuse")), + + # ---------- Discourse-сообщества ---------- + A("aa_dis_openai", "json", "https://community.openai.com/latest.json", "discourse", ("abuse")), + A("aa_dis_hf", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("abuse")), + A("aa_dis_cursor", "json", "https://forum.cursor.com/latest.json", "discourse", ("abuse")), + A("aa_dis_python", "json", "https://discuss.python.org/latest.json", "discourse", ("abuse")), + A("aa_dis_fly", "json", "https://community.fly.io/latest.json", "discourse", ("abuse")), + A("aa_dis_obsidian", "json", "https://forum.obsidian.md/latest.json", "discourse", ("abuse")), + + # ---------- RSS/Atom ---------- + A("aa_rss_habr", "rss", "https://habr.com/ru/rss/best/daily/?fl=ru", "rss", ("abuse")), + A("aa_rss_decoder", "rss", "https://the-decoder.com/feed/", "rss", ("abuse")), + A("aa_rss_marktechpost", "rss", "https://www.marktechpost.com/feed/", "rss", ("abuse")), + A("aa_rss_openai_news", "rss", "https://openai.com/news/rss.xml", "rss", ("abuse")), + A("aa_rss_simonw", "rss", "https://simonwillison.net/atom/everything/", "rss", ("abuse")), + A("aa_rss_hf_blog", "rss", "https://huggingface.co/blog/feed.xml", "rss", ("abuse")), + A("aa_rss_tc_ai", "rss", "https://techcrunch.com/category/artificial-intelligence/feed/", "rss", ("abuse")), + A("aa_rss_52pojie", "rss", "https://www.52pojie.cn/forum.php?mod=rss", "rss", ("abuse")), + + # ---------- Telegram-превью ---------- + A("aa_tg_ai_newz", "html", "https://t.me/s/ai_newz", "tg_web", ("abuse", "tg")), + A("aa_tg_data_secrets", "html", "https://t.me/s/data_secrets", "tg_web", ("abuse", "tg")), + A("aa_tg_denissexy", "html", "https://t.me/s/denissexy", "tg_web", ("abuse", "tg")), + A("aa_tg_neurohive", "json", "https://t.me/s/neurohive", "tg_web", ("abuse")), + A("aa_tg_gpt4telegram", "json", "https://t.me/s/gpt4telegram", "tg_web", ("abuse")), + A("aa_tg_dataleak", "json", "https://t.me/s/dataleak", "tg_web", ("abuse")), + +] diff --git a/tools/auto_cycle.py b/tools/auto_cycle.py new file mode 100644 index 0000000..bc5898b --- /dev/null +++ b/tools/auto_cycle.py @@ -0,0 +1,90 @@ +"""Один цикл автоулучшения: проверить кандидатов → пересобрать sources_auto.py → +докачать новое → пересобрать abuse-индекс → коммит/пуш. Запускать по крону (6ч). + + python tools/auto_cycle.py # полный цикл + python tools/auto_cycle.py --dry # без коммита и без сети (только отчёт) + +Лог: data/auto_cycle.log (append). +""" +import argparse +import os +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PY = sys.executable +LOG = ROOT / "data" / "auto_cycle.log" +ENV = dict(os.environ, PYTHONPATH="", PYTHONIOENCODING="utf-8", PYTHONUTF8="1") + + +def log(msg): + line = "%s %s" % (datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M"), msg) + print(line) + with LOG.open("a", encoding="utf-8") as fh: + fh.write(line + "\n") + + +def run(args, timeout=1800): + t = time.time() + p = subprocess.run([PY] + args, cwd=str(ROOT), env=ENV, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + out = (p.stdout or "") + (p.stderr or "") + return p.returncode, out.strip(), round(time.time() - t, 1) + + +def git(*args, timeout=180): + p = subprocess.run(["git"] + list(args), cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + return p.returncode, ((p.stdout or "") + (p.stderr or "")).strip() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dry", action="store_true") + ap.add_argument("--pause", type=float, default=6.0) + a = ap.parse_args() + log("=== cycle start ===") + + if not a.dry: + code, out, sec = run(["tools/auto_expand.py", "--probe", "--write", "--pause", str(a.pause)]) + tail = [l for l in out.splitlines() if "LIVE" in l or "wrote" in l or "skip (" in l] + log("probe/write rc=%s %ss | %s" % (code, sec, " / ".join(tail[-3:]) or out[-200:])) + + changed = git("status", "--porcelain", "--", "omni/sources_auto.py")[1] + live = 0 + try: + import re + txt = (ROOT / "omni" / "sources_auto.py").read_text(encoding="utf-8") + live = len(re.findall(r"^ A\(", txt, re.M)) + except Exception: + pass + log("sources_auto: live=%d changed=%s" % (live, bool(changed))) + + if changed and not a.dry: + code, out, sec = run(["-m", "omni", "feed", "--only", "aa_", "--limit", "15"]) + log("feed rc=%s %ss | %s" % (code, sec, out.splitlines()[-1] if out else "")) + + if not a.dry: + code, out, sec = run(["-c", "import sys;sys.path.insert(0,'.');" + "from omni import abuse;print('rebuilt', abuse.rebuild())"], timeout=2400) + log("abuse rc=%s %ss | %s" % (code, sec, (out.splitlines() or [""])[-1])) + + if changed and not a.dry: + git("add", "--", "omni/sources_auto.py") + code, out = git("commit", "-m", "auto: источники %d live (%s)" % + (live, datetime.now(timezone.utc).strftime("%d.%m %H:%M UTC"))) + log("commit rc=%s | %s" % (code, out.splitlines()[0] if out else "")) + for remote in ("gitea", "github"): + if not git("remote")[1].find(remote) >= 0: + continue + rc, o = git("push", remote, "main") + log("push %s rc=%s | %s" % (remote, rc, o.splitlines()[-1] if o else "")) + log("=== cycle end ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tools/auto_expand.py b/tools/auto_expand.py new file mode 100644 index 0000000..9b5ec55 --- /dev/null +++ b/tools/auto_expand.py @@ -0,0 +1,218 @@ +"""Авто-расширение источников: тест кандидатов → генерация omni/sources_auto.py. + + python tools/auto_expand.py --probe # только проверка, без записи + python tools/auto_expand.py --write # живые кандидаты → sources_auto.py + python tools/auto_expand.py --discover # добрать кандидатов из public-apis + python tools/auto_expand.py --add NAME URL --adapter rss + +Кандидаты: data/source_candidates.json (list of {name,url,adapter,kind,extra,tags,limit}). +Здоровье: data/source_health.json — сколько раз источник отдал items и когда. +Мёртвым считается кандидат с 0 items в 3 прогонах подряд → исключается из генерации. +""" +import argparse +import json +import sys +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from omni import feeds # noqa: E402 +from omni.sources import Source # noqa: E402 + +CAND = ROOT / "data" / "source_candidates.json" +HEALTH = ROOT / "data" / "source_health.json" +OUT = ROOT / "omni" / "sources_auto.py" +PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md" + +HEADER = '''"""Авто-расширение: источники на публичных API без ключа. + +Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json` +(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON. + +Обновлено: {ts} · живых {live} из {total} +""" + + +def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None): + return {{"name": name, "category": "abuse", "kind": kind, "url": url, + "adapter": adapter, "tags": tuple(tags), "limit": limit, + "detail": False, "extra": dict(extra or {{}})}} + + +AUTO = [ +{body}] +''' + +GROUP_RU = { + "json": "# ---------- открытые JSON API без ключа ----------", + "reddit": "# ---------- Reddit через pullpush.io (без ключа) ----------", + "social": "# ---------- Bluesky / Mastodon ----------", + "discourse": "# ---------- Discourse-сообщества ----------", + "rss": "# ---------- RSS/Atom ----------", + "tg": "# ---------- Telegram-превью ----------", +} + + +def load(path, default): + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return default + + +def save(path, data): + path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") + + +def to_source(c): + return Source(name=c["name"], category=c.get("category", "abuse"), kind=c.get("kind", "json"), + url=c["url"], adapter=c["adapter"], tags=tuple(c.get("tags") or ("abuse",)), + limit=int(c.get("limit", 20)), detail=False, extra=dict(c.get("extra") or {})) + + +def probe(cands, pause=2.0): + health = load(HEALTH, {}) + res = [] + last_host = None + for c in cands: + host = c["url"].split("/")[2] if "//" in c["url"] else "" + if host == last_host and pause: + time.sleep(pause) + last_host = host + try: + items = feeds.fetch(to_source(c), limit=min(20, int(c.get("limit", 20)))) + n = len(items) + sample = [i.title[:70] for i in items[:2]] + err = "" + except Exception as exc: + n, sample, err = 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:90]) + h = health.setdefault(c["name"], {"ok": 0, "fail": 0, "items": 0}) + h["items"] = n + h["ok"] = h.get("ok", 0) + 1 if n else 0 + h["fail"] = 0 if n else h.get("fail", 0) + 1 + h["last"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M") + h["url"] = c["url"] + if err: + h["error"] = err + else: + h.pop("error", None) + res.append((c, n, sample, err)) + print("%-26s %-4s %3d %s" % (c["name"], "LIVE" if n else "DEAD", n, err or " | ".join(sample))) + save(HEALTH, health) + return res + + +def write_module(cands, results): + health = load(HEALTH, {}) + order = ["json", "reddit", "social", "discourse", "rss", "tg"] + buckets = {k: [] for k in order} + for c, n, _, _ in results: + if not n: + continue + if (health.get(c["name"]) or {}).get("fail", 0) >= 3: + print("skip (3 fails подряд):", c["name"]) + continue + g = c.get("group") or ("tg" if c["adapter"] == "tg_web" else + "discourse" if c["adapter"] == "discourse" else + "reddit" if "pullpush" in c["url"] or "reddit" in c["url"] else + "social" if c["adapter"] in ("bsky_search", "mastodon_tag") else + "rss" if c["adapter"] == "rss" else "json") + buckets.setdefault(g, []).append(c) + body = [] + live = 0 + for g in order + [k for k in buckets if k not in order]: + rows = buckets.get(g) or [] + if not rows: + continue + body.append(" " + GROUP_RU.get(g, "# ---------- прочее ----------")) + for c in rows: + live += 1 + extra = "" + if c.get("extra"): + extra = ", extra=%s" % json.dumps(c["extra"], ensure_ascii=False) + tags = ", ".join('"%s"' % t for t in (c.get("tags") or ("abuse",))) + body.append(' A("%s", "%s", "%s", "%s", (%s)%s),' % ( + c["name"], c.get("kind", "json"), c["url"], c["adapter"], tags, extra)) + body.append("") + ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + OUT.write_text(HEADER.format(ts=ts, live=live, total=len(cands), body="\n".join(body) + "\n"), + encoding="utf-8") + print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands))) + + +def discover(limit=40): + """Добрать бесплатные (Auth=No) API из public-apis.""" + req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"}) + md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace") + rows = [] + for line in md.splitlines(): + if not line.startswith("|") or "http" not in line: + continue + cells = [c.strip() for c in line.strip("|").split("|")] + if len(cells) < 6: + continue + url, auth, https = cells[2], cells[4], cells[5] + if "No" not in auth or "http" not in url: + continue + if any(b in url for b in ("github.com/public-apis", "localhost")): + continue + rows.append({"name": "dis_" + url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_")[:24], + "url": url, "adapter": "generic_json", "kind": "json", + "tags": ["abuse", "public_api"], "group": "json", "limit": 15}) + uniq = {} + for r in rows: + uniq.setdefault(r["name"], r) + cands = load(CAND, []) + have = {c["name"] for c in cands} | {c["url"] for c in cands} + new = [r for r in uniq.values() if r["name"] not in have and r["url"] not in have][:limit] + cands.extend(new) + save(CAND, cands) + print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands))) + return new + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--probe", action="store_true") + ap.add_argument("--write", action="store_true") + ap.add_argument("--write-only", action="store_true", help="сгенерировать из data/source_health.json без сети") + ap.add_argument("--discover", action="store_true") + ap.add_argument("--add", nargs=2, metavar=("NAME", "URL")) + ap.add_argument("--adapter", default="generic_json") + ap.add_argument("--pause", type=float, default=2.0) + a = ap.parse_args() + + if a.add: + cands = load(CAND, []) + cands.append({"name": a.add[0], "url": a.add[1], "adapter": a.adapter, + "kind": "json", "tags": ["abuse"], "limit": 20}) + save(CAND, cands) + print("added", a.add[0]) + return 0 + if a.discover: + discover() + return 0 + + cands = load(CAND, []) + if a.write_only: + health = load(HEALTH, {}) + results = [(c, (health.get(c["name"]) or {}).get("items", 0), [], "") for c in cands] + write_module(cands, results) + return 0 + if not cands: + print("нет кандидатов в", CAND) + return 1 + results = probe(cands, pause=a.pause) + live = sum(1 for _, n, _, _ in results if n) + print("\nLIVE %d / %d, items %d" % (live, len(results), sum(n for _, n, _, _ in results))) + if a.write: + write_module(cands, results) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tools/check_yield.py b/tools/check_yield.py new file mode 100644 index 0000000..b590de5 --- /dev/null +++ b/tools/check_yield.py @@ -0,0 +1,31 @@ +"""Проверка yield каждого источника: имя | items | первых 2 заголовка.""" +import sys +import traceback +from concurrent.futures import ThreadPoolExecutor + +sys.path.insert(0, ".") + +from omni import feeds, sources # noqa: E402 + +prefix = sys.argv[1] if len(sys.argv) > 1 else "aa_" +picked = [s for s in sources.SOURCES if s.name.startswith(prefix)] +print("sources:", len(picked)) + + +def run(src): + try: + items = feeds.fetch(src, limit=min(20, src.limit or 20)) + return src.name, len(items), [i.title[:60] for i in items[:2]], "" + except Exception as exc: + return src.name, 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:70]) + + +with ThreadPoolExecutor(max_workers=10) as ex: + rows = list(ex.map(run, picked)) + +live = 0 +for name, n, titles, err in sorted(rows, key=lambda r: -r[1]): + flag = "LIVE" if n else "DEAD" + live += bool(n) + print("%-26s %-4s %3d %s" % (name, flag, n, err or " | ".join(titles))) +print("LIVE %d / %d, items %d" % (live, len(rows), sum(r[1] for r in rows))) \ No newline at end of file diff --git a/tools/probe_candidates.py b/tools/probe_candidates.py new file mode 100644 index 0000000..0d6c53b --- /dev/null +++ b/tools/probe_candidates.py @@ -0,0 +1,89 @@ +"""Живость кандидатов: статус, размер, число элементов в JSON/ленте.""" +import json +import sys +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0 Safari/537.36") + +CAND = [ + ("devto", "https://dev.to/api/articles?per_page=25&top=7"), + ("lobsters", "https://lobste.rs/hottest.json"), + ("npm", "https://registry.npmjs.org/-/v1/search?text=llm&size=25"), + ("crates", "https://crates.io/api/v1/crates?q=llm&per_page=25"), + ("dockerhub", "https://hub.docker.com/v2/repositories/library/?page_size=25"), + ("stackoverflow", "https://api.stackexchange.com/2.3/questions?order=desc&sort=activity&tagged=openai&site=stackoverflow&pagesize=25"), + ("gitlab", "https://gitlab.com/api/v4/projects?sort=desc&order_by=created_at&per_page=25"), + ("devpost", "https://devpost.com/api/hackathons?status[]=open"), + ("hf_spaces", "https://huggingface.co/api/spaces?sort=likes&limit=25"), + ("mastodon", "https://mastodon.social/api/v1/timelines/tag/llm?limit=25"), + ("bsky", "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=free%20api&limit=25"), + ("reddit_localllama", "https://www.reddit.com/r/LocalLLaMA/.json?limit=25"), + ("reddit_chatgptproxies", "https://www.reddit.com/r/ChatGPTProxies/.json?limit=25"), + ("reddit_jailbreak", "https://www.reddit.com/r/jailbreak/.json?limit=25"), + ("reddit_cybersecurity", "https://www.reddit.com/r/cybersecurity/.json?limit=25"), + ("reddit_selfhosted", "https://www.reddit.com/r/selfhosted/.json?limit=25"), + ("reddit_piracy", "https://www.reddit.com/r/Piracy/.json?limit=25"), + ("dis_openai", "https://community.openai.com/latest.json"), + ("dis_hf", "https://discuss.huggingface.co/latest.json"), + ("dis_cursor", "https://forum.cursor.com/latest.json"), + ("dis_python", "https://discuss.python.org/latest.json"), + ("dis_fly", "https://community.fly.io/latest.json"), + ("dis_obsidian", "https://forum.obsidian.md/latest.json"), + ("rss_habr", "https://habr.com/ru/rss/best/daily/?fl=ru"), + ("rss_decoder", "https://the-decoder.com/feed/"), + ("rss_marktechpost", "https://www.marktechpost.com/feed/"), + ("rss_openai", "https://openai.com/news/rss.xml"), + ("rss_simonw", "https://simonwillison.net/atom/everything/"), + ("rss_hf", "https://huggingface.co/blog/feed.xml"), + ("rss_tc_ai", "https://techcrunch.com/category/artificial-intelligence/feed/"), + ("tg_ai_newz", "https://t.me/s/ai_newz"), + ("tg_seeallo", "https://t.me/s/seeallo"), + ("tg_data_secrets", "https://t.me/s/data_secrets"), + ("tg_gptblog", "https://t.me/s/gptblog"), + ("tg_denissexy", "https://t.me/s/denissexy"), + ("gh_trending", "https://github.com/trending?since=daily"), +] + + +def probe(item): + name, url = item + req = urllib.request.Request(url, headers={"User-Agent": UA, + "Accept": "application/json, text/html, application/xml;q=0.9"}) + try: + with urllib.request.urlopen(req, timeout=20) as r: + raw = r.read(400000) + code = r.status + except Exception as exc: + return name, str(exc)[:60], 0, 0 + n = 0 + try: + d = json.loads(raw.decode("utf-8", "replace")) + if isinstance(d, list): + n = len(d) + elif isinstance(d, dict): + for k in ("items", "posts", "results", "objects", "crates", "hackathons", "data", "articles", "children"): + v = d.get(k) + if isinstance(v, list): + n = len(v) + break + if not n: + n = len([v for v in d.values() if isinstance(v, list)]) + except Exception: + try: + text = raw.decode("utf-8", "replace") + n = text.count("