245 lines
10 KiB
Python
245 lines
10 KiB
Python
"""Авто-расширение источников: тест кандидатов → генерация omni/sources_auto.py.
|
||
|
||
python tools/auto_expand.py --probe # только проверка, без записи
|
||
python tools/auto_expand.py --write # живые кандидаты → sources_auto.py
|
||
python tools/auto_expand.py --discover # добрать кандидатов из public-apis
|
||
python tools/auto_expand.py --add NAME URL --adapter rss
|
||
|
||
Кандидаты: data/source_candidates.json (list of {name,url,adapter,kind,extra,tags,limit}).
|
||
Здоровье: data/source_health.json — сколько раз источник отдал items и когда.
|
||
Мёртвым считается кандидат с 0 items в 3 прогонах подряд → исключается из генерации.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from omni import feeds # noqa: E402
|
||
from omni.sources import Source # noqa: E402
|
||
|
||
CAND = ROOT / "data" / "source_candidates.json"
|
||
HEALTH = ROOT / "data" / "source_health.json"
|
||
OUT = ROOT / "omni" / "sources_auto.py"
|
||
PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md"
|
||
RE_URL = __import__("re").compile(r"\((https?://[^)\s]+)\)")
|
||
RE_LINK = __import__("re").compile(r"\[([^\]]+)\]\([^)]+\)")
|
||
|
||
HEADER = '''"""Авто-расширение: источники на публичных API без ключа.
|
||
|
||
Файл генерируется `tools/auto_expand.py` из `data/source_candidates.json`
|
||
(живые кандидаты) — правки руками перезапишутся, добавляй кандидатов в JSON.
|
||
|
||
Обновлено: {ts} · живых {live} из {total}
|
||
"""
|
||
|
||
|
||
def A(name, kind, url, adapter, tags=("abuse",), limit=20, extra=None):
|
||
return {{"name": name, "category": "abuse", "kind": kind, "url": url,
|
||
"adapter": adapter, "tags": tuple(tags), "limit": limit,
|
||
"detail": False, "extra": dict(extra or {{}})}}
|
||
|
||
|
||
AUTO = [
|
||
{body}]
|
||
'''
|
||
|
||
GROUP_RU = {
|
||
"json": "# ---------- открытые JSON API без ключа ----------",
|
||
"reddit": "# ---------- Reddit через pullpush.io (без ключа) ----------",
|
||
"social": "# ---------- Bluesky / Mastodon ----------",
|
||
"discourse": "# ---------- Discourse-сообщества ----------",
|
||
"rss": "# ---------- RSS/Atom ----------",
|
||
"tg": "# ---------- Telegram-превью ----------",
|
||
}
|
||
|
||
|
||
def load(path, default):
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def save(path, data):
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
|
||
def to_source(c):
|
||
return Source(name=c["name"], category=c.get("category", "abuse"), kind=c.get("kind", "json"),
|
||
url=c["url"], adapter=c["adapter"], tags=tuple(c.get("tags") or ("abuse",)),
|
||
limit=int(c.get("limit", 20)), detail=False, extra=dict(c.get("extra") or {}))
|
||
|
||
|
||
def probe(cands, pause=2.0):
|
||
health = load(HEALTH, {})
|
||
res = []
|
||
last_host = None
|
||
for c in cands:
|
||
host = c["url"].split("/")[2] if "//" in c["url"] else ""
|
||
if host == last_host and pause:
|
||
time.sleep(pause)
|
||
last_host = host
|
||
try:
|
||
items = feeds.fetch(to_source(c), limit=min(20, int(c.get("limit", 20))))
|
||
n = len(items)
|
||
sample = [i.title[:70] for i in items[:2]]
|
||
err = ""
|
||
except Exception as exc:
|
||
n, sample, err = 0, [], "%s: %s" % (type(exc).__name__, str(exc)[:90])
|
||
h = health.setdefault(c["name"], {"ok": 0, "fail": 0, "items": 0})
|
||
h["items"] = n
|
||
h["ok"] = h.get("ok", 0) + 1 if n else 0
|
||
h["fail"] = 0 if n else h.get("fail", 0) + 1
|
||
h["last"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M")
|
||
h["url"] = c["url"]
|
||
if err:
|
||
h["error"] = err
|
||
else:
|
||
h.pop("error", None)
|
||
res.append((c, n, sample, err))
|
||
print("%-26s %-4s %3d %s" % (c["name"], "LIVE" if n else "DEAD", n, err or " | ".join(sample)))
|
||
save(HEALTH, health)
|
||
return res
|
||
|
||
|
||
def write_module(cands, results):
|
||
health = load(HEALTH, {})
|
||
order = ["json", "reddit", "social", "discourse", "rss", "tg"]
|
||
buckets = {k: [] for k in order}
|
||
for c, n, _, _ in results:
|
||
if not n:
|
||
continue
|
||
if (health.get(c["name"]) or {}).get("fail", 0) >= 3:
|
||
print("skip (3 fails подряд):", c["name"])
|
||
continue
|
||
g = c.get("group") or ("tg" if c["adapter"] == "tg_web" else
|
||
"discourse" if c["adapter"] == "discourse" else
|
||
"reddit" if "pullpush" in c["url"] or "reddit" in c["url"] else
|
||
"social" if c["adapter"] in ("bsky_search", "mastodon_tag") else
|
||
"rss" if c["adapter"] == "rss" else "json")
|
||
buckets.setdefault(g, []).append(c)
|
||
body = []
|
||
live = 0
|
||
for g in order + [k for k in buckets if k not in order]:
|
||
rows = buckets.get(g) or []
|
||
if not rows:
|
||
continue
|
||
body.append(" " + GROUP_RU.get(g, "# ---------- прочее ----------"))
|
||
for c in rows:
|
||
live += 1
|
||
extra = ""
|
||
if c.get("extra"):
|
||
extra = ", extra=%s" % json.dumps(c["extra"], ensure_ascii=False)
|
||
tags = ", ".join('"%s"' % t for t in (c.get("tags") or ("abuse",)))
|
||
body.append(' A("%s", "%s", "%s", "%s", (%s)%s),' % (
|
||
c["name"], c.get("kind", "json"), c["url"], c["adapter"], tags, extra))
|
||
body.append("")
|
||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||
OUT.write_text(HEADER.format(ts=ts, live=live, total=len(cands), body="\n".join(body) + "\n"),
|
||
encoding="utf-8")
|
||
print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands)))
|
||
|
||
|
||
CAT_OK = ("Artificial Intelligence", "Machine Learning", "Security", "News", "Open Data",
|
||
"Development", "Text Analysis", "Data Validation", "Sports", "Cryptocurrency")
|
||
KEEP_ANY_CAT = ("llm", "gpt", "openai", "threat", "cve", "vuln", "leak", "hack", "malware",
|
||
"phish", "proxy", "temp mail", "sms", "telegram", "scrap", "dns", "whois")
|
||
BAD = ("waifu", "anime", "cat fact", "dog", "joke", "meme", "pokemon", "quote", "trivia",
|
||
"recipe", "cocktail", "game", "3d", "texture", "insurance", "crypto price")
|
||
KW_OK = ("ai", "llm", "gpt", "model", "secur", "threat", "cve", "vuln", "leak", "hack", "news",
|
||
"feed", "telegram", "proxy", "mail", "sms", "text", "translate", "scrap", "search",
|
||
"crypto", "token", "domain", "dns", "ip", "geo")
|
||
|
||
|
||
def discover(limit=40):
|
||
"""Добрать бесплатные (Auth=No) API из public-apis: только релевантные категории/слова."""
|
||
req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"})
|
||
md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace")
|
||
rows, cat = [], ""
|
||
for line in md.splitlines():
|
||
if line.startswith("### "):
|
||
cat = line[4:].strip()
|
||
continue
|
||
if not line.startswith("|") or "http" not in line:
|
||
continue
|
||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||
if len(cells) < 5:
|
||
continue
|
||
m = RE_URL.search(cells[0])
|
||
if not m:
|
||
continue
|
||
url = m.group(1)
|
||
title = RE_LINK.sub(r"\1", cells[0]).strip()
|
||
desc, auth = cells[1], cells[2]
|
||
if "No" not in auth:
|
||
continue
|
||
blob = (title + " " + desc).lower()
|
||
if cat not in CAT_OK and not any(k in blob for k in KEEP_ANY_CAT):
|
||
continue
|
||
if any(b in blob for b in BAD):
|
||
continue
|
||
host = url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_").replace("-", "_")
|
||
rows.append({"name": "dis_" + host[:26], "url": url, "adapter": "generic_json",
|
||
"kind": "json", "tags": ["abuse", "public_api"], "group": "json", "limit": 15,
|
||
"note": (title + " — " + desc)[:120], "cat": cat})
|
||
uniq = {}
|
||
for r in rows:
|
||
uniq.setdefault(r["name"], r)
|
||
cands = load(CAND, [])
|
||
have = {c["name"] for c in cands} | {c["url"] for c in cands}
|
||
new = [r for r in uniq.values() if r["name"] not in have and r["url"] not in have][:limit]
|
||
cands.extend(new)
|
||
save(CAND, cands)
|
||
print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands)))
|
||
for r in new[:12]:
|
||
print(" + %-30s %s" % (r["name"], r["note"][:80]))
|
||
return new
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--probe", action="store_true")
|
||
ap.add_argument("--write", action="store_true")
|
||
ap.add_argument("--write-only", action="store_true", help="сгенерировать из data/source_health.json без сети")
|
||
ap.add_argument("--discover", action="store_true")
|
||
ap.add_argument("--add", nargs=2, metavar=("NAME", "URL"))
|
||
ap.add_argument("--adapter", default="generic_json")
|
||
ap.add_argument("--pause", type=float, default=2.0)
|
||
a = ap.parse_args()
|
||
|
||
if a.add:
|
||
cands = load(CAND, [])
|
||
cands.append({"name": a.add[0], "url": a.add[1], "adapter": a.adapter,
|
||
"kind": "json", "tags": ["abuse"], "limit": 20})
|
||
save(CAND, cands)
|
||
print("added", a.add[0])
|
||
return 0
|
||
if a.discover:
|
||
discover()
|
||
return 0
|
||
|
||
cands = load(CAND, [])
|
||
if a.write_only:
|
||
health = load(HEALTH, {})
|
||
results = [(c, (health.get(c["name"]) or {}).get("items", 0), [], "") for c in cands]
|
||
write_module(cands, results)
|
||
return 0
|
||
if not cands:
|
||
print("нет кандидатов в", CAND)
|
||
return 1
|
||
results = probe(cands, pause=a.pause)
|
||
live = sum(1 for _, n, _, _ in results if n)
|
||
print("\nLIVE %d / %d, items %d" % (live, len(results), sum(n for _, n, _, _ in results)))
|
||
if a.write:
|
||
write_module(cands, results)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main()) |