omni: менеджер источников в панели (338 источников, вкл/выкл, добавление, health-check, фоновые джобы)

omni/doctor.py

@@ -0,0 +1,115 @@
+"""Доктор источников: проверяет каждый, а с --fix сам находит рабочий фид по HTML-странице.
+
+Причина: у половины блогов угаданный /rss.xml отдаёт 404, у части — Cloudflare.
+Для rss-источников лечим автопоиском <link rel="alternate" type="...rss..."> на главной,
+результат кладём в data/feeds_cache.json (переопределяет url без правки реестра).
+"""
+import json
+import pathlib
+import re
+import urllib.parse
+from concurrent.futures import ThreadPoolExecutor
+
+from bs4 import BeautifulSoup
+
+from . import feeds, sources
+
+CACHE = pathlib.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
+
+
+def load_cache():
+    try:
+        return json.loads(CACHE.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+
+
+def save_cache(d):
+    CACHE.parent.mkdir(parents=True, exist_ok=True)
+    CACHE.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")
+
+
+def discover(home):
+    """Ищем ссылки на фиды в HTML главной/раздела."""
+    r = feeds.net.get(home, headers=feeds.UA)
+    if r.status != 200 or not r.text:
+        return []
+    out = []
+    soup = BeautifulSoup(r.text, "lxml")
+    for ln in soup.find_all("link", href=True):
+        typ = (ln.get("type") or "").lower()
+        if "rss" in typ or "atom" in typ:
+            out.append(urllib.parse.urljoin(home, str(ln["href"])))
+    for a in soup.find_all("a", href=True):
+        h = str(a["href"])
+        if re.search(r"(rss|feed|atom)(\.xml|\.rss)?/?$", h) and "comment" not in h:
+            out.append(urllib.parse.urljoin(home, h))
+    seen, uniq = set(), []
+    for u in out:
+        if u not in seen:
+            seen.add(u)
+            uniq.append(u)
+    return uniq[:8]
+
+
+def probe(src, limit=3):
+    try:
+        got = feeds.fetch(src, limit)
+        return src.name, len(got), (got[0].title[:60] if got else "")
+    except Exception as e:
+        return src.name, 0, f"{type(e).__name__}"
+
+
+def fix(src):
+    """Подобрать рабочий фид: кандидаты с главной (и её подразделов)."""
+    base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
+    cands = []
+    for home in (src.url, base + "/", base + "/blog", base + "/news", base + "/research"):
+        try:
+            cands += discover(home)
+        except Exception:
+            continue
+    seen = set()
+    for u in cands:
+        if u in seen:
+            continue
+        seen.add(u)
+        trial = sources.Source(src.name, src.category, src.kind, u, "rss", src.tags)
+        try:
+            got = feeds.fetch(trial, 3)
+        except Exception:
+            got = []
+        if got:
+            return u, len(got)
+    return None, 0
+
+
+def run(only=None, do_fix=False, workers=8, limit=3):
+    from . import source_state
+    picked = sources.pick(None, only, None)
+    cache = load_cache()
+    results = []
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        for name, n, extra in ex.map(lambda s: probe(s, limit), picked):
+            results.append({"name": name, "n": n, "extra": extra})
+            source_state.record_check(name, 200 if n else 0, n, "ok" if n else str(extra)[:120])
+    bad = [r for r in results if r["n"] == 0]
+    print(f"проверено {len(results)} | живых {len(results) - len(bad)} | пустых {len(bad)}")
+    if do_fix:
+        by_name = {s.name: s for s in picked}
+        fixed = 0
+        for r in bad:
+            src = by_name[r["name"]]
+            if src.adapter not in ("rss", "html_listing", "github_org"):
+                continue
+            url, n = fix(src)
+            if url:
+                cache[src.name] = {"url": url, "adapter": "rss", "n": n}
+                fixed += 1
+                print(f"  фикс {src.name:24s} -> {url}  ({n})")
+        save_cache(cache)
+        print(f"починено {fixed}, кэш: {len(cache)} записей -> {CACHE}")
+    else:
+        for r in bad:
+            print(f"  пусто {r['name']:26s} {r['extra']}")
+    return results, cache