119 lines
4.3 KiB
Python
119 lines
4.3 KiB
Python
"""Доктор источников: проверяет каждый, а с --fix сам находит рабочий фид по HTML-странице.
|
||
|
||
Причина: у половины блогов угаданный /rss.xml отдаёт 404, у части — Cloudflare.
|
||
Для rss-источников лечим автопоиском <link rel="alternate" type="...rss..."> на главной,
|
||
результат кладём в data/feeds_cache.json (переопределяет url без правки реестра).
|
||
"""
|
||
import json
|
||
import pathlib
|
||
import re
|
||
import urllib.parse
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
from bs4 import BeautifulSoup
|
||
|
||
from . import feeds, sources
|
||
|
||
CACHE = pathlib.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
|
||
|
||
|
||
def load_cache():
|
||
try:
|
||
return json.loads(CACHE.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_cache(d):
|
||
CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||
CACHE.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
|
||
def discover(home):
|
||
"""Ищем ссылки на фиды в HTML главной/раздела."""
|
||
r = feeds.net.get(home, headers=feeds.UA)
|
||
if r.status != 200 or not r.text:
|
||
return []
|
||
out = []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
for ln in soup.find_all("link", href=True):
|
||
typ = (ln.get("type") or "").lower()
|
||
if "rss" in typ or "atom" in typ:
|
||
out.append(urllib.parse.urljoin(home, str(ln["href"])))
|
||
for a in soup.find_all("a", href=True):
|
||
h = str(a["href"])
|
||
if re.search(r"(rss|feed|atom)(\.xml|\.rss)?/?$", h) and "comment" not in h:
|
||
out.append(urllib.parse.urljoin(home, h))
|
||
seen, uniq = set(), []
|
||
for u in out:
|
||
if u not in seen:
|
||
seen.add(u)
|
||
uniq.append(u)
|
||
return uniq[:8]
|
||
|
||
|
||
def probe(src, limit=3):
|
||
try:
|
||
got = feeds.fetch(src, limit)
|
||
return src.name, len(got), (got[0].title[:60] if got else "")
|
||
except Exception as e:
|
||
return src.name, 0, f"{type(e).__name__}"
|
||
|
||
|
||
def fix(src):
|
||
"""Подобрать рабочий фид: кандидаты с главной (и её подразделов)."""
|
||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||
cands = []
|
||
for home in (src.url, base + "/", base + "/blog", base + "/news", base + "/research"):
|
||
try:
|
||
cands += discover(home)
|
||
except Exception:
|
||
continue
|
||
seen = set()
|
||
for u in cands:
|
||
if u in seen:
|
||
continue
|
||
seen.add(u)
|
||
trial = sources.Source(src.name, src.category, src.kind, u, "rss", src.tags)
|
||
try:
|
||
got = feeds.fetch(trial, 3)
|
||
except Exception:
|
||
got = []
|
||
if got:
|
||
return u, len(got)
|
||
return None, 0
|
||
|
||
|
||
def run(only=None, do_fix=False, workers=8, limit=3):
|
||
from . import source_state
|
||
only = (only or "").strip() or None
|
||
if only and only in sources.CATEGORIES:
|
||
picked = sources.pick(category=only, respect_enabled=False)
|
||
else:
|
||
picked = sources.pick(only=only, respect_enabled=False)
|
||
cache = load_cache()
|
||
results = []
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
for name, n, extra in ex.map(lambda s: probe(s, limit), picked):
|
||
results.append({"name": name, "n": n, "extra": extra})
|
||
source_state.record_check(name, 200 if n else 0, n, "ok" if n else str(extra)[:120])
|
||
bad = [r for r in results if r["n"] == 0]
|
||
print(f"проверено {len(results)} | живых {len(results) - len(bad)} | пустых {len(bad)}")
|
||
if do_fix:
|
||
by_name = {s.name: s for s in picked}
|
||
fixed = 0
|
||
for r in bad:
|
||
src = by_name[r["name"]]
|
||
if src.adapter not in ("rss", "html_listing", "github_org"):
|
||
continue
|
||
url, n = fix(src)
|
||
if url:
|
||
cache[src.name] = {"url": url, "adapter": "rss", "n": n}
|
||
fixed += 1
|
||
print(f" фикс {src.name:24s} -> {url} ({n})")
|
||
save_cache(cache)
|
||
print(f"починено {fixed}, кэш: {len(cache)} записей -> {CACHE}")
|
||
else:
|
||
for r in bad:
|
||
print(f" пусто {r['name']:26s} {r['extra']}")
|
||
return results, cache |