164 lines
6.6 KiB
Python
164 lines
6.6 KiB
Python
"""Автодобавление источников: находит фиды по домашним страницам и каталогам.
|
|
|
|
Логика: берём затравки (хосты уже подключённых источников + известные каталоги
|
|
сообществ), вытаскиваем ссылки на фиды (link rel=alternate, /feed, /rss,
|
|
/latest.json, подфорумы), прогоняем каждый кандидат реальным адаптером и
|
|
добавляем только те, что отдали записи. Всё добавленное помечается auto и
|
|
попадает в панель как обычный источник.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.parse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from . import feeds, net, source_state, sources
|
|
|
|
FEED_PATHS = ("/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml",
|
|
"/latest.json", "/?feed=rss2", "/feed/", "/blog/feed")
|
|
SEED_CATALOGS = [
|
|
# каталоги сообществ и подфорумов, откуда удобно цеплять новые фиды
|
|
"https://www.reddit.com/r/LocalLLaMA/about.json",
|
|
"https://news.ycombinator.com/",
|
|
"https://lobste.rs/",
|
|
]
|
|
UA = getattr(feeds, "UA", {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
|
|
|
|
|
|
def _hosts():
|
|
seen = {}
|
|
for s in sources.all_sources():
|
|
try:
|
|
h = urllib.parse.urlsplit(s.url).netloc.lower()
|
|
except Exception:
|
|
continue
|
|
if not h or h in seen:
|
|
continue
|
|
seen[h] = s.category
|
|
return seen
|
|
|
|
|
|
def _feed_links(html, base):
|
|
out = []
|
|
soup = BeautifulSoup(html, "lxml")
|
|
for ln in soup.find_all("link", href=True):
|
|
t = (ln.get("type") or "").lower()
|
|
rel = " ".join(ln.get("rel") or []).lower()
|
|
if "alternate" in rel and ("rss" in t or "atom" in t or "json" in t):
|
|
out.append(urllib.parse.urljoin(base, ln["href"]))
|
|
for a in soup.find_all("a", href=True):
|
|
href = str(a["href"])
|
|
if re.search(r"(/feed|/rss|\.rss|atom\.xml|/latest\.json)$", href):
|
|
out.append(urllib.parse.urljoin(base, href))
|
|
return out
|
|
|
|
|
|
def candidates(limit_hosts=60, extra_seeds=()):
|
|
hosts = _hosts()
|
|
seeds = []
|
|
for h, cat in list(hosts.items())[:limit_hosts]:
|
|
seeds.append(("https://%s/" % h, cat))
|
|
for u in list(extra_seeds) + SEED_CATALOGS:
|
|
seeds.append((u, "auto"))
|
|
return seeds
|
|
|
|
|
|
def probe(url, category="auto", limit=6):
|
|
"""Проверка одного URL реальным адаптером. Возвращает (ok, n, adapter)."""
|
|
low = url.lower()
|
|
if "t.me/s/" in low:
|
|
ad, kind = "tg_web", "html"
|
|
elif low.endswith(".json") or "/latest.json" in low:
|
|
ad, kind = ("discourse", "json") if "/latest.json" in low else ("generic_json", "json")
|
|
elif any(x in low for x in (".rss", "/rss", "/feed", "atom.xml", "feed.xml", "index.xml")):
|
|
ad, kind = "rss", "rss"
|
|
else:
|
|
ad, kind = "html_listing", "html"
|
|
src = sources.Source(name="probe", category=category, kind=kind, url=url,
|
|
adapter=ad, tags=("auto",), limit=limit, detail=False, extra={})
|
|
fn = feeds.ADAPTERS.get(ad)
|
|
if not fn:
|
|
return False, 0, ad
|
|
try:
|
|
items = fn(src, limit) or []
|
|
except Exception:
|
|
return False, 0, ad
|
|
return len(items) > 0, len(items), ad
|
|
|
|
|
|
def discover(limit_hosts=40, workers=8, max_add=40, dry=False):
|
|
seeds = candidates(limit_hosts=limit_hosts)
|
|
found = []
|
|
def scan(seed):
|
|
base, cat = seed
|
|
try:
|
|
r = net.get(base, headers=UA)
|
|
if r.status != 200:
|
|
return []
|
|
except Exception:
|
|
return []
|
|
links = _feed_links(r.text[:400000], base)
|
|
dom = urllib.parse.urlsplit(base)
|
|
for p in FEED_PATHS:
|
|
links.append("%s://%s%s" % (dom.scheme, dom.netloc, p))
|
|
# подфорумы Discourse/Reddit
|
|
if "reddit.com" in base:
|
|
for sub in ("LocalLLaMA", "MachineLearning", "mcp", "selfhosted"):
|
|
links.append("https://www.reddit.com/r/%s/.rss" % sub)
|
|
return [(u, cat) for u in dict.fromkeys(links)]
|
|
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
for res in ex.map(scan, seeds):
|
|
found.extend(res)
|
|
print("кандидатов найдено:", len(found), flush=True)
|
|
|
|
known = {s.url for s in sources.all_sources()}
|
|
found = [(u, c) for u, c in found if u not in known]
|
|
print("после дедупа:", len(found), flush=True)
|
|
|
|
def check(pair):
|
|
url, cat = pair
|
|
ok, n, ad = probe(url, cat)
|
|
return url, cat, ok, n, ad
|
|
|
|
# только машинные фиды: html-листинги на автодобавлении дают мусор
|
|
best = {}
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
for url, cat, ok, n, ad in ex.map(check, found):
|
|
if not ok or ad == "html_listing" or n < 3:
|
|
continue
|
|
dom = urllib.parse.urlsplit(url).netloc.lower()
|
|
key = (dom, ad)
|
|
if key not in best or n > best[key][2]:
|
|
best[key] = (url, cat, n, ad)
|
|
|
|
added = []
|
|
for (dom, ad), (url, cat, n, _ad) in sorted(best.items(), key=lambda kv: -kv[1][2]):
|
|
name = "auto_" + re.sub(r"[^a-z0-9]+", "_", dom).strip("_")[:34]
|
|
if any(a["name"] == name for a in added):
|
|
continue
|
|
rec = {"name": name, "category": cat if cat in ("abuse", "forum", "security") else "community",
|
|
"kind": "rss" if ad == "rss" else ("json" if ad in ("generic_json", "discourse") else "html"),
|
|
"url": url, "adapter": ad, "tags": ["auto"], "lim": 15, "detail": False}
|
|
added.append(rec)
|
|
if len(added) >= max_add:
|
|
break
|
|
print("живых фидов:", len(added), flush=True)
|
|
if not dry:
|
|
for rec in added:
|
|
source_state.add_custom(rec["name"], rec["category"], rec["url"], kind=rec["kind"],
|
|
adapter=rec["adapter"], tags=rec["tags"], limit=rec["lim"],
|
|
detail=rec["detail"])
|
|
return {"found": len(found), "alive": len(added), "added": added if dry else [a["name"] for a in added]}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
lim = int(sys.argv[1]) if len(sys.argv) > 1 else 30
|
|
res = discover(limit_hosts=lim)
|
|
print(json.dumps({"found": res["found"], "alive": res["alive"], "added": res["added"][:20]},
|
|
ensure_ascii=False, indent=1)) |