omni: абузы (31 живой источник + классификатор), авто-добавление фидов, Telegram-бот, вкладка абузов; 589 источников

omni/discover.py

@@ -0,0 +1,164 @@
+"""Автодобавление источников: находит фиды по домашним страницам и каталогам.
+
+Логика: берём затравки (хосты уже подключённых источников + известные каталоги
+сообществ), вытаскиваем ссылки на фиды (link rel=alternate, /feed, /rss,
+/latest.json, подфорумы), прогоняем каждый кандидат реальным адаптером и
+добавляем только те, что отдали записи. Всё добавленное помечается auto и
+попадает в панель как обычный источник.
+"""
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.parse
+from concurrent.futures import ThreadPoolExecutor
+
+from bs4 import BeautifulSoup
+
+from . import feeds, net, source_state, sources
+
+FEED_PATHS = ("/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml",
+              "/latest.json", "/?feed=rss2", "/feed/", "/blog/feed")
+SEED_CATALOGS = [
+    # каталоги сообществ и подфорумов, откуда удобно цеплять новые фиды
+    "https://www.reddit.com/r/LocalLLaMA/about.json",
+    "https://news.ycombinator.com/",
+    "https://lobste.rs/",
+]
+UA = getattr(feeds, "UA", {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
+
+
+def _hosts():
+    seen = {}
+    for s in sources.all_sources():
+        try:
+            h = urllib.parse.urlsplit(s.url).netloc.lower()
+        except Exception:
+            continue
+        if not h or h in seen:
+            continue
+        seen[h] = s.category
+    return seen
+
+
+def _feed_links(html, base):
+    out = []
+    soup = BeautifulSoup(html, "lxml")
+    for ln in soup.find_all("link", href=True):
+        t = (ln.get("type") or "").lower()
+        rel = " ".join(ln.get("rel") or []).lower()
+        if "alternate" in rel and ("rss" in t or "atom" in t or "json" in t):
+            out.append(urllib.parse.urljoin(base, ln["href"]))
+    for a in soup.find_all("a", href=True):
+        href = str(a["href"])
+        if re.search(r"(/feed|/rss|\.rss|atom\.xml|/latest\.json)$", href):
+            out.append(urllib.parse.urljoin(base, href))
+    return out
+
+
+def candidates(limit_hosts=60, extra_seeds=()):
+    hosts = _hosts()
+    seeds = []
+    for h, cat in list(hosts.items())[:limit_hosts]:
+        seeds.append(("https://%s/" % h, cat))
+    for u in list(extra_seeds) + SEED_CATALOGS:
+        seeds.append((u, "auto"))
+    return seeds
+
+
+def probe(url, category="auto", limit=6):
+    """Проверка одного URL реальным адаптером. Возвращает (ok, n, adapter)."""
+    low = url.lower()
+    if "t.me/s/" in low:
+        ad, kind = "tg_web", "html"
+    elif low.endswith(".json") or "/latest.json" in low:
+        ad, kind = ("discourse", "json") if "/latest.json" in low else ("generic_json", "json")
+    elif any(x in low for x in (".rss", "/rss", "/feed", "atom.xml", "feed.xml", "index.xml")):
+        ad, kind = "rss", "rss"
+    else:
+        ad, kind = "html_listing", "html"
+    src = sources.Source(name="probe", category=category, kind=kind, url=url,
+                         adapter=ad, tags=("auto",), limit=limit, detail=False, extra={})
+    fn = feeds.ADAPTERS.get(ad)
+    if not fn:
+        return False, 0, ad
+    try:
+        items = fn(src, limit) or []
+    except Exception:
+        return False, 0, ad
+    return len(items) > 0, len(items), ad
+
+
+def discover(limit_hosts=40, workers=8, max_add=40, dry=False):
+    seeds = candidates(limit_hosts=limit_hosts)
+    found = []
+    def scan(seed):
+        base, cat = seed
+        try:
+            r = net.get(base, headers=UA)
+            if r.status != 200:
+                return []
+        except Exception:
+            return []
+        links = _feed_links(r.text[:400000], base)
+        dom = urllib.parse.urlsplit(base)
+        for p in FEED_PATHS:
+            links.append("%s://%s%s" % (dom.scheme, dom.netloc, p))
+        # подфорумы Discourse/Reddit
+        if "reddit.com" in base:
+            for sub in ("LocalLLaMA", "MachineLearning", "mcp", "selfhosted"):
+                links.append("https://www.reddit.com/r/%s/.rss" % sub)
+        return [(u, cat) for u in dict.fromkeys(links)]
+
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        for res in ex.map(scan, seeds):
+            found.extend(res)
+    print("кандидатов найдено:", len(found), flush=True)
+
+    known = {s.url for s in sources.all_sources()}
+    found = [(u, c) for u, c in found if u not in known]
+    print("после дедупа:", len(found), flush=True)
+
+    def check(pair):
+        url, cat = pair
+        ok, n, ad = probe(url, cat)
+        return url, cat, ok, n, ad
+
+    # только машинные фиды: html-листинги на автодобавлении дают мусор
+    best = {}
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        for url, cat, ok, n, ad in ex.map(check, found):
+            if not ok or ad == "html_listing" or n < 3:
+                continue
+            dom = urllib.parse.urlsplit(url).netloc.lower()
+            key = (dom, ad)
+            if key not in best or n > best[key][2]:
+                best[key] = (url, cat, n, ad)
+
+    added = []
+    for (dom, ad), (url, cat, n, _ad) in sorted(best.items(), key=lambda kv: -kv[1][2]):
+        name = "auto_" + re.sub(r"[^a-z0-9]+", "_", dom).strip("_")[:34]
+        if any(a["name"] == name for a in added):
+            continue
+        rec = {"name": name, "category": cat if cat in ("abuse", "forum", "security") else "community",
+               "kind": "rss" if ad == "rss" else ("json" if ad in ("generic_json", "discourse") else "html"),
+               "url": url, "adapter": ad, "tags": ["auto"], "lim": 15, "detail": False}
+        added.append(rec)
+        if len(added) >= max_add:
+            break
+    print("живых фидов:", len(added), flush=True)
+    if not dry:
+        for rec in added:
+            source_state.add_custom(rec["name"], rec["category"], rec["url"], kind=rec["kind"],
+                                    adapter=rec["adapter"], tags=rec["tags"], limit=rec["lim"],
+                                    detail=rec["detail"])
+    return {"found": len(found), "alive": len(added), "added": added if dry else [a["name"] for a in added]}
+
+
+if __name__ == "__main__":
+    import sys
+    lim = int(sys.argv[1]) if len(sys.argv) > 1 else 30
+    res = discover(limit_hosts=lim)
+    print(json.dumps({"found": res["found"], "alive": res["alive"], "added": res["added"][:20]},
+                     ensure_ascii=False, indent=1))