абузы: free-панели new-api на вкладке (роут /api/panels + грид), CSS вынесен из JS, панельный классификатор даёт freeapi-вес, тесты

omni/feeds.py

@@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET
 from bs4 import BeautifulSoup
 
 from . import config, net
+from .dates import parse_published
 from .model import Item
 
 UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
@@ -38,8 +39,9 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None):
         m = re.search(r"CVE-\d{4}-\d{4,7}", body or "")
         if m:
             t = "%s · %s" % (m.group(0), t[:120])
+    dt = parse_published(published)
     return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
-                author=_txt(author)[:80], published=str(published or ""), source=src.name,
+                author=_txt(author)[:80], published=dt.isoformat(timespec="seconds") if dt else str(published or ""), source=src.name,
                 extra=ex)
 
 
@@ -100,17 +102,26 @@ def _rss(src, limit):
         root = ET.fromstring(r.text.encode("utf-8", "ignore"))
     except Exception:
         return _rss_regex(src, r.text, limit)
-    ns = {"a": "http://www.w3.org/2005/Atom"}
+    ns = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/elements/1.1/"}
     entries = root.findall(".//item") or root.findall(".//a:entry", ns)
     for e in entries[:limit]:
-        g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "")
+        def g(tag):
+            direct = e.findtext(tag, namespaces=ns) or ""
+            if direct or ":" in tag:
+                return direct
+            return e.findtext("a:" + tag, namespaces=ns) or ""
+
         link = g("link")
         if not link:
-            le = e.find("a:link", ns) or e.find("link")
-            link = le.get("href") if le is not None and le.get("href") else ""
+            links = list(e.findall("a:link", ns)) + list(e.findall("link"))
+            le = next((el for el in links if el.get("rel", "alternate") == "alternate" and el.get("href")), None)
+            link = le.get("href", "") if le is not None else ""
+        if link:
+            link = urllib.parse.urljoin(src.url, link)
         body = g("description") or g("summary") or g("content")
+        author = e.findtext("a:author/a:name", namespaces=ns) or g("author") or g("dc:creator")
         out.append(_item(src, "entry", link, g("title"), body,
-                         g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated")))
+                         author, g("pubDate") or g("published") or g("updated")))
     if not out and "arxiv.org" in src.url:
         return _arxiv_api(src, limit)
     return out
@@ -910,18 +921,52 @@ ADAPTERS = {
 }
 
 
+_CHALLENGE_TITLE = re.compile(
+    r"(?:just a moment|access denied|403 forbidden|404(?: not found)?|not found|"
+    r"verify (?:that )?you are human|checking your browser|"
+    r"attention required!?(?:\s*\|\s*cloudflare)?|please enable javascript)[.!…\s]*",
+    re.I,
+)
+
+
+def _valid_items(items):
+    """Drop broken records and obvious challenge pages, not short real posts."""
+    out, seen = [], set()
+    for it in items:
+        if not isinstance(it, Item) or any(not isinstance(getattr(it, field, None), str)
+                                          for field in ("url", "title", "text")):
+            continue
+        try:
+            u = urllib.parse.urlsplit(it.url)
+            valid_url = u.scheme in ("http", "https") and bool(u.hostname)
+        except ValueError:
+            valid_url = False
+        if not valid_url or not (it.title.strip() or it.text.strip()):
+            continue
+        if _CHALLENGE_TITLE.fullmatch(it.title.strip()) and (
+                len(it.text.strip()) < 160 or re.search(
+                    r"cloudflare|captcha|verify you are human|checking your browser|just a moment|attention required",
+                    it.text, re.I)):
+            continue
+        if it.url in seen:
+            continue
+        seen.add(it.url)
+        out.append(it)
+    return out
+
+
 def fetch(src, limit=None):
     fn = ADAPTERS.get(src.adapter)
     if not fn:
         return []
     n = limit or src.limit
     try:
-        return fn(src, n) or []
+        return _valid_items(fn(src, n) or [])
     except Exception:
         return []
 
 
-SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co")
+SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co", "services.nvd.nist.gov")
 
 
 def fetch_many(sources, limit=None, workers=8):
@@ -944,5 +989,6 @@ def fetch_many(sources, limit=None, workers=8):
             got = fetch(s, limit)
         report.append((s.name, len(got)))
         items.extend(got)
-        time.sleep(3.5 if "reddit.com" in s.url else 1.0)
+        delay = 6.5 if "services.nvd.nist.gov" in s.url else (3.5 if "reddit.com" in s.url else 1.0)
+        time.sleep(delay)
     return items, report