omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/providers/cards.py

@@ -0,0 +1,102 @@
+"""Карточки товаров: JSON-LD Product/Offer, og-теги, микроразметка, fallback по цене."""
+import json
+import re
+
+from .. import net
+from ..model import Item, price_of
+
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+JSONLD = re.compile(r'<script[^>]+type="application/ld\+json"[^>]*>(.*?)</script>', re.S)
+
+RULES = [
+    ("title", [r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"', r"<title>(.*?)</title>"]),
+    ("price", [r'<meta[^>]+property="product:price:amount"[^>]+content="([^"]+)"',
+               r'"price"\s*:\s*"?([0-9][0-9.,]*)"?']),
+    ("currency", [r'<meta[^>]+property="product:price:currency"[^>]+content="([^"]+)"',
+                  r'"priceCurrency"\s*:\s*"([A-Z]{3})"']),
+    ("avail", [r'"availability"\s*:\s*"[^"]*?([A-Za-z]+)"?\s*$']),
+]
+
+
+def available(name="cards"):
+    return True, "забирает карточку по URL, ключи не нужны"
+
+
+def _ld_product(html_text):
+    for m in JSONLD.finditer(html_text):
+        raw = m.group(1).strip()
+        try:
+            data = json.loads(raw)
+        except Exception:
+            continue
+        stack = data if isinstance(data, list) else [data]
+        while stack:
+            node = stack.pop()
+            if isinstance(node, dict):
+                if node.get("@graph"):
+                    stack.extend(node["@graph"])
+                t = node.get("@type")
+                types = t if isinstance(t, list) else [t]
+                if any(str(x).lower() in ("product", "offer") for x in types if x):
+                    yield node
+                for v in node.values():
+                    if isinstance(v, (dict, list)):
+                        stack.append(v)
+            elif isinstance(node, list):
+                stack.extend(node)
+
+
+def _num(s):
+    try:
+        return float(str(s).replace(",", ".").replace(" ", ""))
+    except Exception:
+        return None
+
+
+def extract_cards(urls, limit=10):
+    items = []
+    for u in (urls if isinstance(urls, (list, tuple)) else [urls])[:limit]:
+        r = net.get(u, headers=UA)
+        if r.status != 200:
+            continue
+        title = ""
+        price = cur = ""
+        avail = ""
+        for prod in _ld_product(r.text):
+            offers = prod.get("offers") or {}
+            if isinstance(offers, list):
+                offers = offers[0] if offers else {}
+            price = offers.get("price") or offers.get("lowPrice") or price
+            cur = offers.get("priceCurrency") or cur
+            avail = (offers.get("availability") or avail)
+            title = prod.get("name") or title
+            if price:
+                break
+        for field, pats in RULES:
+            for p in pats:
+                m = re.search(p, r.text, re.I | re.S)
+                if not m:
+                    continue
+                val = re.sub(r"<[^>]+>", "", m.group(1)).strip()
+                if field == "title" and not title:
+                    title = val
+                elif field == "price" and not price:
+                    price = val
+                elif field == "currency" and not cur:
+                    cur = val
+                elif field == "avail" and not avail:
+                    avail = val
+                break
+        text = r.text
+        body = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", text, flags=re.S)
+        body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", body))[:2500]
+        pval = _num(price)
+        if not pval:
+            pval, cur2 = price_of(body)
+            cur = cur or (cur2 or "")
+        it = Item("cards", "card", url=u, title=title[:300], text=body, source=u.split("/")[2] if "://" in u else "",
+                  extra={"availability": avail, "raw_price": price})
+        it.price = pval
+        it.currency = (cur or "")[:4].upper() or None
+        items.append(it)
+    return items