102 lines
3.8 KiB
Python
102 lines
3.8 KiB
Python
"""Карточки товаров: JSON-LD Product/Offer, og-теги, микроразметка, fallback по цене."""
|
|
import json
|
|
import re
|
|
|
|
from .. import net
|
|
from ..model import Item, price_of
|
|
|
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
|
|
JSONLD = re.compile(r'<script[^>]+type="application/ld\+json"[^>]*>(.*?)</script>', re.S)
|
|
|
|
RULES = [
|
|
("title", [r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"', r"<title>(.*?)</title>"]),
|
|
("price", [r'<meta[^>]+property="product:price:amount"[^>]+content="([^"]+)"',
|
|
r'"price"\s*:\s*"?([0-9][0-9.,]*)"?']),
|
|
("currency", [r'<meta[^>]+property="product:price:currency"[^>]+content="([^"]+)"',
|
|
r'"priceCurrency"\s*:\s*"([A-Z]{3})"']),
|
|
("avail", [r'"availability"\s*:\s*"[^"]*?([A-Za-z]+)"?\s*$']),
|
|
]
|
|
|
|
|
|
def available(name="cards"):
|
|
return True, "забирает карточку по URL, ключи не нужны"
|
|
|
|
|
|
def _ld_product(html_text):
|
|
for m in JSONLD.finditer(html_text):
|
|
raw = m.group(1).strip()
|
|
try:
|
|
data = json.loads(raw)
|
|
except Exception:
|
|
continue
|
|
stack = data if isinstance(data, list) else [data]
|
|
while stack:
|
|
node = stack.pop()
|
|
if isinstance(node, dict):
|
|
if node.get("@graph"):
|
|
stack.extend(node["@graph"])
|
|
t = node.get("@type")
|
|
types = t if isinstance(t, list) else [t]
|
|
if any(str(x).lower() in ("product", "offer") for x in types if x):
|
|
yield node
|
|
for v in node.values():
|
|
if isinstance(v, (dict, list)):
|
|
stack.append(v)
|
|
elif isinstance(node, list):
|
|
stack.extend(node)
|
|
|
|
|
|
def _num(s):
|
|
try:
|
|
return float(str(s).replace(",", ".").replace(" ", ""))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def extract_cards(urls, limit=10):
|
|
items = []
|
|
for u in (urls if isinstance(urls, (list, tuple)) else [urls])[:limit]:
|
|
r = net.get(u, headers=UA)
|
|
if r.status != 200:
|
|
continue
|
|
title = ""
|
|
price = cur = ""
|
|
avail = ""
|
|
for prod in _ld_product(r.text):
|
|
offers = prod.get("offers") or {}
|
|
if isinstance(offers, list):
|
|
offers = offers[0] if offers else {}
|
|
price = offers.get("price") or offers.get("lowPrice") or price
|
|
cur = offers.get("priceCurrency") or cur
|
|
avail = (offers.get("availability") or avail)
|
|
title = prod.get("name") or title
|
|
if price:
|
|
break
|
|
for field, pats in RULES:
|
|
for p in pats:
|
|
m = re.search(p, r.text, re.I | re.S)
|
|
if not m:
|
|
continue
|
|
val = re.sub(r"<[^>]+>", "", m.group(1)).strip()
|
|
if field == "title" and not title:
|
|
title = val
|
|
elif field == "price" and not price:
|
|
price = val
|
|
elif field == "currency" and not cur:
|
|
cur = val
|
|
elif field == "avail" and not avail:
|
|
avail = val
|
|
break
|
|
text = r.text
|
|
body = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", text, flags=re.S)
|
|
body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", body))[:2500]
|
|
pval = _num(price)
|
|
if not pval:
|
|
pval, cur2 = price_of(body)
|
|
cur = cur or (cur2 or "")
|
|
it = Item("cards", "card", url=u, title=title[:300], text=body, source=u.split("/")[2] if "://" in u else "",
|
|
extra={"availability": avail, "raw_price": price})
|
|
it.price = pval
|
|
it.currency = (cur or "")[:4].upper() or None
|
|
items.append(it)
|
|
return items |