99 lines
3.6 KiB
Python
99 lines
3.6 KiB
Python
"""Модель записи: единая схема для всех источников + simhash + разбор цены."""
|
||
import hashlib
|
||
import re
|
||
import urllib.parse
|
||
|
||
PRICE_RE = re.compile(r"(?:(?:[$€£₽¥])\s?(\d{1,6}(?:[.,]\d{1,2})?)|(\d{1,6}(?:[.,]\d{1,2})?)\s?(?:usd|eur|rub|грн|\$|€|₽|руб))",
|
||
re.IGNORECASE)
|
||
CYR = re.compile(r"[а-яё]", re.IGNORECASE)
|
||
TAGS = ("abuse", "proxy", "combo", "api", "jailbreak", "leak", "traffic", "crypto", "tg", "bot")
|
||
|
||
|
||
def canon(url):
|
||
if not url:
|
||
return ""
|
||
u = urllib.parse.urlsplit(url.strip())
|
||
q = [(k, v) for k, v in urllib.parse.parse_qsl(u.query) if not k.startswith(("utm_", "ref", "fbclid"))]
|
||
path = u.path.rstrip("/") or "/"
|
||
return urllib.parse.urlunsplit((u.scheme or "https", u.netloc.lower(), path, urllib.parse.urlencode(q), ""))
|
||
|
||
|
||
def shingles(text, k=4):
|
||
words = re.findall(r"[\w]+", (text or "").lower())[:400]
|
||
if len(words) < k:
|
||
return {" ".join(words)} if words else set()
|
||
return {" ".join(words[i:i + k]) for i in range(len(words) - k + 1)}
|
||
|
||
|
||
def simhash(text):
|
||
bits = [0] * 64
|
||
for sh in shingles(text):
|
||
h = int(hashlib.blake2b(sh.encode(), digest_size=8).hexdigest(), 16)
|
||
for i in range(64):
|
||
bits[i] += 1 if (h >> i) & 1 else -1
|
||
out = 0
|
||
for i in range(64):
|
||
if bits[i] > 0:
|
||
out |= 1 << i
|
||
return out
|
||
|
||
|
||
def price_of(text):
|
||
m = PRICE_RE.search(text or "")
|
||
if not m:
|
||
return None, None
|
||
raw = (m.group(1) or m.group(2) or "").replace(",", ".")
|
||
try:
|
||
val = float(raw)
|
||
except Exception:
|
||
return None, None
|
||
if val <= 0 or val > 1_000_000:
|
||
return None, None
|
||
cur = ""
|
||
frag = m.group(0).lower()
|
||
for sym, code in (("$", "USD"), ("€", "EUR"), ("₽", "RUB"), ("£", "GBP"), ("usd", "USD"),
|
||
("eur", "EUR"), ("rub", "RUB"), ("руб", "RUB"), ("грн", "UAH"), ("¥", "CNY")):
|
||
if sym in frag:
|
||
cur = code
|
||
break
|
||
return val, cur
|
||
|
||
|
||
def lang_of(text):
|
||
t = (text or "")[:400]
|
||
cyr = len(CYR.findall(t))
|
||
return "ru" if cyr > max(6, len(t) * 0.05) else "en"
|
||
|
||
|
||
class Item:
|
||
__slots__ = ("provider", "source", "kind", "url", "title", "text", "author", "published",
|
||
"score", "price", "currency", "lang", "tags", "extra", "h")
|
||
|
||
def __init__(self, provider, kind, url="", title="", text="", source="", author="",
|
||
published="", score=0.0, extra=None):
|
||
self.provider = provider
|
||
self.kind = kind
|
||
self.url = url
|
||
self.title = (title or "").strip()[:400]
|
||
self.text = (text or "").strip()[:6000]
|
||
self.source = source or urllib.parse.urlsplit(url).netloc
|
||
self.author = author or ""
|
||
self.published = (published or "")[:32]
|
||
self.score = float(score or 0)
|
||
self.lang = lang_of(self.title + " " + self.text)
|
||
body = (self.title + " " + self.text).lower()
|
||
self.tags = [t for t in TAGS if t in body][:6]
|
||
self.price, self.currency = price_of(self.title + " " + self.text)
|
||
self.extra = extra or {}
|
||
self.h = simhash(self.title + " " + self.text) if self.text or self.title else 0
|
||
self.url = canon(self.url)
|
||
|
||
def key(self):
|
||
return hashlib.sha1((self.provider + "|" + (self.url or self.title + self.text[:120]))
|
||
.encode("utf-8", "ignore")).hexdigest()
|
||
|
||
def as_dict(self):
|
||
return {k: getattr(self, k) for k in self.__slots__ if k != "h"} | {"h": self.h}
|
||
|
||
def __repr__(self):
|
||
return f"<{self.provider}/{self.kind} {self.title[:48]!r}>" |