omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/model.py

@@ -0,0 +1,99 @@
+"""Модель записи: единая схема для всех источников + simhash + разбор цены."""
+import hashlib
+import re
+import urllib.parse
+
+PRICE_RE = re.compile(r"(?:(?:[$€£₽¥])\s?(\d{1,6}(?:[.,]\d{1,2})?)|(\d{1,6}(?:[.,]\d{1,2})?)\s?(?:usd|eur|rub|грн|\$|€|₽|руб))",
+                      re.IGNORECASE)
+CYR = re.compile(r"[а-яё]", re.IGNORECASE)
+TAGS = ("abuse", "proxy", "combo", "api", "jailbreak", "leak", "traffic", "crypto", "tg", "bot")
+
+
+def canon(url):
+    if not url:
+        return ""
+    u = urllib.parse.urlsplit(url.strip())
+    q = [(k, v) for k, v in urllib.parse.parse_qsl(u.query) if not k.startswith(("utm_", "ref", "fbclid"))]
+    path = u.path.rstrip("/") or "/"
+    return urllib.parse.urlunsplit((u.scheme or "https", u.netloc.lower(), path, urllib.parse.urlencode(q), ""))
+
+
+def shingles(text, k=4):
+    words = re.findall(r"[\w]+", (text or "").lower())[:400]
+    if len(words) < k:
+        return {" ".join(words)} if words else set()
+    return {" ".join(words[i:i + k]) for i in range(len(words) - k + 1)}
+
+
+def simhash(text):
+    bits = [0] * 64
+    for sh in shingles(text):
+        h = int(hashlib.blake2b(sh.encode(), digest_size=8).hexdigest(), 16)
+        for i in range(64):
+            bits[i] += 1 if (h >> i) & 1 else -1
+    out = 0
+    for i in range(64):
+        if bits[i] > 0:
+            out |= 1 << i
+    return out
+
+
+def price_of(text):
+    m = PRICE_RE.search(text or "")
+    if not m:
+        return None, None
+    raw = (m.group(1) or m.group(2) or "").replace(",", ".")
+    try:
+        val = float(raw)
+    except Exception:
+        return None, None
+    if val <= 0 or val > 1_000_000:
+        return None, None
+    cur = ""
+    frag = m.group(0).lower()
+    for sym, code in (("$", "USD"), ("€", "EUR"), ("₽", "RUB"), ("£", "GBP"), ("usd", "USD"),
+                      ("eur", "EUR"), ("rub", "RUB"), ("руб", "RUB"), ("грн", "UAH"), ("¥", "CNY")):
+        if sym in frag:
+            cur = code
+            break
+    return val, cur
+
+
+def lang_of(text):
+    t = (text or "")[:400]
+    cyr = len(CYR.findall(t))
+    return "ru" if cyr > max(6, len(t) * 0.05) else "en"
+
+
+class Item:
+    __slots__ = ("provider", "source", "kind", "url", "title", "text", "author", "published",
+                 "score", "price", "currency", "lang", "tags", "extra", "h")
+
+    def __init__(self, provider, kind, url="", title="", text="", source="", author="",
+                 published="", score=0.0, extra=None):
+        self.provider = provider
+        self.kind = kind
+        self.url = url
+        self.title = (title or "").strip()[:400]
+        self.text = (text or "").strip()[:6000]
+        self.source = source or urllib.parse.urlsplit(url).netloc
+        self.author = author or ""
+        self.published = (published or "")[:32]
+        self.score = float(score or 0)
+        self.lang = lang_of(self.title + " " + self.text)
+        body = (self.title + " " + self.text).lower()
+        self.tags = [t for t in TAGS if t in body][:6]
+        self.price, self.currency = price_of(self.title + " " + self.text)
+        self.extra = extra or {}
+        self.h = simhash(self.title + " " + self.text) if self.text or self.title else 0
+        self.url = canon(self.url)
+
+    def key(self):
+        return hashlib.sha1((self.provider + "|" + (self.url or self.title + self.text[:120]))
+                            .encode("utf-8", "ignore")).hexdigest()
+
+    def as_dict(self):
+        return {k: getattr(self, k) for k in self.__slots__ if k != "h"} | {"h": self.h}
+
+    def __repr__(self):
+        return f"<{self.provider}/{self.kind} {self.title[:48]!r}>"