omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/providers/__init__.py

@@ -0,0 +1,49 @@
+"""Реестр провайдеров: имя -> (search, available, note)."""
+from . import cards, forums, reddit, searchers, treg, x
+
+MODULES = {
+    "reddit": reddit,
+    "x": x,
+    "forums": forums,
+    "tavily": searchers,
+    "exa": searchers,
+    "you": searchers,
+    "firecrawl": searchers,
+    "ddg": searchers,
+    "mojeek": searchers,
+    "bing": searchers,
+    "cards": cards,
+    "treg": treg,
+}
+
+# какой функцией звать внутри модуля
+FUNCS = {
+    "reddit": "search_reddit",
+    "x": "search_x",
+    "forums": "search_forums",
+    "tavily": "search_tavily",
+    "exa": "search_exa",
+    "you": "search_you",
+    "firecrawl": "search_firecrawl",
+    "ddg": "search_ddg",
+    "mojeek": "search_mojeek",
+    "bing": "search_bing",
+    "cards": "extract_cards",
+    "treg": "search_treg",
+}
+
+
+def get(name):
+    mod = MODULES.get(name)
+    if not mod:
+        return None
+    return getattr(mod, FUNCS[name], None)
+
+
+def status():
+    rows = []
+    for name, mod in MODULES.items():
+        fn = getattr(mod, "available", None)
+        ok, note = fn(name) if callable(fn) else (True, "")
+        rows.append((name, ok, note))
+    return rows

omni/providers/cards.py

@@ -0,0 +1,102 @@
+"""Карточки товаров: JSON-LD Product/Offer, og-теги, микроразметка, fallback по цене."""
+import json
+import re
+
+from .. import net
+from ..model import Item, price_of
+
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+JSONLD = re.compile(r'<script[^>]+type="application/ld\+json"[^>]*>(.*?)</script>', re.S)
+
+RULES = [
+    ("title", [r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"', r"<title>(.*?)</title>"]),
+    ("price", [r'<meta[^>]+property="product:price:amount"[^>]+content="([^"]+)"',
+               r'"price"\s*:\s*"?([0-9][0-9.,]*)"?']),
+    ("currency", [r'<meta[^>]+property="product:price:currency"[^>]+content="([^"]+)"',
+                  r'"priceCurrency"\s*:\s*"([A-Z]{3})"']),
+    ("avail", [r'"availability"\s*:\s*"[^"]*?([A-Za-z]+)"?\s*$']),
+]
+
+
+def available(name="cards"):
+    return True, "забирает карточку по URL, ключи не нужны"
+
+
+def _ld_product(html_text):
+    for m in JSONLD.finditer(html_text):
+        raw = m.group(1).strip()
+        try:
+            data = json.loads(raw)
+        except Exception:
+            continue
+        stack = data if isinstance(data, list) else [data]
+        while stack:
+            node = stack.pop()
+            if isinstance(node, dict):
+                if node.get("@graph"):
+                    stack.extend(node["@graph"])
+                t = node.get("@type")
+                types = t if isinstance(t, list) else [t]
+                if any(str(x).lower() in ("product", "offer") for x in types if x):
+                    yield node
+                for v in node.values():
+                    if isinstance(v, (dict, list)):
+                        stack.append(v)
+            elif isinstance(node, list):
+                stack.extend(node)
+
+
+def _num(s):
+    try:
+        return float(str(s).replace(",", ".").replace(" ", ""))
+    except Exception:
+        return None
+
+
+def extract_cards(urls, limit=10):
+    items = []
+    for u in (urls if isinstance(urls, (list, tuple)) else [urls])[:limit]:
+        r = net.get(u, headers=UA)
+        if r.status != 200:
+            continue
+        title = ""
+        price = cur = ""
+        avail = ""
+        for prod in _ld_product(r.text):
+            offers = prod.get("offers") or {}
+            if isinstance(offers, list):
+                offers = offers[0] if offers else {}
+            price = offers.get("price") or offers.get("lowPrice") or price
+            cur = offers.get("priceCurrency") or cur
+            avail = (offers.get("availability") or avail)
+            title = prod.get("name") or title
+            if price:
+                break
+        for field, pats in RULES:
+            for p in pats:
+                m = re.search(p, r.text, re.I | re.S)
+                if not m:
+                    continue
+                val = re.sub(r"<[^>]+>", "", m.group(1)).strip()
+                if field == "title" and not title:
+                    title = val
+                elif field == "price" and not price:
+                    price = val
+                elif field == "currency" and not cur:
+                    cur = val
+                elif field == "avail" and not avail:
+                    avail = val
+                break
+        text = r.text
+        body = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", text, flags=re.S)
+        body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", body))[:2500]
+        pval = _num(price)
+        if not pval:
+            pval, cur2 = price_of(body)
+            cur = cur or (cur2 or "")
+        it = Item("cards", "card", url=u, title=title[:300], text=body, source=u.split("/")[2] if "://" in u else "",
+                  extra={"availability": avail, "raw_price": price})
+        it.price = pval
+        it.currency = (cur or "")[:4].upper() or None
+        items.append(it)
+    return items

omni/providers/forums.py

@@ -0,0 +1,151 @@
+"""Форумы без ключей: Discourse, HN (Algolia), Lobsters, XenForo, Telegram-веб."""
+import html
+import json
+import re
+import urllib.parse
+
+from .. import net
+from ..model import Item
+
+DISCOURSE = [
+    "meta.discourse.org", "discuss.python.org", "community.openai.com", "forum.cursor.com",
+    "discuss.huggingface.co", "community.cloudflare.com", "forum.gitlab.com", "discuss.elastic.co",
+]
+XENFORO = ["www.blackhatworld.com", "xenforo.com/community"]
+TG_CHANNELS = ["Abuz_ai_soft", "AlStack", "iishkogatewaybot"]
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+
+
+def available(name="forums"):
+    return True, "Discourse/HN/Lobsters/TG-веб, ключи не нужны"
+
+
+def _strip(s):
+    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip()
+
+
+def _discourse(host, query, limit=8):
+    out = []
+    url = f"https://{host}/search.json?q={urllib.parse.quote(query)}"
+    r = net.get(url, headers=UA, json_mode=True)
+    if r.status != 200:
+        return out
+    d = r.json() or {}
+    for t in (d.get("topics") or [])[:limit]:
+        slug = t.get("slug") or "t"
+        tid = t.get("id")
+        out.append(Item("forums", "thread", url=f"https://{host}/t/{slug}/{tid}",
+                        title=t.get("title", ""), text=_strip(t.get("blurb") or ""),
+                        source=host, published=str(t.get("created_at") or ""),
+                        score=float(t.get("posts_count") or 0) ** 0.4 +
+                              float(t.get("like_count") or 0) ** 0.3,
+                        extra={"posts": t.get("posts_count"), "views": t.get("views")}))
+    for p in (d.get("posts") or [])[:limit]:
+        if len(out) >= limit:
+            break
+        out.append(Item("forums", "post", url=f"https://{host}/t/{p.get('topic_id')}/{p.get('post_number', 1)}",
+                        title=(p.get("topic_title") or "")[:200], text=_strip(p.get("blurb") or ""),
+                        source=host, author=p.get("username", ""), published=str(p.get("created_at") or "")))
+    return out
+
+
+def _hn(query, limit=10):
+    out = []
+    url = ("https://hn.algolia.com/api/v1/search?query=" + urllib.parse.quote(query)
+           + f"&tags=story&hitsPerPage={min(limit, 30)}")
+    r = net.get(url, headers=UA, json_mode=True)
+    for h in ((r.json() or {}).get("hits") or []):
+        title = h.get("title") or h.get("story_title") or ""
+        if not title:
+            continue
+        it = Item("forums", "thread", url=h.get("url") or f"https://news.ycombinator.com/item?id={h.get('objectID')}",
+                  title=title, text=(h.get("story_text") or "")[:1500], source="news.ycombinator.com",
+                  author=h.get("author", ""), published=h.get("created_at", ""),
+                  extra={"points": h.get("points"), "comments": h.get("num_comments")})
+        it.score = float(h.get("points") or 0) ** 0.5 + float(h.get("num_comments") or 0) ** 0.4
+        out.append(it)
+    return out
+
+
+def _lobsters(query, limit=8):
+    out = []
+    r = net.get("https://lobste.rs/search.json?q=" + urllib.parse.quote(query) + "&what=stories&order=newest",
+                headers=UA, json_mode=True)
+    for s in ((r.json() if r.status == 200 else []) or [])[:limit]:
+        it = Item("forums", "thread", url=s.get("comments_url") or s.get("url") or "",
+                  title=s.get("title", ""), text=_strip(s.get("description") or ""),
+                  source="lobste.rs", author=s.get("submitter_user", "") if isinstance(s.get("submitter_user"), str)
+                  else (s.get("submitter_user") or {}).get("username", ""),
+                  published=s.get("created_at", ""),
+                  extra={"score": s.get("score"), "comments": s.get("comment_count")})
+        it.score = float(s.get("score") or 0) ** 0.5
+        out.append(it)
+    return out
+
+
+def _xenforo(host, query, limit=6):
+    out = []
+    base = f"https://{host}" if "//" not in host else host
+    url = f"{base}/search/?q={urllib.parse.quote(query)}&o=date"
+    r = net.get(url, headers=UA)
+    if r.status != 200:
+        return out
+    for m in re.finditer(r'<h3 class="contentRow-title">\s*<a href="([^"]+)"[^>]*>(.*?)</a>', r.text, re.S):
+        href, title = m.group(1), _strip(m.group(2))
+        if not title:
+            continue
+        link = href if href.startswith("http") else base + href
+        out.append(Item("forums", "thread", url=link, title=title, source=host))
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _tg(channel, query, limit=8):
+    out = []
+    r = net.get(f"https://t.me/s/{channel}", headers=UA)
+    if r.status != 200:
+        return out
+    blocks = re.split(r'<div class="tgme_widget_message_wrap', r.text)[1:]
+    ql = query.lower()
+    for b in reversed(blocks):
+        mtext = re.search(r'<div class="tgme_widget_message_text[^"]*"[^>]*>(.*?)</div>', b, re.S)
+        if not mtext:
+            continue
+        text = _strip(mtext.group(1))
+        if ql not in text.lower():
+            continue
+        mlink = re.search(r'data-post="([^"]+)"', b)
+        link = f"https://t.me/{mlink.group(1)}" if mlink else f"https://t.me/s/{channel}"
+        it = Item("forums", "tg_post", url=link, title=text[:120], text=text, source="t.me/" + channel,
+                  published=(re.search(r'datetime="([^"]+)"', b).group(1) if re.search(r'datetime="([^"]+)"', b) else ""))
+        out.append(it)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _relevant(query, item):
+    """Дискурс любит отдавать мусор по одному совпавшему слову: требуем покрытие терминов."""
+    toks = [t for t in re.findall(r"\w{3,}", (query or "").lower())]
+    if not toks:
+        return True
+    body = ((item.title or "") + " " + (item.text or "")).lower()
+    hit = sum(1 for t in toks if t in body)
+    title_hit = sum(1 for t in toks if t in (item.title or "").lower())
+    return (hit >= max(2, int(len(toks) * 0.7))) and title_hit >= 1 if len(toks) > 1 else hit == 1
+
+
+def search_forums(query, limit=20, hosts=None, channels=None):
+    items = []
+    for host in (hosts or DISCOURSE)[:5]:
+        items += _discourse(host, query, limit=6)
+    items += _hn(query, limit=10)
+    items += _lobsters(query, limit=6)
+    for ch in (channels or TG_CHANNELS)[:3]:
+        items += _tg(ch, query, limit=6)
+    if len(items) < limit:
+        for host in (hosts or []) and XENFORO[:1]:
+            items += _xenforo(host, query, limit=5)
+    items = [i for i in items if _relevant(query, i) or i.kind == "tg_post"]
+    return items[:limit * 2]

omni/providers/reddit.py

@@ -0,0 +1,99 @@
+"""Reddit без ключей: поиск по всему реддиту и по сабам, сортировка new/top."""
+import urllib.parse
+
+from .. import net
+from ..model import Item
+
+DEFAULT_SUBS = ["netsec", "cybersecurity", "webscraping", "selfhosted", "webdev", "Python"]
+UA = {"User-Agent": "omni-parse/1.0 (research; contact: local)"}
+
+
+def available(name="reddit"):
+    return True, "pullpush (архив) + публичный json, ключи не нужны"
+
+
+def _pullpush(query, limit=25, subreddit=None, kind="submission"):
+    q = urllib.parse.quote(query or "")
+    url = f"https://api.pullpush.io/reddit/search/{kind}/?q={q}&size={min(limit, 100)}&sort=desc"
+    if subreddit:
+        url += f"&subreddit={subreddit}"
+    r = net.get(url, headers=UA, json_mode=True)
+    data = (r.json() or {}).get("data") or []
+    out = []
+    for d in data:
+        if not (d.get("title") or d.get("body")):
+            continue
+        it = _mk(d, subreddit)
+        it.kind = "thread" if kind == "submission" else "comment"
+        if kind == "comment":
+            it.title = (d.get("link_title") or it.text[:100])
+        out.append(it)
+    return out
+
+
+def _rows(payload):
+    out = []
+    for ch in (payload or {}).get("data", {}).get("children", []):
+        d = ch.get("data") or {}
+        if not d.get("title"):
+            continue
+        out.append(d)
+    return out
+
+
+def _mk(d, sub=None):
+    perm = d.get("permalink") or ""
+    url = "https://www.reddit.com" + perm if perm else d.get("url", "")
+    body = (d.get("selftext") or "")[:1800]
+    it = Item("reddit", "thread", url=url, title=d.get("title", ""), text=body,
+              source="r/" + (d.get("subreddit") or sub or ""), author=d.get("author", ""),
+              published=str(int(d.get("created_utc") or 0)),
+              extra={"ups": d.get("ups"), "comments": d.get("num_comments"),
+                     "sub": d.get("subreddit"), "ext": d.get("url", "")[:300]})
+    it.score = float(d.get("ups") or 0) ** 0.5 + float(d.get("num_comments") or 0) ** 0.4
+    return it
+
+
+def _reddit_native(query, limit=20):
+    q = urllib.parse.quote(query)
+    r = net.get(f"https://www.reddit.com/search.json?q={q}&limit={min(limit, 25)}&sort=relevance&t=month",
+                headers={**UA, "Accept": "application/json"}, json_mode=True)
+    if r.status != 200:
+        return []
+    return [_mk(d) for d in _rows(r.json())]
+
+
+def _relevant(query, item):
+    toks = [t for t in __import__("re").findall(r"\w{3,}", (query or "").lower())]
+    if not toks:
+        return True
+    body = ((item.title or "") + " " + (item.text or "")).lower()
+    hit = sum(1 for t in toks if t in body)
+    title_hit = sum(1 for t in toks if t in (item.title or "").lower())
+    return hit >= max(2, int(len(toks) * 0.7)) and title_hit >= 1
+
+
+def search_reddit(query, limit=20, subs=None, include_comments=True):
+    items = _pullpush(query, limit=limit)
+    if len(items) < 5:
+        items += _reddit_native(query, limit=limit)
+    for sub in (subs or [])[:3]:
+        items += _pullpush(query, limit=10, subreddit=sub)
+    if include_comments and len(items) < limit:
+        items += _pullpush(query, limit=10, kind="comment")[:8]
+    items = [i for i in items if _relevant(query, i)]
+    uniq, seen = [], set()
+    for it in items:
+        k = it.url or it.title
+        if k in seen:
+            continue
+        seen.add(k)
+        uniq.append(it)
+    return uniq[:limit * 2]
+
+
+def hot(subs=None, limit=20):
+    out = []
+    for sub in (subs or DEFAULT_SUBS)[:6]:
+        out += _pullpush("", limit=min(limit, 15), subreddit=sub)
+    return out

omni/providers/searchers.py

@@ -0,0 +1,153 @@
+"""Поисковые API и бесплатные движки: Tavily, Exa, You, Firecrawl, DDG, Mojeek, Bing."""
+import html
+import re
+import urllib.parse
+
+from .. import config, net
+from ..model import Item
+
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+
+
+def _mk(provider, title, url, text="", score=0.0, published="", extra=None):
+    return Item(provider, "result", url=url, title=title, text=text, score=score,
+                published=published, extra=extra or {})
+
+
+def _key(name):
+    return config.KEYS.get(name)
+
+
+def available(name):
+    need = {"tavily": "TAVILY_API_KEY", "exa": "EXA_API_KEY", "you": "YOU_API_KEY",
+            "firecrawl": "FIRECRAWL_API_KEY"}.get(name)
+    if not need:
+        return True, "без ключей"
+    return bool(_key(need)), ("готов" if _key(need) else f"нужен {need}")
+
+
+def search_tavily(query, limit=15, depth="basic"):
+    k = _key("TAVILY_API_KEY")
+    if not k:
+        return []
+    r = net.post_json("https://api.tavily.com/search",
+                      {"api_key": k, "query": query, "max_results": min(max(limit, 5), 20),
+                       "search_depth": depth, "include_answer": False, "include_raw_content": False})
+    d = r.json() or {}
+    return [_mk("tavily", x.get("title", ""), x.get("url", ""), (x.get("content") or "")[:2000],
+                float(x.get("score") or 0)) for x in (d.get("results") or [])]
+
+
+def search_exa(query, limit=15):
+    k = _key("EXA_API_KEY")
+    if not k:
+        return []
+    r = net.post_json("https://api.exa.ai/search",
+                      {"query": query, "numResults": min(max(limit, 5), 25),
+                       "contents": {"text": {"maxCharacters": 1200}}},
+                      headers={"x-api-key": k})
+    d = r.json() or {}
+    return [_mk("exa", x.get("title") or "", x.get("url", ""), (x.get("text") or "")[:2000],
+                float(x.get("score") or 0), x.get("publishedDate") or "") for x in (d.get("results") or [])]
+
+
+def search_you(query, limit=15):
+    k = _key("YOU_API_KEY")
+    if not k:
+        return []
+    url = f"https://ydc-index.io/v1/search?query={urllib.parse.quote(query)}&count={min(limit, 20)}"
+    r = net.get(url, headers={**UA, "X-API-Key": k}, json_mode=True)
+    d = r.json() or {}
+    out = []
+    for section in ("hits", "web"):
+        for x in (d.get(section) or []):
+            snips = x.get("snippets") or []
+            txt = " ".join(snips) if isinstance(snips, list) else (x.get("description") or "")
+            out.append(_mk("you", x.get("title") or "", x.get("url") or "", txt[:2000],
+                           float(x.get("score") or 0), x.get("page_age") or ""))
+        if out:
+            break
+    return out[:limit]
+
+
+def search_firecrawl(query, limit=15):
+    k = _key("FIRECRAWL_API_KEY")
+    if not k:
+        return []
+    r = net.post_json("https://api.firecrawl.dev/v1/search",
+                      {"query": query, "limit": min(max(limit, 3), 20)},
+                      headers={"Authorization": f"Bearer {k}"})
+    d = r.json() or {}
+    return [_mk("firecrawl", x.get("title") or "", x.get("url", ""),
+                (x.get("description") or "")[:2000]) for x in (d.get("data") or [])]
+
+
+def search_ddg(query, limit=15):
+    url = "https://html.duckduckgo.com/html/?q=" + urllib.parse.quote(query) + "&kl=ru-ru"
+    r = net.get(url, headers=UA)
+    if r.status != 200:
+        return []
+    out = []
+    blocks = re.split(r'<div class="result results_links', r.text)[1:]
+    for b in blocks:
+        m = re.search(r'class="result__a" href="([^"]+)"[^>]*>(.*?)</a>', b, re.S)
+        if not m:
+            continue
+        href = html.unescape(m.group(1))
+        if "uddg=" in href:
+            href = urllib.parse.unquote(href.split("uddg=")[1].split("&")[0])
+        title = html.unescape(re.sub(r"<[^>]+>", "", m.group(2))).strip()
+        sn = re.search(r'class="result__snippet"[^>]*>(.*?)</a>', b, re.S)
+        snip = html.unescape(re.sub(r"<[^>]+>", "", sn.group(1))).strip() if sn else ""
+        out.append(_mk("ddg", title, href, snip[:1500]))
+        if len(out) >= limit:
+            break
+    return out
+
+
+def search_mojeek(query, limit=15):
+    """Mojeek: собственный индекс, html без капчи."""
+    url = "https://www.mojeek.com/search?q=" + urllib.parse.quote(query)
+    r = net.get(url, headers=UA)
+    if r.status != 200:
+        return []
+    out = []
+    for m in re.finditer(r'<a class="ob"[^>]*href="([^"]+)"[^>]*>(.*?)</a>.*?<p class="s">(.*?)</p>',
+                         r.text, re.S):
+        href, title, snip = m.group(1), m.group(2), m.group(3)
+        out.append(_mk("mojeek", html.unescape(re.sub(r"<[^>]+>", "", title)).strip(),
+                       html.unescape(href), html.unescape(re.sub(r"<[^>]+>", "", snip))[:1500]))
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _bing_decode(href):
+    """Bing прячет ссылку в ck/a?...&u=a1<base64url>."""
+    if "u=a1" not in href:
+        return href
+    import base64
+    raw = href.split("u=a1", 1)[1].split("&")[0]
+    raw += "=" * (-len(raw) % 4)
+    try:
+        return base64.urlsafe_b64decode(raw).decode("utf-8", "ignore")
+    except Exception:
+        return href
+
+
+def search_bing(query, limit=15):
+    """Bing html: без ключа, первая страница. Ссылки приходят в base64."""
+    url = "https://www.bing.com/search?q=" + urllib.parse.quote(query) + f"&count={min(limit, 30)}"
+    r = net.get(url, headers={**UA, "Accept-Language": "ru,en;q=0.8"})
+    if r.status != 200:
+        return []
+    out = []
+    for m in re.finditer(r'<li class="b_algo".*?<h2>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', r.text, re.S):
+        href = _bing_decode(html.unescape(m.group(1)))
+        title = html.unescape(re.sub(r"<[^>]+>", "", m.group(2))).strip()
+        if not href.startswith("http") or "bing.com/ck" in href:
+            continue
+        out.append(_mk("bing", title, href))
+        if len(out) >= limit:
+            break
+    return out

omni/providers/treg.py

@@ -0,0 +1,37 @@
+"""treg.to: публичного поискового API нет, работаем только со своим токеном.
+
+Провайдер существует, чтобы в общем списке было видно: источник требует доступ,
+а не молча отсутствует. Если TREG_TOKEN задан и у аккаунта есть веб-поиск,
+провайдер отдаст результаты, иначе сразу скажет причину.
+"""
+import urllib.parse
+
+from .. import config, net
+from ..model import Item
+
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+
+
+def available(name="treg"):
+    if not config.KEYS.get("TREG_TOKEN"):
+        return False, "нужен TREG_TOKEN (публичного поиска нет)"
+    return True, "токен есть"
+
+
+def search_treg(query, limit=10):
+    tok = config.KEYS.get("TREG_TOKEN")
+    if not tok:
+        return []
+    # основной путь: внутренний поиск по каталогу моделей/задач
+    url = f"https://treg.to/api/search?q={urllib.parse.quote(query)}&limit={min(limit, 20)}"
+    r = net.get(url, headers={**UA, "Authorization": f"Bearer {tok}"}, json_mode=True)
+    d = r.json() or {}
+    rows = d.get("results") or d.get("data") or []
+    out = []
+    for x in rows[:limit]:
+        if not isinstance(x, dict):
+            continue
+        out.append(Item("treg", "result", url=x.get("url") or x.get("link") or "https://treg.to",
+                        title=x.get("title") or x.get("name") or "", text=(x.get("description") or "")[:1500],
+                        score=float(x.get("score") or 0)))
+    return out

omni/providers/x.py

@@ -0,0 +1,106 @@
+"""X/Twitter без платного API.
+
+Три пути:
+1. X_BEARER есть -> официальный /2/tweets/search/recent.
+2. Иначе -> ищем ссылки на твиты через поисковики, текст достаём из publish.twitter.com/oembed (без ключей).
+3. Профиль целиком -> syndication.twitter.com/srv/timeline-profile (отдаёт JSON в HTML, без ключей).
+"""
+import json
+import re
+import urllib.parse
+
+from .. import config, net
+from ..model import Item
+
+TWEET_RE = re.compile(r"https?://(?:www\.)?(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})/status/(\d+)")
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
+
+
+def available(name="x"):
+    if config.KEYS.get("X_BEARER"):
+        return True, "официальный API v2 (X_BEARER)"
+    return True, "без ключей: oembed + syndication timeline"
+
+
+def _oembed(url):
+    api = "https://publish.twitter.com/oembed?omit_script=1&dnt=1&url=" + urllib.parse.quote(url, safe="")
+    r = net.get(api, headers=UA, json_mode=True)
+    if r.status != 200:
+        return None
+    d = r.json() or {}
+    html = d.get("html") or ""
+    text = re.sub(r"<[^>]+>", " ", html)
+    text = re.sub(r"\s+", " ", text).replace("—", "-").strip()
+    return {"author": d.get("author_name") or "", "text": text,
+            "url": d.get("url") or url, "handle": (d.get("author_url") or "").rstrip("/").split("/")[-1]}
+
+
+def search_x(query, limit=15, handles=None):
+    items = []
+    bearer = config.KEYS.get("X_BEARER")
+    if bearer:
+        q = urllib.parse.quote(f"{query} -is:retweet lang:ru OR lang:en")
+        url = (f"https://api.x.com/2/tweets/search/recent?query={q}"
+               f"&max_results={min(max(limit, 10), 100)}&tweet.fields=created_at,public_metrics,lang")
+        r = net.get(url, headers={"Authorization": f"Bearer {bearer}"}, json_mode=True)
+        for t in ((r.json() or {}).get("data") or []):
+            m = t.get("public_metrics") or {}
+            it = Item("x", "tweet", url=f"https://x.com/i/status/{t['id']}", title=t["text"][:120],
+                      text=t["text"], author="", published=t.get("created_at", ""),
+                      extra={"likes": m.get("like_count"), "rt": m.get("retweet_count")})
+            it.score = (m.get("like_count") or 0) ** 0.5
+            items.append(it)
+    else:
+        # путь 2: находим ссылки поисковиками, текст тянем oembed-ом
+        from . import searchers
+        found = []
+        for eng in ("ddg", "mojeek", "bing", "tavily"):
+            try:
+                res = getattr(searchers, "search_" + eng)(f"{query} site:x.com OR site:twitter.com", limit=limit)
+            except Exception:
+                res = []
+            for r0 in res:
+                for m in TWEET_RE.finditer(r0.url or ""):
+                    found.append(m.group(0))
+            if len(found) >= limit:
+                break
+        seen = set()
+        for u in found[:limit * 2]:
+            if u in seen:
+                continue
+            seen.add(u)
+            d = _oembed(u)
+            if not d or len(d["text"]) < 20:
+                continue
+            it = Item("x", "tweet", url=d["url"], title=d["text"][:120], text=d["text"],
+                      author=d["author"], source="x.com/" + (d["handle"] or ""),
+                      extra={"handle": d["handle"]})
+            items.append(it)
+
+    # путь 3: конкретные профили
+    for h in (handles or [])[:4]:
+        r = net.get(f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{h}",
+                    headers=UA)
+        if r.status != 200:
+            continue
+        m = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', r.text, re.S)
+        if not m:
+            continue
+        try:
+            data = json.loads(m.group(1))
+        except Exception:
+            continue
+        entries = (((data.get("props") or {}).get("pageProps") or {}).get("timeline") or {}).get("entries") or []
+        for e in entries[:12]:
+            tw = ((e.get("content") or {}).get("tweet") or {})
+            txt = tw.get("full_text") or tw.get("text") or ""
+            if not txt:
+                continue
+            tid = tw.get("id_str") or ""
+            it = Item("x", "tweet", url=f"https://x.com/{h}/status/{tid}" if tid else f"https://x.com/{h}",
+                      title=txt[:120], text=txt, author=h, source="x.com/" + h,
+                      published=tw.get("created_at", ""),
+                      extra={"likes": tw.get("favorite_count"), "rt": tw.get("retweet_count")})
+            it.score = float(tw.get("favorite_count") or 0) ** 0.5
+            items.append(it)
+    return items[:limit * 2]