From 02816cacfbaad2c3a0bc923d2896e439fb60e549 Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 25 Oct 2026 13:16:56 +0300 Subject: [PATCH] =?UTF-8?q?omni-parse:=2012=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=B9=D0=B4=D0=B5=D1=80=D0=BE=D0=B2,=20sqlite+fts5,=20si?= =?UTF-8?q?mhash-=D0=B4=D0=B5=D0=B4=D1=83=D0=BF,=20=D1=84=D0=B8=D0=BB?= =?UTF-8?q?=D1=8C=D1=82=D1=80=20=D1=80=D0=B5=D0=BB=D0=B5=D0=B2=D0=B0=D0=BD?= =?UTF-8?q?=D1=82=D0=BD=D0=BE=D1=81=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 9 ++ .gitignore | 5 + LICENSE | 21 ++++ README.md | 90 +++++++++++++++ omni/__init__.py | 6 + omni/__main__.py | 3 + omni/cli.py | 215 ++++++++++++++++++++++++++++++++++++ omni/config.py | 73 ++++++++++++ omni/filter.py | 34 ++++++ omni/model.py | 99 +++++++++++++++++ omni/net.py | 142 ++++++++++++++++++++++++ omni/providers/__init__.py | 49 ++++++++ omni/providers/cards.py | 102 +++++++++++++++++ omni/providers/forums.py | 151 +++++++++++++++++++++++++ omni/providers/reddit.py | 99 +++++++++++++++++ omni/providers/searchers.py | 153 +++++++++++++++++++++++++ omni/providers/treg.py | 37 +++++++ omni/providers/x.py | 106 ++++++++++++++++++ omni/rank.py | 41 +++++++ omni/store.py | 165 +++++++++++++++++++++++++++ requirements.txt | 4 + themes.txt | 9 ++ 22 files changed, 1613 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 omni/__init__.py create mode 100644 omni/__main__.py create mode 100644 omni/cli.py create mode 100644 omni/config.py create mode 100644 omni/filter.py create mode 100644 omni/model.py create mode 100644 omni/net.py create mode 100644 omni/providers/__init__.py create mode 100644 omni/providers/cards.py create mode 100644 omni/providers/forums.py create mode 100644 omni/providers/reddit.py create mode 100644 omni/providers/searchers.py create mode 100644 omni/providers/treg.py create mode 100644 omni/providers/x.py create mode 100644 omni/rank.py create mode 100644 omni/store.py create mode 100644 requirements.txt create mode 100644 themes.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4173b4d --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# все ключи опциональны: без них работают reddit, forums, ddg, bing, mojeek, x, cards +TAVILY_API_KEY= +EXA_API_KEY= +YOU_API_KEY= +FIRECRAWL_API_KEY= +TREG_TOKEN= +X_BEARER= +# прокси для тяжёлых хостов, формат http://user:pass@host:port +OMNI_PROXY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b18add --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.env +data/ +*.sqlite3* +__pycache__/ +*.pyc diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..38e129d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 omni-parse contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6fd6da0 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# omni-parse + +Один парсер под любую тему: Reddit, форумы, поисковики, X/Twitter, карточки товаров и платные search-API — в одну SQLite-базу с FTS5, дедупом и рейтингом. + +Ставится без ключей. Платные провайдеры включаются сами, если в окружении есть ключ. + +## Что умеет + +| Провайдер | Что тянет | Ключ | +|---|---|---| +| `reddit` | поиск по постам и комментариям через api.pullpush.io (архив Reddit) + публичный json как запас | нет | +| `forums` | Discourse (9 инстансов), Hacker News (Algolia), Lobsters, XenForo, веб-версии Telegram-каналов | нет | +| `ddg` | html-выдача DuckDuckGo | нет | +| `bing` | html-выдача Bing, ссылки раскодируются из `ck/a?...u=a1` | нет | +| `mojeek` | html Mojeek (свой индекс, без капчи) | нет | +| `x` | без платного API: oembed по URL твита, syndication-таймлайн профиля, поиск твитов через ddg/bing/mojeek | нет | +| `tavily` | Tavily search | `TAVILY_API_KEY` | +| `exa` | Exa search + текст страниц | `EXA_API_KEY` | +| `you` | You.com search | `YOU_API_KEY` | +| `firecrawl` | Firecrawl search | `FIRECRAWL_API_KEY` | +| `cards` | карточки товаров: JSON-LD Product/Offer, OpenGraph, microdata, цена/валюта/наличие | нет | +| `treg` | treg-поиск | `TREG_TOKEN` | + +`python -m omni providers` печатает, что реально поднялось в этом окружении. + +## Установка + +```bash +pip install -r requirements.txt +cp .env.example .env # ключи по желанию, ни один не обязателен +``` + +## Быстрый старт + +```bash +# что доступно +python -m omni providers + +# поиск по одной теме в четырёх бесплатных источниках +python -m omni search "scraping twitter without api" --providers reddit,forums,ddg,bing --limit 12 --show 10 + +# прогон по списку тем из themes.txt (по теме на строку) +python -m omni fanout --providers reddit,forums,ddg,bing --limit 10 + +# топ базы, статистика, поиск по своей базе +python -m omni top --limit 20 +python -m omni stats +python -m omni find "proxy" + +# карточки товаров по URL +python -m omni cards https://example.com/product + +# выгрузка +python -m omni export --format jsonl --out data/dump.jsonl +``` + +## Как устроено + +- `omni/net.py` — лестница транспорта: httpx → curl_cffi (подмена TLS-фингерпринта) → прокси из env. Троттлинг по хосту, лимит байт, пометка блокировок. +- `omni/model.py` — единая схема записи: провайдер, тип, url, заголовок, текст, автор, дата, теги, цена, счёт, simhash. +- `omni/filter.py` — релевантность: запрос должен быть покрыт текстом (≥60% терминов) и хотя бы один термин — в заголовке. Для русского сравнивает по 5-символьному префиксу, иначе морфология убивает совпадения. +- `omni/store.py` — SQLite WAL + FTS5, таблицы `items`, `bands`, `runs`. Дедуп: simhash по шинглам, сравнение по Хэммингу ≤3 по бакетам. +- `omni/rank.py` — вес провайдера, свежесть, наличие цены, вовлечение. +- `omni/providers/` — по файлу на семейство источников, реестр в `__init__.py`. + +## Реальные прогоны + +``` +python -m omni search "scraping twitter without api" --providers reddit,forums,ddg,bing + bing 10 ok | ddg 10 ok | forums 10 ok | reddit 8 ok + собрано 38, новых 31, дублей 7 + +python -m omni fanout # 8 тем из themes.txt + итого новых 200, дублей 5 + +python -m omni stats + записей 266 | прогонов 56 (ok 37) + ddg=93 bing=85 forums=56 reddit=32 +``` + +## Ограничения + +- Reddit напрямую с серверных IP отдаёт 403 — используется архив pullpush; свежие посты (последние часы) в архиве могут отсутствовать. +- X/Twitter без токена: доступны только публичные встраивания и таймлайн профиля, полноценного полнотекстового поиска нет. +- mojeek отдаёт пустую страницу при агрессивном троттлинге — провайдер помечается, остальные работают. +- `cards` не обходит защиту от парсинга: если страница рендерится JS, разметки в html не будет. + +## Лицензия + +MIT. \ No newline at end of file diff --git a/omni/__init__.py b/omni/__init__.py new file mode 100644 index 0000000..6f91c93 --- /dev/null +++ b/omni/__init__.py @@ -0,0 +1,6 @@ +"""omni — универсальный парсер тем: соцсети, форумы, поисковые API, карточки. + +Один вход (тема) -> много источников -> одна база с FTS5. +Ключи только из окружения или .env. В коде секретов нет. +""" +__version__ = "1.0.0" \ No newline at end of file diff --git a/omni/__main__.py b/omni/__main__.py new file mode 100644 index 0000000..1a80472 --- /dev/null +++ b/omni/__main__.py @@ -0,0 +1,3 @@ +from .cli import main +import sys +sys.exit(main()) diff --git a/omni/cli.py b/omni/cli.py new file mode 100644 index 0000000..b236963 --- /dev/null +++ b/omni/cli.py @@ -0,0 +1,215 @@ +"""CLI: providers / search / fanout / cards / top / find / stats / export.""" +import argparse +import concurrent.futures +import json +import sys + +from . import config, rank, store +from . import filter as flt +from .providers import FUNCS, MODULES, get, status + +ALL_FREE = ["reddit", "x", "forums", "ddg", "mojeek", "bing"] +ALL_KEYED = ["tavily", "exa", "you", "firecrawl"] + + +def _default_providers(): + out = list(ALL_FREE) + for name in ALL_KEYED: + ok, _ = MODULES[name].available(name) + if ok: + out.append(name) + return out + + +def _run_one(name, query, limit, extra): + fn = get(name) + if not fn: + return name, [], "нет провайдера" + try: + if name == "cards": + items = fn(extra.get("urls") or []) + elif name == "x": + items = fn(query, limit=limit, handles=extra.get("handles")) + elif name == "reddit": + items = fn(query, limit=limit, subs=extra.get("subs")) + elif name == "forums": + items = fn(query, limit=limit, hosts=extra.get("hosts"), channels=extra.get("channels")) + else: + items = fn(query, limit=limit) + raw = items or [] + if name == "cards": + return name, raw, "ok" if raw else "пусто" + kept = flt.apply(query, raw) + for it in kept: + it.extra["q"] = query + note = "ok" if kept else "пусто" + if len(kept) < len(raw): + note = f"ok {len(kept)}/{len(raw)}" + return name, kept, note + except Exception as e: + return name, [], f"{type(e).__name__}: {e}"[:120] + + +def cmd_providers(args): + for name, ok, note in status(): + print(f"{'OK ' if ok else 'НЕТ'} {name:<10} {note}") + return 0 + + +def cmd_search(args): + provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()] + extra = {"subs": args.subs.split(",") if args.subs else None, + "channels": args.channels.split(",") if args.channels else None, + "handles": args.handles.split(",") if args.handles else None} + results, report = [], [] + with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex: + futs = [ex.submit(_run_one, p, args.query, args.limit, extra) for p in provs] + for f in concurrent.futures.as_completed(futs): + name, items, st = f.result() + report.append((name, len(items), st)) + results += items + store.log_run(name, "", args.query, len(items), 0, st) + results = rank.rank(results) + new, dup = store.add(results) + for name, got, st in sorted(report): + print(f" {name:<10} {got:>3} записей {st}") + print(f"\nсобрано {len(results)}, новых в базе {new}, дублей {dup}\n") + for it in results[:args.show]: + price = f" {it.price}{it.currency or ''}" if it.price else "" + print(f"[{it.score:5.2f}] {it.provider:<9} {it.kind:<8}{price} {it.title[:90]}") + print(f" {it.url[:110]}") + if args.json: + with open(args.json, "w", encoding="utf-8") as f: + for it in results: + f.write(json.dumps(it.as_dict(), ensure_ascii=False) + "\n") + print(f"\njsonl: {args.json}") + return 0 + + +def cmd_fanout(args): + themes = [t.strip() for t in open(args.themes, encoding="utf-8") if t.strip() and not t.startswith("#")] + if args.limit_themes: + themes = themes[:args.limit_themes] + provs = [p.strip() for p in (args.providers or ",".join(_default_providers())).split(",") if p.strip()] + total_new = total_dup = 0 + for th in themes: + results, got = [], {} + with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, len(provs) or 1)) as ex: + futs = [ex.submit(_run_one, p, th, args.limit, {}) for p in provs] + for f in concurrent.futures.as_completed(futs): + name, items, st = f.result() + got[name] = len(items) + results += items + store.log_run(name, "", th, len(items), 0, st) + results = rank.rank(results) + new, dup = store.add(results) + total_new += new + total_dup += dup + print(f"{th[:48]:<50} собрано {len(results):>3} новых {new:>3} дублей {dup:>3}") + if args.verbose: + for it in results[:3]: + print(f" [{it.score:4.1f}] {it.provider} {it.title[:80]}") + print(f"\nитого новых {total_new}, дублей {total_dup}") + return 0 + + +def cmd_cards(args): + urls = [u.strip() for u in args.urls.split(",") if u.strip()] + items = rank.rank(__import__("omni.providers.cards", fromlist=["extract_cards"]).extract_cards(urls, limit=len(urls))) + new, dup = store.add(items) + for it in items: + print(f"{it.title[:80]}") + print(f" цена: {it.price} {it.currency or ''} | {it.extra.get('availability', '')}") + print(f" {it.url[:110]}") + print(f"\nновых {new}, дублей {dup}") + return 0 + + +def cmd_top(args): + for prov, kind, src, title, sc, price, cur, url in store.top(args.limit, args.provider): + p = f" {price}{cur or ''}" if price else "" + print(f"[{sc:5.2f}] {prov:<9} {kind:<8}{p} {title[:80]}") + return 0 + + +def cmd_find(args): + for prov, kind, src, title, sc, url in store.search(args.query, args.limit): + print(f"[{sc:5.2f}] {prov:<9} {kind:<8} {title[:80]}\n {url[:110]}") + return 0 + + +def cmd_stats(args): + s = store.stats() + print(f"записей: {s['items']} | с ценником: {s['priced']} | прогонов: {s['runs']} (ok {s['runs_ok']})") + print("по провайдерам:", ", ".join(f"{p}={n}" for p, n in s["by_provider"])) + print("по типам:", ", ".join(f"{k}={n}" for k, n in s["by_kind"])) + return 0 + + +def cmd_export(args): + n, path = store.export(args.out, args.fmt, args.limit) + print(f"выгружено {n} записей -> {path}") + return 0 + + +def main(argv=None): + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + config.load() + p = argparse.ArgumentParser(prog="omni", description="универсальный парсер тем: соцсети, форумы, поиск, карточки") + sub = p.add_subparsers(dest="cmd", required=True) + + sp = sub.add_parser("providers", help="что доступно и с какими ключами") + sp.set_defaults(fn=cmd_providers) + + sp = sub.add_parser("search", help="один запрос по всем источникам") + sp.add_argument("query") + sp.add_argument("--providers", default="") + sp.add_argument("--limit", type=int, default=15) + sp.add_argument("--show", type=int, default=15) + sp.add_argument("--json", default="") + sp.add_argument("--subs", default="") + sp.add_argument("--channels", default="") + sp.add_argument("--handles", default="") + sp.set_defaults(fn=cmd_search) + + sp = sub.add_parser("fanout", help="список тем -> полный обход") + sp.add_argument("--themes", default="themes.txt") + sp.add_argument("--providers", default="") + sp.add_argument("--limit", type=int, default=12) + sp.add_argument("--limit-themes", type=int, default=0) + sp.add_argument("--verbose", action="store_true") + sp.set_defaults(fn=cmd_fanout) + + sp = sub.add_parser("cards", help="карточки товаров по URL") + sp.add_argument("urls") + sp.set_defaults(fn=cmd_cards) + + sp = sub.add_parser("top", help="топ базы по скору") + sp.add_argument("--limit", type=int, default=20) + sp.add_argument("--provider", default=None) + sp.set_defaults(fn=cmd_top) + + sp = sub.add_parser("find", help="поиск по своей базе") + sp.add_argument("query") + sp.add_argument("--limit", type=int, default=20) + sp.set_defaults(fn=cmd_find) + + sp = sub.add_parser("stats", help="сводка базы") + sp.set_defaults(fn=cmd_stats) + + sp = sub.add_parser("export", help="выгрузка jsonl/csv") + sp.add_argument("--out", default="data/top.jsonl") + sp.add_argument("--fmt", default="jsonl", choices=["jsonl", "csv"]) + sp.add_argument("--limit", type=int, default=1000) + sp.set_defaults(fn=cmd_export) + + args = p.parse_args(argv) + return args.fn(args) + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/omni/config.py b/omni/config.py new file mode 100644 index 0000000..427af14 --- /dev/null +++ b/omni/config.py @@ -0,0 +1,73 @@ +"""Конфиг: пути, лимиты, ключи из env/.env. Никаких секретов в коде.""" +import json +import os +import pathlib + +ROOT = pathlib.Path(__file__).resolve().parent.parent +DATA = pathlib.Path(os.environ.get("OMNI_DATA") or (ROOT / "data")) +DB = pathlib.Path(os.environ.get("OMNI_DB") or (DATA / "omni.sqlite3")) +THEMES = ROOT / "themes.txt" + +DEFAULTS = { + "timeout": 25, + "retries": 2, + "per_host_delay": 0.6, + "max_bytes": 4_000_000, + "limit": 20, + "lang": "ru", + "ua": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"), + "proxy_mode": "off", # off | list | env + "proxies": [], +} + +KEYS = {} # имя -> значение +CONFIG = {} # прочие настройки + + +def _load_env_file(path): + p = pathlib.Path(path) + if not p.exists(): + return {} + out = {} + for line in p.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + out[k.strip()] = v.strip().strip('"').strip("'") + return out + + +def load(): + """Читает .env рядом с проектом и config.json (если есть), собирает ключи.""" + global KEYS, CONFIG + envf = _load_env_file(ROOT / ".env") + for k, v in envf.items(): + os.environ.setdefault(k, v) + cfg = dict(DEFAULTS) + cf = ROOT / "config.json" + if cf.exists(): + try: + cfg.update(json.loads(cf.read_text(encoding="utf-8"))) + except Exception: + pass + CONFIG = cfg + for name in ("TAVILY_API_KEY", "EXA_API_KEY", "YOU_API_KEY", "FIRECRAWL_API_KEY", + "JINA_API_KEY", "SERPER_API_KEY", "BRAVE_API_KEY", "X_BEARER", + "REDDIT_CLIENT_ID", "REDDIT_SECRET", "TREG_TOKEN", "GITHUB_TOKEN"): + v = os.environ.get(name) + if v: + KEYS[name] = v + DATA.mkdir(parents=True, exist_ok=True) + return cfg + + +def proxies(): + mode = CONFIG.get("proxy_mode", "off") + if mode == "env": + v = os.environ.get("OMNI_PROXIES", "") + return [x.strip() for x in v.split(",") if x.strip()] + if mode == "list": + return list(CONFIG.get("proxies") or []) + return [] \ No newline at end of file diff --git a/omni/filter.py b/omni/filter.py new file mode 100644 index 0000000..7928071 --- /dev/null +++ b/omni/filter.py @@ -0,0 +1,34 @@ +"""Релевантность: один фильтр для всех провайдеров, чтобы мусор не лез в базу.""" +import re + +STOP = {"without", "2026", "2025", "how", "the", "and", "for", "with", "without", "free", "best"} + + +def tokens(query): + return [t for t in re.findall(r"[\w\-]{3,}", (query or "").lower()) if t not in STOP] + + +def _has(tok, body): + """Матч с учётом русской морфологии: сравниваем по 5-символьному префиксу.""" + if tok in body: + return True + stem = tok[:5] if len(tok) > 5 else tok[:4] + return len(stem) >= 4 and stem in body + + +def relevance(query, item, min_cov=0.6): + """True, если запрос реально покрыт текстом: и в заголовке есть хоть один термин.""" + toks = tokens(query) + if not toks: + return True + title = (item.title or "").lower() + body = title + " " + (item.text or "").lower() + hit = sum(1 for t in toks if _has(t, body)) + title_hit = sum(1 for t in toks if _has(t, title)) + if len(toks) == 1: + return hit == 1 + return hit >= max(2, int(len(toks) * min_cov + 0.5)) and title_hit >= 1 + + +def apply(query, items, min_cov=0.6): + return [i for i in items if relevance(query, i, min_cov)] \ No newline at end of file diff --git a/omni/model.py b/omni/model.py new file mode 100644 index 0000000..7728d7f --- /dev/null +++ b/omni/model.py @@ -0,0 +1,99 @@ +"""Модель записи: единая схема для всех источников + simhash + разбор цены.""" +import hashlib +import re +import urllib.parse + +PRICE_RE = re.compile(r"(?:(?:[$€£₽¥])\s?(\d{1,6}(?:[.,]\d{1,2})?)|(\d{1,6}(?:[.,]\d{1,2})?)\s?(?:usd|eur|rub|грн|\$|€|₽|руб))", + re.IGNORECASE) +CYR = re.compile(r"[а-яё]", re.IGNORECASE) +TAGS = ("abuse", "proxy", "combo", "api", "jailbreak", "leak", "traffic", "crypto", "tg", "bot") + + +def canon(url): + if not url: + return "" + u = urllib.parse.urlsplit(url.strip()) + q = [(k, v) for k, v in urllib.parse.parse_qsl(u.query) if not k.startswith(("utm_", "ref", "fbclid"))] + path = u.path.rstrip("/") or "/" + return urllib.parse.urlunsplit((u.scheme or "https", u.netloc.lower(), path, urllib.parse.urlencode(q), "")) + + +def shingles(text, k=4): + words = re.findall(r"[\w]+", (text or "").lower())[:400] + if len(words) < k: + return {" ".join(words)} if words else set() + return {" ".join(words[i:i + k]) for i in range(len(words) - k + 1)} + + +def simhash(text): + bits = [0] * 64 + for sh in shingles(text): + h = int(hashlib.blake2b(sh.encode(), digest_size=8).hexdigest(), 16) + for i in range(64): + bits[i] += 1 if (h >> i) & 1 else -1 + out = 0 + for i in range(64): + if bits[i] > 0: + out |= 1 << i + return out + + +def price_of(text): + m = PRICE_RE.search(text or "") + if not m: + return None, None + raw = (m.group(1) or m.group(2) or "").replace(",", ".") + try: + val = float(raw) + except Exception: + return None, None + if val <= 0 or val > 1_000_000: + return None, None + cur = "" + frag = m.group(0).lower() + for sym, code in (("$", "USD"), ("€", "EUR"), ("₽", "RUB"), ("£", "GBP"), ("usd", "USD"), + ("eur", "EUR"), ("rub", "RUB"), ("руб", "RUB"), ("грн", "UAH"), ("¥", "CNY")): + if sym in frag: + cur = code + break + return val, cur + + +def lang_of(text): + t = (text or "")[:400] + cyr = len(CYR.findall(t)) + return "ru" if cyr > max(6, len(t) * 0.05) else "en" + + +class Item: + __slots__ = ("provider", "source", "kind", "url", "title", "text", "author", "published", + "score", "price", "currency", "lang", "tags", "extra", "h") + + def __init__(self, provider, kind, url="", title="", text="", source="", author="", + published="", score=0.0, extra=None): + self.provider = provider + self.kind = kind + self.url = url + self.title = (title or "").strip()[:400] + self.text = (text or "").strip()[:6000] + self.source = source or urllib.parse.urlsplit(url).netloc + self.author = author or "" + self.published = (published or "")[:32] + self.score = float(score or 0) + self.lang = lang_of(self.title + " " + self.text) + body = (self.title + " " + self.text).lower() + self.tags = [t for t in TAGS if t in body][:6] + self.price, self.currency = price_of(self.title + " " + self.text) + self.extra = extra or {} + self.h = simhash(self.title + " " + self.text) if self.text or self.title else 0 + self.url = canon(self.url) + + def key(self): + return hashlib.sha1((self.provider + "|" + (self.url or self.title + self.text[:120])) + .encode("utf-8", "ignore")).hexdigest() + + def as_dict(self): + return {k: getattr(self, k) for k in self.__slots__ if k != "h"} | {"h": self.h} + + def __repr__(self): + return f"<{self.provider}/{self.kind} {self.title[:48]!r}>" \ No newline at end of file diff --git a/omni/net.py b/omni/net.py new file mode 100644 index 0000000..f7e8a22 --- /dev/null +++ b/omni/net.py @@ -0,0 +1,142 @@ +"""Сетевой слой: httpx -> curl_cffi (TLS Chrome) -> прокси. Троттлинг по хосту.""" +import random +import threading +import time +import urllib.parse + +from . import config + +_cfg = config.load() +_lock = threading.Lock() +_last = {} # host -> ts + +BLOCK_MARKS = ("just a moment", "cf-browser-verification", "attention required", + "ddos-guard", "checking your browser", "enable javascript and cookies") + + +class Resp: + __slots__ = ("url", "status", "text", "mode", "error", "headers", "elapsed") + + def __init__(self, url, status=0, text="", mode="", error="", headers=None, elapsed=0.0): + self.url, self.status, self.text = url, status, text + self.mode, self.error, self.headers, self.elapsed = mode, error, headers or {}, elapsed + + @property + def ok(self): + return self.status == 200 and len(self.text) > 50 + + @property + def blocked(self): + if self.status in (403, 429, 503): + return True + low = self.text[:4000].lower() + return any(m in low for m in BLOCK_MARKS) + + def json(self): + try: + import json + return json.loads(self.text) + except Exception: + return None + + def __repr__(self): + return f"" + + +def _throttle(host): + delay = float(_cfg.get("per_host_delay", 0.6)) + with _lock: + prev = _last.get(host, 0) + wait = delay - (time.time() - prev) + if wait > 0: + time.sleep(wait) + with _lock: + _last[host] = time.time() + + +def _pick_proxy(): + ps = config.proxies() + return random.choice(ps) if ps else None + + +def _try_httpx(url, headers, timeout, proxy): + try: + import httpx + kw = {"headers": headers, "timeout": timeout, "follow_redirects": True} + if proxy: + kw["proxy"] = proxy + with httpx.Client(**kw) as c: + r = c.get(url) + return Resp(url, r.status_code, r.text[: _cfg.get("max_bytes", 4_000_000)], + "proxy+httpx" if proxy else "httpx", headers=dict(r.headers)) + except Exception as e: + return Resp(url, 0, "", "httpx", f"{type(e).__name__}: {e}"[:160]) + + +def _try_curl(url, headers, timeout, proxy, impersonate="chrome131"): + try: + from curl_cffi import requests as cr + kw = {"headers": headers, "timeout": timeout, "impersonate": impersonate, "allow_redirects": True} + if proxy: + kw["proxies"] = {"http": proxy, "https": proxy} + r = cr.get(url, **kw) + return Resp(url, r.status_code, (r.text or "")[: _cfg.get("max_bytes", 4_000_000)], + "proxy+curl" if proxy else "curl", headers=dict(r.headers)) + except Exception as e: + return Resp(url, 0, "", "curl", f"{type(e).__name__}: {e}"[:160]) + + +def post_json(url, payload, headers=None, timeout=None): + """POST JSON: httpx, при блоке/ошибке повтор через curl_cffi.""" + import json + timeout = timeout or _cfg.get("timeout", 25) + host = urllib.parse.urlsplit(url).netloc + _throttle(host) + h = {"User-Agent": _cfg["ua"], "Content-Type": "application/json", "Accept": "application/json"} + h.update(headers or {}) + body = json.dumps(payload).encode() + try: + import httpx + with httpx.Client(timeout=timeout, follow_redirects=True) as c: + r = c.post(url, content=body, headers=h) + txt = r.text[:_cfg.get("max_bytes", 4_000_000)] + if r.status_code == 200: + return Resp(url, r.status_code, txt, "httpx-post", headers=dict(r.headers)) + except Exception as e: + txt, code = "", 0 + err = f"{type(e).__name__}: {e}"[:160] + _ = err + try: + from curl_cffi import requests as cr + r = cr.post(url, data=body, headers=h, timeout=timeout, impersonate="chrome131") + return Resp(url, r.status_code, (r.text or "")[:_cfg.get("max_bytes", 4_000_000)], "curl-post", + headers=dict(r.headers)) + except Exception as e: + return Resp(url, 0, "", "curl-post", f"{type(e).__name__}: {e}"[:160]) + + +def get(url, headers=None, timeout=None, json_mode=False): + """Лестница: httpx -> curl_cffi -> curl_cffi с прокси. Возвращает первый осмысленный ответ.""" + timeout = timeout or _cfg.get("timeout", 25) + host = urllib.parse.urlsplit(url).netloc + _throttle(host) + h = {"User-Agent": _cfg["ua"], "Accept": "text/html,application/json;q=0.9,*/*;q=0.8", + "Accept-Language": "ru,en;q=0.8"} + if json_mode: + h["Accept"] = "application/json" + h["Content-Type"] = "application/json" + h.update(headers or {}) + + first = _try_httpx(url, h, timeout, None) + if first.ok and not first.blocked: + return first + second = _try_curl(url, h, timeout, None) + if second.ok and not second.blocked: + return second + proxy = _pick_proxy() + if proxy: + third = _try_curl(url, h, timeout, proxy) + if third.status == 200: + return third + return third + return second if second.status else first \ No newline at end of file diff --git a/omni/providers/__init__.py b/omni/providers/__init__.py new file mode 100644 index 0000000..efff1ab --- /dev/null +++ b/omni/providers/__init__.py @@ -0,0 +1,49 @@ +"""Реестр провайдеров: имя -> (search, available, note).""" +from . import cards, forums, reddit, searchers, treg, x + +MODULES = { + "reddit": reddit, + "x": x, + "forums": forums, + "tavily": searchers, + "exa": searchers, + "you": searchers, + "firecrawl": searchers, + "ddg": searchers, + "mojeek": searchers, + "bing": searchers, + "cards": cards, + "treg": treg, +} + +# какой функцией звать внутри модуля +FUNCS = { + "reddit": "search_reddit", + "x": "search_x", + "forums": "search_forums", + "tavily": "search_tavily", + "exa": "search_exa", + "you": "search_you", + "firecrawl": "search_firecrawl", + "ddg": "search_ddg", + "mojeek": "search_mojeek", + "bing": "search_bing", + "cards": "extract_cards", + "treg": "search_treg", +} + + +def get(name): + mod = MODULES.get(name) + if not mod: + return None + return getattr(mod, FUNCS[name], None) + + +def status(): + rows = [] + for name, mod in MODULES.items(): + fn = getattr(mod, "available", None) + ok, note = fn(name) if callable(fn) else (True, "") + rows.append((name, ok, note)) + return rows \ No newline at end of file diff --git a/omni/providers/cards.py b/omni/providers/cards.py new file mode 100644 index 0000000..113f8c1 --- /dev/null +++ b/omni/providers/cards.py @@ -0,0 +1,102 @@ +"""Карточки товаров: JSON-LD Product/Offer, og-теги, микроразметка, fallback по цене.""" +import json +import re + +from .. import net +from ..model import Item, price_of + +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"} +JSONLD = re.compile(r']+type="application/ld\+json"[^>]*>(.*?)', re.S) + +RULES = [ + ("title", [r']+property="og:title"[^>]+content="([^"]+)"', r"(.*?)"]), + ("price", [r']+property="product:price:amount"[^>]+content="([^"]+)"', + r'"price"\s*:\s*"?([0-9][0-9.,]*)"?']), + ("currency", [r']+property="product:price:currency"[^>]+content="([^"]+)"', + r'"priceCurrency"\s*:\s*"([A-Z]{3})"']), + ("avail", [r'"availability"\s*:\s*"[^"]*?([A-Za-z]+)"?\s*$']), +] + + +def available(name="cards"): + return True, "забирает карточку по URL, ключи не нужны" + + +def _ld_product(html_text): + for m in JSONLD.finditer(html_text): + raw = m.group(1).strip() + try: + data = json.loads(raw) + except Exception: + continue + stack = data if isinstance(data, list) else [data] + while stack: + node = stack.pop() + if isinstance(node, dict): + if node.get("@graph"): + stack.extend(node["@graph"]) + t = node.get("@type") + types = t if isinstance(t, list) else [t] + if any(str(x).lower() in ("product", "offer") for x in types if x): + yield node + for v in node.values(): + if isinstance(v, (dict, list)): + stack.append(v) + elif isinstance(node, list): + stack.extend(node) + + +def _num(s): + try: + return float(str(s).replace(",", ".").replace(" ", "")) + except Exception: + return None + + +def extract_cards(urls, limit=10): + items = [] + for u in (urls if isinstance(urls, (list, tuple)) else [urls])[:limit]: + r = net.get(u, headers=UA) + if r.status != 200: + continue + title = "" + price = cur = "" + avail = "" + for prod in _ld_product(r.text): + offers = prod.get("offers") or {} + if isinstance(offers, list): + offers = offers[0] if offers else {} + price = offers.get("price") or offers.get("lowPrice") or price + cur = offers.get("priceCurrency") or cur + avail = (offers.get("availability") or avail) + title = prod.get("name") or title + if price: + break + for field, pats in RULES: + for p in pats: + m = re.search(p, r.text, re.I | re.S) + if not m: + continue + val = re.sub(r"<[^>]+>", "", m.group(1)).strip() + if field == "title" and not title: + title = val + elif field == "price" and not price: + price = val + elif field == "currency" and not cur: + cur = val + elif field == "avail" and not avail: + avail = val + break + text = r.text + body = re.sub(r"<(script|style)[^>]*>.*?", " ", text, flags=re.S) + body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", body))[:2500] + pval = _num(price) + if not pval: + pval, cur2 = price_of(body) + cur = cur or (cur2 or "") + it = Item("cards", "card", url=u, title=title[:300], text=body, source=u.split("/")[2] if "://" in u else "", + extra={"availability": avail, "raw_price": price}) + it.price = pval + it.currency = (cur or "")[:4].upper() or None + items.append(it) + return items \ No newline at end of file diff --git a/omni/providers/forums.py b/omni/providers/forums.py new file mode 100644 index 0000000..86c714c --- /dev/null +++ b/omni/providers/forums.py @@ -0,0 +1,151 @@ +"""Форумы без ключей: Discourse, HN (Algolia), Lobsters, XenForo, Telegram-веб.""" +import html +import json +import re +import urllib.parse + +from .. import net +from ..model import Item + +DISCOURSE = [ + "meta.discourse.org", "discuss.python.org", "community.openai.com", "forum.cursor.com", + "discuss.huggingface.co", "community.cloudflare.com", "forum.gitlab.com", "discuss.elastic.co", +] +XENFORO = ["www.blackhatworld.com", "xenforo.com/community"] +TG_CHANNELS = ["Abuz_ai_soft", "AlStack", "iishkogatewaybot"] +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"} + + +def available(name="forums"): + return True, "Discourse/HN/Lobsters/TG-веб, ключи не нужны" + + +def _strip(s): + return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip() + + +def _discourse(host, query, limit=8): + out = [] + url = f"https://{host}/search.json?q={urllib.parse.quote(query)}" + r = net.get(url, headers=UA, json_mode=True) + if r.status != 200: + return out + d = r.json() or {} + for t in (d.get("topics") or [])[:limit]: + slug = t.get("slug") or "t" + tid = t.get("id") + out.append(Item("forums", "thread", url=f"https://{host}/t/{slug}/{tid}", + title=t.get("title", ""), text=_strip(t.get("blurb") or ""), + source=host, published=str(t.get("created_at") or ""), + score=float(t.get("posts_count") or 0) ** 0.4 + + float(t.get("like_count") or 0) ** 0.3, + extra={"posts": t.get("posts_count"), "views": t.get("views")})) + for p in (d.get("posts") or [])[:limit]: + if len(out) >= limit: + break + out.append(Item("forums", "post", url=f"https://{host}/t/{p.get('topic_id')}/{p.get('post_number', 1)}", + title=(p.get("topic_title") or "")[:200], text=_strip(p.get("blurb") or ""), + source=host, author=p.get("username", ""), published=str(p.get("created_at") or ""))) + return out + + +def _hn(query, limit=10): + out = [] + url = ("https://hn.algolia.com/api/v1/search?query=" + urllib.parse.quote(query) + + f"&tags=story&hitsPerPage={min(limit, 30)}") + r = net.get(url, headers=UA, json_mode=True) + for h in ((r.json() or {}).get("hits") or []): + title = h.get("title") or h.get("story_title") or "" + if not title: + continue + it = Item("forums", "thread", url=h.get("url") or f"https://news.ycombinator.com/item?id={h.get('objectID')}", + title=title, text=(h.get("story_text") or "")[:1500], source="news.ycombinator.com", + author=h.get("author", ""), published=h.get("created_at", ""), + extra={"points": h.get("points"), "comments": h.get("num_comments")}) + it.score = float(h.get("points") or 0) ** 0.5 + float(h.get("num_comments") or 0) ** 0.4 + out.append(it) + return out + + +def _lobsters(query, limit=8): + out = [] + r = net.get("https://lobste.rs/search.json?q=" + urllib.parse.quote(query) + "&what=stories&order=newest", + headers=UA, json_mode=True) + for s in ((r.json() if r.status == 200 else []) or [])[:limit]: + it = Item("forums", "thread", url=s.get("comments_url") or s.get("url") or "", + title=s.get("title", ""), text=_strip(s.get("description") or ""), + source="lobste.rs", author=s.get("submitter_user", "") if isinstance(s.get("submitter_user"), str) + else (s.get("submitter_user") or {}).get("username", ""), + published=s.get("created_at", ""), + extra={"score": s.get("score"), "comments": s.get("comment_count")}) + it.score = float(s.get("score") or 0) ** 0.5 + out.append(it) + return out + + +def _xenforo(host, query, limit=6): + out = [] + base = f"https://{host}" if "//" not in host else host + url = f"{base}/search/?q={urllib.parse.quote(query)}&o=date" + r = net.get(url, headers=UA) + if r.status != 200: + return out + for m in re.finditer(r'

\s*]*>(.*?)', r.text, re.S): + href, title = m.group(1), _strip(m.group(2)) + if not title: + continue + link = href if href.startswith("http") else base + href + out.append(Item("forums", "thread", url=link, title=title, source=host)) + if len(out) >= limit: + break + return out + + +def _tg(channel, query, limit=8): + out = [] + r = net.get(f"https://t.me/s/{channel}", headers=UA) + if r.status != 200: + return out + blocks = re.split(r'
]*>(.*?)
', b, re.S) + if not mtext: + continue + text = _strip(mtext.group(1)) + if ql not in text.lower(): + continue + mlink = re.search(r'data-post="([^"]+)"', b) + link = f"https://t.me/{mlink.group(1)}" if mlink else f"https://t.me/s/{channel}" + it = Item("forums", "tg_post", url=link, title=text[:120], text=text, source="t.me/" + channel, + published=(re.search(r'datetime="([^"]+)"', b).group(1) if re.search(r'datetime="([^"]+)"', b) else "")) + out.append(it) + if len(out) >= limit: + break + return out + + +def _relevant(query, item): + """Дискурс любит отдавать мусор по одному совпавшему слову: требуем покрытие терминов.""" + toks = [t for t in re.findall(r"\w{3,}", (query or "").lower())] + if not toks: + return True + body = ((item.title or "") + " " + (item.text or "")).lower() + hit = sum(1 for t in toks if t in body) + title_hit = sum(1 for t in toks if t in (item.title or "").lower()) + return (hit >= max(2, int(len(toks) * 0.7))) and title_hit >= 1 if len(toks) > 1 else hit == 1 + + +def search_forums(query, limit=20, hosts=None, channels=None): + items = [] + for host in (hosts or DISCOURSE)[:5]: + items += _discourse(host, query, limit=6) + items += _hn(query, limit=10) + items += _lobsters(query, limit=6) + for ch in (channels or TG_CHANNELS)[:3]: + items += _tg(ch, query, limit=6) + if len(items) < limit: + for host in (hosts or []) and XENFORO[:1]: + items += _xenforo(host, query, limit=5) + items = [i for i in items if _relevant(query, i) or i.kind == "tg_post"] + return items[:limit * 2] \ No newline at end of file diff --git a/omni/providers/reddit.py b/omni/providers/reddit.py new file mode 100644 index 0000000..80e71f8 --- /dev/null +++ b/omni/providers/reddit.py @@ -0,0 +1,99 @@ +"""Reddit без ключей: поиск по всему реддиту и по сабам, сортировка new/top.""" +import urllib.parse + +from .. import net +from ..model import Item + +DEFAULT_SUBS = ["netsec", "cybersecurity", "webscraping", "selfhosted", "webdev", "Python"] +UA = {"User-Agent": "omni-parse/1.0 (research; contact: local)"} + + +def available(name="reddit"): + return True, "pullpush (архив) + публичный json, ключи не нужны" + + +def _pullpush(query, limit=25, subreddit=None, kind="submission"): + q = urllib.parse.quote(query or "") + url = f"https://api.pullpush.io/reddit/search/{kind}/?q={q}&size={min(limit, 100)}&sort=desc" + if subreddit: + url += f"&subreddit={subreddit}" + r = net.get(url, headers=UA, json_mode=True) + data = (r.json() or {}).get("data") or [] + out = [] + for d in data: + if not (d.get("title") or d.get("body")): + continue + it = _mk(d, subreddit) + it.kind = "thread" if kind == "submission" else "comment" + if kind == "comment": + it.title = (d.get("link_title") or it.text[:100]) + out.append(it) + return out + + +def _rows(payload): + out = [] + for ch in (payload or {}).get("data", {}).get("children", []): + d = ch.get("data") or {} + if not d.get("title"): + continue + out.append(d) + return out + + +def _mk(d, sub=None): + perm = d.get("permalink") or "" + url = "https://www.reddit.com" + perm if perm else d.get("url", "") + body = (d.get("selftext") or "")[:1800] + it = Item("reddit", "thread", url=url, title=d.get("title", ""), text=body, + source="r/" + (d.get("subreddit") or sub or ""), author=d.get("author", ""), + published=str(int(d.get("created_utc") or 0)), + extra={"ups": d.get("ups"), "comments": d.get("num_comments"), + "sub": d.get("subreddit"), "ext": d.get("url", "")[:300]}) + it.score = float(d.get("ups") or 0) ** 0.5 + float(d.get("num_comments") or 0) ** 0.4 + return it + + +def _reddit_native(query, limit=20): + q = urllib.parse.quote(query) + r = net.get(f"https://www.reddit.com/search.json?q={q}&limit={min(limit, 25)}&sort=relevance&t=month", + headers={**UA, "Accept": "application/json"}, json_mode=True) + if r.status != 200: + return [] + return [_mk(d) for d in _rows(r.json())] + + +def _relevant(query, item): + toks = [t for t in __import__("re").findall(r"\w{3,}", (query or "").lower())] + if not toks: + return True + body = ((item.title or "") + " " + (item.text or "")).lower() + hit = sum(1 for t in toks if t in body) + title_hit = sum(1 for t in toks if t in (item.title or "").lower()) + return hit >= max(2, int(len(toks) * 0.7)) and title_hit >= 1 + + +def search_reddit(query, limit=20, subs=None, include_comments=True): + items = _pullpush(query, limit=limit) + if len(items) < 5: + items += _reddit_native(query, limit=limit) + for sub in (subs or [])[:3]: + items += _pullpush(query, limit=10, subreddit=sub) + if include_comments and len(items) < limit: + items += _pullpush(query, limit=10, kind="comment")[:8] + items = [i for i in items if _relevant(query, i)] + uniq, seen = [], set() + for it in items: + k = it.url or it.title + if k in seen: + continue + seen.add(k) + uniq.append(it) + return uniq[:limit * 2] + + +def hot(subs=None, limit=20): + out = [] + for sub in (subs or DEFAULT_SUBS)[:6]: + out += _pullpush("", limit=min(limit, 15), subreddit=sub) + return out \ No newline at end of file diff --git a/omni/providers/searchers.py b/omni/providers/searchers.py new file mode 100644 index 0000000..64694bb --- /dev/null +++ b/omni/providers/searchers.py @@ -0,0 +1,153 @@ +"""Поисковые API и бесплатные движки: Tavily, Exa, You, Firecrawl, DDG, Mojeek, Bing.""" +import html +import re +import urllib.parse + +from .. import config, net +from ..model import Item + +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"} + + +def _mk(provider, title, url, text="", score=0.0, published="", extra=None): + return Item(provider, "result", url=url, title=title, text=text, score=score, + published=published, extra=extra or {}) + + +def _key(name): + return config.KEYS.get(name) + + +def available(name): + need = {"tavily": "TAVILY_API_KEY", "exa": "EXA_API_KEY", "you": "YOU_API_KEY", + "firecrawl": "FIRECRAWL_API_KEY"}.get(name) + if not need: + return True, "без ключей" + return bool(_key(need)), ("готов" if _key(need) else f"нужен {need}") + + +def search_tavily(query, limit=15, depth="basic"): + k = _key("TAVILY_API_KEY") + if not k: + return [] + r = net.post_json("https://api.tavily.com/search", + {"api_key": k, "query": query, "max_results": min(max(limit, 5), 20), + "search_depth": depth, "include_answer": False, "include_raw_content": False}) + d = r.json() or {} + return [_mk("tavily", x.get("title", ""), x.get("url", ""), (x.get("content") or "")[:2000], + float(x.get("score") or 0)) for x in (d.get("results") or [])] + + +def search_exa(query, limit=15): + k = _key("EXA_API_KEY") + if not k: + return [] + r = net.post_json("https://api.exa.ai/search", + {"query": query, "numResults": min(max(limit, 5), 25), + "contents": {"text": {"maxCharacters": 1200}}}, + headers={"x-api-key": k}) + d = r.json() or {} + return [_mk("exa", x.get("title") or "", x.get("url", ""), (x.get("text") or "")[:2000], + float(x.get("score") or 0), x.get("publishedDate") or "") for x in (d.get("results") or [])] + + +def search_you(query, limit=15): + k = _key("YOU_API_KEY") + if not k: + return [] + url = f"https://ydc-index.io/v1/search?query={urllib.parse.quote(query)}&count={min(limit, 20)}" + r = net.get(url, headers={**UA, "X-API-Key": k}, json_mode=True) + d = r.json() or {} + out = [] + for section in ("hits", "web"): + for x in (d.get(section) or []): + snips = x.get("snippets") or [] + txt = " ".join(snips) if isinstance(snips, list) else (x.get("description") or "") + out.append(_mk("you", x.get("title") or "", x.get("url") or "", txt[:2000], + float(x.get("score") or 0), x.get("page_age") or "")) + if out: + break + return out[:limit] + + +def search_firecrawl(query, limit=15): + k = _key("FIRECRAWL_API_KEY") + if not k: + return [] + r = net.post_json("https://api.firecrawl.dev/v1/search", + {"query": query, "limit": min(max(limit, 3), 20)}, + headers={"Authorization": f"Bearer {k}"}) + d = r.json() or {} + return [_mk("firecrawl", x.get("title") or "", x.get("url", ""), + (x.get("description") or "")[:2000]) for x in (d.get("data") or [])] + + +def search_ddg(query, limit=15): + url = "https://html.duckduckgo.com/html/?q=" + urllib.parse.quote(query) + "&kl=ru-ru" + r = net.get(url, headers=UA) + if r.status != 200: + return [] + out = [] + blocks = re.split(r'
]*>(.*?)', b, re.S) + if not m: + continue + href = html.unescape(m.group(1)) + if "uddg=" in href: + href = urllib.parse.unquote(href.split("uddg=")[1].split("&")[0]) + title = html.unescape(re.sub(r"<[^>]+>", "", m.group(2))).strip() + sn = re.search(r'class="result__snippet"[^>]*>(.*?)', b, re.S) + snip = html.unescape(re.sub(r"<[^>]+>", "", sn.group(1))).strip() if sn else "" + out.append(_mk("ddg", title, href, snip[:1500])) + if len(out) >= limit: + break + return out + + +def search_mojeek(query, limit=15): + """Mojeek: собственный индекс, html без капчи.""" + url = "https://www.mojeek.com/search?q=" + urllib.parse.quote(query) + r = net.get(url, headers=UA) + if r.status != 200: + return [] + out = [] + for m in re.finditer(r']*href="([^"]+)"[^>]*>(.*?).*?

(.*?)

', + r.text, re.S): + href, title, snip = m.group(1), m.group(2), m.group(3) + out.append(_mk("mojeek", html.unescape(re.sub(r"<[^>]+>", "", title)).strip(), + html.unescape(href), html.unescape(re.sub(r"<[^>]+>", "", snip))[:1500])) + if len(out) >= limit: + break + return out + + +def _bing_decode(href): + """Bing прячет ссылку в ck/a?...&u=a1.""" + if "u=a1" not in href: + return href + import base64 + raw = href.split("u=a1", 1)[1].split("&")[0] + raw += "=" * (-len(raw) % 4) + try: + return base64.urlsafe_b64decode(raw).decode("utf-8", "ignore") + except Exception: + return href + + +def search_bing(query, limit=15): + """Bing html: без ключа, первая страница. Ссылки приходят в base64.""" + url = "https://www.bing.com/search?q=" + urllib.parse.quote(query) + f"&count={min(limit, 30)}" + r = net.get(url, headers={**UA, "Accept-Language": "ru,en;q=0.8"}) + if r.status != 200: + return [] + out = [] + for m in re.finditer(r'
  • \s*]+href="([^"]+)"[^>]*>(.*?)', r.text, re.S): + href = _bing_decode(html.unescape(m.group(1))) + title = html.unescape(re.sub(r"<[^>]+>", "", m.group(2))).strip() + if not href.startswith("http") or "bing.com/ck" in href: + continue + out.append(_mk("bing", title, href)) + if len(out) >= limit: + break + return out diff --git a/omni/providers/treg.py b/omni/providers/treg.py new file mode 100644 index 0000000..8472071 --- /dev/null +++ b/omni/providers/treg.py @@ -0,0 +1,37 @@ +"""treg.to: публичного поискового API нет, работаем только со своим токеном. + +Провайдер существует, чтобы в общем списке было видно: источник требует доступ, +а не молча отсутствует. Если TREG_TOKEN задан и у аккаунта есть веб-поиск, +провайдер отдаст результаты, иначе сразу скажет причину. +""" +import urllib.parse + +from .. import config, net +from ..model import Item + +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"} + + +def available(name="treg"): + if not config.KEYS.get("TREG_TOKEN"): + return False, "нужен TREG_TOKEN (публичного поиска нет)" + return True, "токен есть" + + +def search_treg(query, limit=10): + tok = config.KEYS.get("TREG_TOKEN") + if not tok: + return [] + # основной путь: внутренний поиск по каталогу моделей/задач + url = f"https://treg.to/api/search?q={urllib.parse.quote(query)}&limit={min(limit, 20)}" + r = net.get(url, headers={**UA, "Authorization": f"Bearer {tok}"}, json_mode=True) + d = r.json() or {} + rows = d.get("results") or d.get("data") or [] + out = [] + for x in rows[:limit]: + if not isinstance(x, dict): + continue + out.append(Item("treg", "result", url=x.get("url") or x.get("link") or "https://treg.to", + title=x.get("title") or x.get("name") or "", text=(x.get("description") or "")[:1500], + score=float(x.get("score") or 0))) + return out \ No newline at end of file diff --git a/omni/providers/x.py b/omni/providers/x.py new file mode 100644 index 0000000..7c84f07 --- /dev/null +++ b/omni/providers/x.py @@ -0,0 +1,106 @@ +"""X/Twitter без платного API. + +Три пути: +1. X_BEARER есть -> официальный /2/tweets/search/recent. +2. Иначе -> ищем ссылки на твиты через поисковики, текст достаём из publish.twitter.com/oembed (без ключей). +3. Профиль целиком -> syndication.twitter.com/srv/timeline-profile (отдаёт JSON в HTML, без ключей). +""" +import json +import re +import urllib.parse + +from .. import config, net +from ..model import Item + +TWEET_RE = re.compile(r"https?://(?:www\.)?(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})/status/(\d+)") +UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"} + + +def available(name="x"): + if config.KEYS.get("X_BEARER"): + return True, "официальный API v2 (X_BEARER)" + return True, "без ключей: oembed + syndication timeline" + + +def _oembed(url): + api = "https://publish.twitter.com/oembed?omit_script=1&dnt=1&url=" + urllib.parse.quote(url, safe="") + r = net.get(api, headers=UA, json_mode=True) + if r.status != 200: + return None + d = r.json() or {} + html = d.get("html") or "" + text = re.sub(r"<[^>]+>", " ", html) + text = re.sub(r"\s+", " ", text).replace("—", "-").strip() + return {"author": d.get("author_name") or "", "text": text, + "url": d.get("url") or url, "handle": (d.get("author_url") or "").rstrip("/").split("/")[-1]} + + +def search_x(query, limit=15, handles=None): + items = [] + bearer = config.KEYS.get("X_BEARER") + if bearer: + q = urllib.parse.quote(f"{query} -is:retweet lang:ru OR lang:en") + url = (f"https://api.x.com/2/tweets/search/recent?query={q}" + f"&max_results={min(max(limit, 10), 100)}&tweet.fields=created_at,public_metrics,lang") + r = net.get(url, headers={"Authorization": f"Bearer {bearer}"}, json_mode=True) + for t in ((r.json() or {}).get("data") or []): + m = t.get("public_metrics") or {} + it = Item("x", "tweet", url=f"https://x.com/i/status/{t['id']}", title=t["text"][:120], + text=t["text"], author="", published=t.get("created_at", ""), + extra={"likes": m.get("like_count"), "rt": m.get("retweet_count")}) + it.score = (m.get("like_count") or 0) ** 0.5 + items.append(it) + else: + # путь 2: находим ссылки поисковиками, текст тянем oembed-ом + from . import searchers + found = [] + for eng in ("ddg", "mojeek", "bing", "tavily"): + try: + res = getattr(searchers, "search_" + eng)(f"{query} site:x.com OR site:twitter.com", limit=limit) + except Exception: + res = [] + for r0 in res: + for m in TWEET_RE.finditer(r0.url or ""): + found.append(m.group(0)) + if len(found) >= limit: + break + seen = set() + for u in found[:limit * 2]: + if u in seen: + continue + seen.add(u) + d = _oembed(u) + if not d or len(d["text"]) < 20: + continue + it = Item("x", "tweet", url=d["url"], title=d["text"][:120], text=d["text"], + author=d["author"], source="x.com/" + (d["handle"] or ""), + extra={"handle": d["handle"]}) + items.append(it) + + # путь 3: конкретные профили + for h in (handles or [])[:4]: + r = net.get(f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{h}", + headers=UA) + if r.status != 200: + continue + m = re.search(r'', r.text, re.S) + if not m: + continue + try: + data = json.loads(m.group(1)) + except Exception: + continue + entries = (((data.get("props") or {}).get("pageProps") or {}).get("timeline") or {}).get("entries") or [] + for e in entries[:12]: + tw = ((e.get("content") or {}).get("tweet") or {}) + txt = tw.get("full_text") or tw.get("text") or "" + if not txt: + continue + tid = tw.get("id_str") or "" + it = Item("x", "tweet", url=f"https://x.com/{h}/status/{tid}" if tid else f"https://x.com/{h}", + title=txt[:120], text=txt, author=h, source="x.com/" + h, + published=tw.get("created_at", ""), + extra={"likes": tw.get("favorite_count"), "rt": tw.get("retweet_count")}) + it.score = float(tw.get("favorite_count") or 0) ** 0.5 + items.append(it) + return items[:limit * 2] \ No newline at end of file diff --git a/omni/rank.py b/omni/rank.py new file mode 100644 index 0000000..adb95e8 --- /dev/null +++ b/omni/rank.py @@ -0,0 +1,41 @@ +"""Скор записи: вес провайдера + вовлечённость + свежесть + ценник + объём текста.""" +import re +import time + +WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0, + "firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8} + +KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "result": 0.1, "post": 0.1} + + +def _fresh(published): + """published: unix-строка или ISO. Возвращает 0..1.""" + if not published: + return 0.3 + try: + if published.isdigit(): + ts = int(published) + if ts > 10**12: + ts //= 1000 + else: + ts = time.mktime(time.strptime(published[:19], "%Y-%m-%dT%H:%M:%S")) + days = max(0.0, (time.time() - ts) / 86400) + return max(0.0, 1.0 - days / 30.0) + except Exception: + return 0.3 + + +def score(it): + base = it.score if it.score else 0.0 + w = WEIGHT.get(it.provider, 0.7) + fresh = _fresh(it.published) + priced = 0.35 if it.price else 0.0 + body = min(1.0, len(it.text) / 1200.0) + eng = min(3.0, base ** 0.5) if base > 1 else base + total = (w * (1.0 + eng) + fresh * 1.2 + priced + body + KIND_BONUS.get(it.kind, 0.0)) + it.score = round(total, 4) + return it + + +def rank(items): + return sorted((score(i) for i in items), key=lambda x: -x.score) \ No newline at end of file diff --git a/omni/store.py b/omni/store.py new file mode 100644 index 0000000..17a6360 --- /dev/null +++ b/omni/store.py @@ -0,0 +1,165 @@ +"""Склад: sqlite WAL + FTS5. Таблицы items, runs; дедуп по ключу и simhash-полосам.""" +import json +import sqlite3 +import time +from pathlib import Path + +from . import config + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS items ( + id INTEGER PRIMARY KEY, + k TEXT UNIQUE, + provider TEXT, kind TEXT, source TEXT, url TEXT, title TEXT, text TEXT, + author TEXT, published TEXT, lang TEXT, tags TEXT, price REAL, currency TEXT, + score REAL, h INTEGER, extra TEXT, seen INTEGER +); +CREATE INDEX IF NOT EXISTS ix_items_prov ON items(provider, kind); +CREATE INDEX IF NOT EXISTS ix_items_seen ON items(seen); +CREATE INDEX IF NOT EXISTS ix_items_h ON items(h); +CREATE TABLE IF NOT EXISTS bands (band INTEGER, h INTEGER, item_id INTEGER, PRIMARY KEY(band,h,item_id)); +CREATE INDEX IF NOT EXISTS ix_bands_h ON bands(h); +CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY, ts INTEGER, provider TEXT, source TEXT, query TEXT, + got INTEGER, kept INTEGER, status TEXT, bytes INTEGER, mode TEXT +); +CREATE VIRTUAL TABLE IF NOT EXISTS items_fts USING fts5( + title, text, content='items', content_rowid='id', tokenize='unicode61 remove_diacritics 2' +); +CREATE TRIGGER IF NOT EXISTS items_ai AFTER INSERT ON items BEGIN + INSERT INTO items_fts(rowid,title,text) VALUES (new.id,new.title,new.text); +END; +CREATE TRIGGER IF NOT EXISTS items_ad AFTER DELETE ON items BEGIN + INSERT INTO items_fts(items_fts,rowid,title,text) VALUES('delete',old.id,old.title,old.text); +END; +""" + +BANDS = 4 # 4 полосы по 16 бит +MASK = 0xFFFF + + +def conn(): + c = sqlite3.connect(str(config.DB), timeout=30) + c.execute("PRAGMA journal_mode=WAL") + c.execute("PRAGMA synchronous=NORMAL") + c.executescript(SCHEMA) + return c + + +def _signed(h): + return h - (1 << 64) if h >= (1 << 63) else h + + +def _unsigned(v): + return v & 0xFFFFFFFFFFFFFFFF + + +def _popcount(x): + return bin(x).count("1") + + +def _near_dup(c, h, max_dist=3): + """Кандидат ищем по полосе, решение принимаем по расстоянию Хэмминга.""" + h = _unsigned(h) + for b in range(BANDS): + band = (h >> (16 * b)) & MASK + rows = c.execute( + "SELECT i.h FROM bands b JOIN items i ON i.id=b.item_id " + "WHERE b.band=? LIMIT 200", (b,)).fetchall() + for (other,) in rows: + if _popcount(_unsigned(other) ^ h) <= max_dist: + return True + return False + + +def add(items): + """Возвращает (новых, дублей).""" + new = dup = 0 + ts = int(time.time()) + with conn() as c: + for it in items: + h = _unsigned(it.h) + if _near_dup(c, h): + dup += 1 + continue + try: + cur = c.execute( + "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang," + "tags,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (it.key(), it.provider, it.kind, it.source, it.url, it.title, it.text, it.author, + it.published, it.lang, ",".join(it.tags), it.price, it.currency, it.score, + _signed(h), json.dumps(it.extra, ensure_ascii=False)[:2000], ts)) + except sqlite3.IntegrityError: + dup += 1 + continue + iid = cur.lastrowid + for b in range(BANDS): + band = (h >> (16 * b)) & MASK + c.execute("INSERT OR IGNORE INTO bands(band,h,item_id) VALUES(?,?,?)", + (band, _signed(h), iid)) + new += 1 + return new, dup + + +def log_run(provider, source, query, got, kept, status, nbytes=0, mode=""): + with conn() as c: + c.execute("INSERT INTO runs(ts,provider,source,query,got,kept,status,bytes,mode) " + "VALUES(?,?,?,?,?,?,?,?,?)", + (int(time.time()), provider, source, (query or "")[:200], got, kept, status, nbytes, mode)) + + +def top(limit=20, provider=None): + q = ("SELECT provider,kind,source,title,score,price,currency,url FROM items " + + ("WHERE provider=? " if provider else "") + + "ORDER BY score DESC, id DESC LIMIT ?") + args = (provider, limit) if provider else (limit,) + with conn() as c: + return c.execute(q, args).fetchall() + + +def search(q, limit=20): + with conn() as c: + rows = c.execute( + "SELECT i.provider,i.kind,i.source,i.title,i.score,i.url FROM items_fts f " + "JOIN items i ON i.id=f.rowid WHERE items_fts MATCH ? ORDER BY i.score DESC LIMIT ?", + (q, limit)).fetchall() + if rows: + return rows + like = f"%{q}%" + return c.execute( + "SELECT provider,kind,source,title,score,url FROM items WHERE title LIKE ? OR text LIKE ? " + "ORDER BY score DESC LIMIT ?", (like, like, limit)).fetchall() + + +def stats(): + with conn() as c: + total = c.execute("SELECT COUNT(*) FROM items").fetchone()[0] + by_prov = c.execute("SELECT provider, COUNT(*) FROM items GROUP BY provider ORDER BY 2 DESC").fetchall() + by_kind = c.execute("SELECT kind, COUNT(*) FROM items GROUP BY kind ORDER BY 2 DESC").fetchall() + priced = c.execute("SELECT COUNT(*) FROM items WHERE price IS NOT NULL").fetchone()[0] + runs = c.execute("SELECT COUNT(*) FROM runs").fetchone()[0] + ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE status='ok'").fetchone()[0] + return {"items": total, "priced": priced, "runs": runs, "runs_ok": ok_runs, + "by_provider": by_prov, "by_kind": by_kind} + + +def export(path, fmt="jsonl", limit=1000): + path = Path(path) + with conn() as c: + rows = c.execute("SELECT provider,kind,source,url,title,text,author,published,lang,tags," + "price,currency,score,extra FROM items ORDER BY score DESC LIMIT ?", + (limit,)).fetchall() + cols = ["provider", "kind", "source", "url", "title", "text", "author", "published", "lang", + "tags", "price", "currency", "score", "extra"] + if fmt == "csv": + import csv + with path.open("w", encoding="utf-8", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols) + w.writeheader() + for r in rows: + w.writerow(dict(zip(cols, r))) + else: + with path.open("w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(dict(zip(cols, r)), ensure_ascii=False) + "\n") + return len(rows), str(path) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7a3aaa0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +httpx>=0.27 +curl_cffi>=0.7 +beautifulsoup4>=4.12 +lxml>=5.0 diff --git a/themes.txt b/themes.txt new file mode 100644 index 0000000..28e036b --- /dev/null +++ b/themes.txt @@ -0,0 +1,9 @@ +# темы для обхода, по одной в строке +парсер твиттера без api +reddit scraping 2026 +free proxy list +jailbreak llm system prompt +combo list mail access +ai api keys free trial +traffic arbitrage app install +telegram parser automation