omni-parse/tools/cf_cookies.py

"""Снять cookies с CF-защищённых площадок через реальный браузер (Camoufox).

Домены вроде exploit.in / xss.is / blackhatworld отдают JS-челлендж в ответ на
любой HTTP-клиент. Скрипт открывает площадку настоящим браузером, ждёт прохода
проверки и складывает cookies в data/cookies.json — оттуда их берёт omni/net.py
(config.cookies_for), и html_listing начинает получать контент.

Запуск:
    python tools/cf_cookies.py                 # все из omni.sources_dark.NEED_COOKIE
    python tools/cf_cookies.py xss.is          # конкретные хосты
    python tools/cf_cookies.py --check         # только проверить, что уже есть
"""
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

OUT = ROOT / "data" / "cookies.json"
WAIT = 18  # секунд на проход челленджа


def load():
    try:
        return json.loads(OUT.read_text(encoding="utf-8"))
    except Exception:
        return {}


def save(d):
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")


def grab(host):
    """Открыть https://host, дождаться прохода и вернуть строку cookie."""
    from camoufox.sync_api import Camoufox

    url = host if host.startswith("http") else "https://" + host
    with Camoufox(headless=True, humanize=True, geoip=False) as br:
        pg = br.new_page()
        pg.goto(url, wait_until="domcontentloaded", timeout=60000)
        deadline = time.time() + WAIT
        last = ""
        while time.time() < deadline:
            try:
                title = pg.title() or ""
                html = pg.content() or ""
            except Exception:
                title, html = "", ""
            last = title
            low = (title + html[:3000]).lower()
            if not any(k in low for k in ("just a moment", "verify humanity", "checking your browser",
                                          "attention required", "ddos-guard")):
                break
            time.sleep(2.5)
        ck = pg.context.cookies()
        html_len = len(pg.content() or "")
        br.close()

    parts = []
    for c in ck:
        dom = (c.get("domain") or "").lstrip(".")
        if dom and dom not in host and host not in dom:
            continue
        parts.append("%s=%s" % (c["name"], c["value"]))
    return "; ".join(parts), last, html_len


def probe(host, cookie):
    """Проверить, что с cookie приходит настоящая страница."""
    import urllib.request
    url = host if host.startswith("http") else "https://" + host
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/131.0 Safari/537.36",
        "Cookie": cookie,
        "Accept-Language": "ru,en;q=0.8",
    })
    try:
        with urllib.request.urlopen(req, timeout=25) as r:
            body = r.read(300_000).decode("utf-8", "ignore")
            low = body[:4000].lower()
            bad = any(k in low for k in ("just a moment", "verify humanity", "checking your browser"))
            return r.status, len(body), not bad
    except Exception as e:
        return 0, 0, False


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    check_only = "--check" in sys.argv

    if not args:
        try:
            from omni.sources_dark import NEED_COOKIE
            args = list(NEED_COOKIE)
        except Exception:
            args = ["exploit.in", "xss.is", "www.blackhatworld.com"]

    store = load()
    print("cookies-файл:", OUT, "| записей уже:", len(store))

    if check_only:
        for h in args:
            ck = store.get(h) or ""
            if not ck:
                print("%-26s нет cookie" % h)
                continue
            st, ln, ok = probe(h, ck)
            print("%-26s %s len %-7d %s" % (h, st, ln, "OK" if ok else "закрыт"))
        return 0

    ok_n = 0
    for h in args:
        print("→", h)
        try:
            cookie, title, html_len = grab(h)
        except Exception as e:
            print("   браузер: %s: %s" % (type(e).__name__, str(e)[:90]))
            continue
        st, ln, ok = probe(h, cookie) if cookie else (0, 0, False)
        print("   title=%r html=%d cookie=%d | проверка %s len %d %s"
              % (title[:40], html_len, cookie.count("="), st, ln, "OK" if ok else "закрыт"))
        if cookie and ok:
            store[h] = cookie
            ok_n += 1

    save(store)
    print("\nсохранено рабочих: %d | всего в файле: %d" % (ok_n, len(store)))
    print("теперь: python -m omni feed --only <источник>")
    return 0


if __name__ == "__main__":
    sys.exit(main())