autoreg-skills-max/skills/cloudflare-bypass-scraping/templates/email_scraper_cf.py

#!/usr/bin/env python3
"""
Email scraper with CF bypass — two backends:
  1. curl_cffi (lightweight, no browser, works now)  ← DEFAULT
  2. CF Bypass Server (full browser, for JS-challenge sites)

Usage:
  python email_scraper_cf.py https://target.com
  python email_scraper_cf.py https://target.com --backend server
  python email_scraper_cf.py https://target.com --proxy socks5://user:pass@host:port
"""

import re
import sys
import time
import json
from urllib.parse import urljoin, urlparse
from typing import Optional

# ── Backend 1: curl_cffi (lightweight, TLS fingerprint impersonation) ──
try:
    from curl_cffi import requests as cf_requests
    HAS_CURL_CFFI = True
except ImportError:
    HAS_CURL_CFFI = False

# ── Backend 2: CF Bypass Server (full browser) ──
import requests as std_requests

CF_BYPASS_URL = "http://localhost:8000"

# ── Config ──────────────────────────────────────────────────
MAX_PAGES = 50
REQUEST_TIMEOUT = 30
EMAIL_RE = re.compile(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}')
LINK_RE = re.compile(r'href=["\']([^"\']+)["\']')
# ────────────────────────────────────────────────────────────


def fetch_curl_cffi(url: str, proxy: Optional[str] = None) -> str:
    """Fetch using curl_cffi with Chrome 124 TLS fingerprint impersonation."""
    if not HAS_CURL_CFFI:
        raise RuntimeError("curl_cffi not installed. Run: pip install curl_cffi")
    
    proxies = {"http": proxy, "https": proxy} if proxy else None
    resp = cf_requests.get(
        url,
        impersonate="chrome124",
        proxies=proxies,
        timeout=REQUEST_TIMEOUT,
        headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[REDACTED-IP] Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.9",
        }
    )
    return resp.text


def fetch_cf_server(url: str, proxy: Optional[str] = None) -> str:
    """Fetch through local CF Bypass Server (full browser JS challenge solve)."""
    headers = {"x-hostname": urlparse(url).netloc}
    if proxy:
        headers["x-proxy"] = proxy
    
    resp = std_requests.get(
        f"{CF_BYPASS_URL}/html",
        params={"url": url},
        headers=headers,
        timeout=REQUEST_TIMEOUT + 30,  # browser solve takes longer
    )
    resp.raise_for_status()
    return resp.text


def extract_emails(html: str) -> set[str]:
    return set(EMAIL_RE.findall(html))


def extract_links(html: str, base_url: str) -> list[str]:
    base = f"{urlparse(base_url).scheme}://{urlparse(base_url).netloc}"
    links = []
    for href in LINK_RE.findall(html):
        full = urljoin(base_url, href)
        if full.startswith(base) and not full.endswith(
            (".jpg", ".png", ".pdf", ".zip", ".css", ".js", ".ico", ".svg", ".woff", ".woff2")
        ):
            links.append(full)
    return links


def crawl(start_url: str, proxy: Optional[str] = None, backend: str = "curl_cffi") -> dict[str, set[str]]:
    fetch = fetch_curl_cffi if backend == "curl_cffi" else fetch_cf_server
    
    seen: set[str] = set()
    queue: list[str] = [start_url]
    results: dict[str, set[str]] = {}

    while queue and len(seen) < MAX_PAGES:
        url = queue.pop(0)
        if url in seen:
            continue
        seen.add(url)

        try:
            print(f"[{len(seen)}/{MAX_PAGES}] {url}")
            html = fetch(url, proxy)
        except Exception as e:
            print(f"  ⨯ {type(e).__name__}: {e}")
            continue

        emails = extract_emails(html)
        if emails:
            results[url] = emails
            for e in emails:
                print(f"  ✉ {e}")

        for link in extract_links(html, url):
            if link not in seen:
                queue.append(link)

        time.sleep(0.3)

    return results


def quick_scan(url: str, proxy: Optional[str] = None, backend: str = "curl_cffi") -> set[str]:
    fetch = fetch_curl_cffi if backend == "curl_cffi" else fetch_cf_server
    html = fetch(url, proxy)
    return extract_emails(html)


# ── CLI ─────────────────────────────────────────────────────
if __name__ == "__main__":
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    target = sys.argv[1]
    backend = "curl_cffi"
    proxy = None
    quick = False

    i = 2
    while i < len(sys.argv):
        if sys.argv[i] == "--backend" and i + 1 < len(sys.argv):
            backend = sys.argv[i + 1]; i += 1
        elif sys.argv[i] == "--proxy" and i + 1 < len(sys.argv):
            proxy = sys.argv[i + 1]; i += 1
        elif sys.argv[i] == "--quick":
            quick = True
        i += 1

    if backend == "curl_cffi" and not HAS_CURL_CFFI:
        print("curl_cffi not installed. Install: pip install curl_cffi")
        print("Falling back to 'server' backend. Make sure CF Bypass Server is running on port 8000.")
        backend = "server"

    if quick:
        emails = quick_scan(target, proxy, backend)
        print(f"\n{len(emails)} emails on {target}:")
        for e in sorted(emails):
            print(f"  {e}")
    else:
        results = crawl(target, proxy, backend)
        total = sum(len(v) for v in results.values())
        print(f"\n✓ {total} emails across {len(results)} pages")

        out = {url: sorted(emails) for url, emails in results.items()}
        with open("emails_output.json", "w", encoding="utf-8") as f:
            json.dump(out, f, indent=2, ensure_ascii=False)
        print(f"✓ Saved to emails_output.json")