autoreg-skills-max: initial public release (restored, redacted, hardened)

skills/cloudflare-bypass-scraping/templates/email_scraper_cf.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Email scraper with CF bypass — two backends:
+  1. curl_cffi (lightweight, no browser, works now)  ← DEFAULT
+  2. CF Bypass Server (full browser, for JS-challenge sites)
+
+Usage:
+  python email_scraper_cf.py https://target.com
+  python email_scraper_cf.py https://target.com --backend server
+  python email_scraper_cf.py https://target.com --proxy socks5://user:pass@host:port
+"""
+
+import re
+import sys
+import time
+import json
+from urllib.parse import urljoin, urlparse
+from typing import Optional
+
+# ── Backend 1: curl_cffi (lightweight, TLS fingerprint impersonation) ──
+try:
+    from curl_cffi import requests as cf_requests
+    HAS_CURL_CFFI = True
+except ImportError:
+    HAS_CURL_CFFI = False
+
+# ── Backend 2: CF Bypass Server (full browser) ──
+import requests as std_requests
+
+CF_BYPASS_URL = "http://localhost:8000"
+
+# ── Config ──────────────────────────────────────────────────
+MAX_PAGES = 50
+REQUEST_TIMEOUT = 30
+EMAIL_RE = re.compile(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}')
+LINK_RE = re.compile(r'href=["\']([^"\']+)["\']')
+# ────────────────────────────────────────────────────────────
+
+
+def fetch_curl_cffi(url: str, proxy: Optional[str] = None) -> str:
+    """Fetch using curl_cffi with Chrome 124 TLS fingerprint impersonation."""
+    if not HAS_CURL_CFFI:
+        raise RuntimeError("curl_cffi not installed. Run: pip install curl_cffi")
+    
+    proxies = {"http": proxy, "https": proxy} if proxy else None
+    resp = cf_requests.get(
+        url,
+        impersonate="chrome124",
+        proxies=proxies,
+        timeout=REQUEST_TIMEOUT,
+        headers={
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[REDACTED-IP] Safari/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+        }
+    )
+    return resp.text
+
+
+def fetch_cf_server(url: str, proxy: Optional[str] = None) -> str:
+    """Fetch through local CF Bypass Server (full browser JS challenge solve)."""
+    headers = {"x-hostname": urlparse(url).netloc}
+    if proxy:
+        headers["x-proxy"] = proxy
+    
+    resp = std_requests.get(
+        f"{CF_BYPASS_URL}/html",
+        params={"url": url},
+        headers=headers,
+        timeout=REQUEST_TIMEOUT + 30,  # browser solve takes longer
+    )
+    resp.raise_for_status()
+    return resp.text
+
+
+def extract_emails(html: str) -> set[str]:
+    return set(EMAIL_RE.findall(html))
+
+
+def extract_links(html: str, base_url: str) -> list[str]:
+    base = f"{urlparse(base_url).scheme}://{urlparse(base_url).netloc}"
+    links = []
+    for href in LINK_RE.findall(html):
+        full = urljoin(base_url, href)
+        if full.startswith(base) and not full.endswith(
+            (".jpg", ".png", ".pdf", ".zip", ".css", ".js", ".ico", ".svg", ".woff", ".woff2")
+        ):
+            links.append(full)
+    return links
+
+
+def crawl(start_url: str, proxy: Optional[str] = None, backend: str = "curl_cffi") -> dict[str, set[str]]:
+    fetch = fetch_curl_cffi if backend == "curl_cffi" else fetch_cf_server
+    
+    seen: set[str] = set()
+    queue: list[str] = [start_url]
+    results: dict[str, set[str]] = {}
+
+    while queue and len(seen) < MAX_PAGES:
+        url = queue.pop(0)
+        if url in seen:
+            continue
+        seen.add(url)
+
+        try:
+            print(f"[{len(seen)}/{MAX_PAGES}] {url}")
+            html = fetch(url, proxy)
+        except Exception as e:
+            print(f"  ⨯ {type(e).__name__}: {e}")
+            continue
+
+        emails = extract_emails(html)
+        if emails:
+            results[url] = emails
+            for e in emails:
+                print(f"  ✉ {e}")
+
+        for link in extract_links(html, url):
+            if link not in seen:
+                queue.append(link)
+
+        time.sleep(0.3)
+
+    return results
+
+
+def quick_scan(url: str, proxy: Optional[str] = None, backend: str = "curl_cffi") -> set[str]:
+    fetch = fetch_curl_cffi if backend == "curl_cffi" else fetch_cf_server
+    html = fetch(url, proxy)
+    return extract_emails(html)
+
+
+# ── CLI ─────────────────────────────────────────────────────
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(__doc__)
+        sys.exit(1)
+
+    target = sys.argv[1]
+    backend = "curl_cffi"
+    proxy = None
+    quick = False
+
+    i = 2
+    while i < len(sys.argv):
+        if sys.argv[i] == "--backend" and i + 1 < len(sys.argv):
+            backend = sys.argv[i + 1]; i += 1
+        elif sys.argv[i] == "--proxy" and i + 1 < len(sys.argv):
+            proxy = sys.argv[i + 1]; i += 1
+        elif sys.argv[i] == "--quick":
+            quick = True
+        i += 1
+
+    if backend == "curl_cffi" and not HAS_CURL_CFFI:
+        print("curl_cffi not installed. Install: pip install curl_cffi")
+        print("Falling back to 'server' backend. Make sure CF Bypass Server is running on port 8000.")
+        backend = "server"
+
+    if quick:
+        emails = quick_scan(target, proxy, backend)
+        print(f"\n{len(emails)} emails on {target}:")
+        for e in sorted(emails):
+            print(f"  {e}")
+    else:
+        results = crawl(target, proxy, backend)
+        total = sum(len(v) for v in results.values())
+        print(f"\n✓ {total} emails across {len(results)} pages")
+
+        out = {url: sorted(emails) for url, emails in results.items()}
+        with open("emails_output.json", "w", encoding="utf-8") as f:
+            json.dump(out, f, indent=2, ensure_ascii=False)
+        print(f"✓ Saved to emails_output.json")