"""ETERNAL jsDelivr CDN: walk built npm package files for hardcoded secrets.

Endpoint(s):
  data.jsdelivr.com/v1/stats/packages  (paginated, max 100 pages×100)
  data.jsdelivr.com/v1/packages/npm/<pkg>  (tags + versions)
  data.jsdelivr.com/v1/packages/npm/<pkg>@<ver>  (file tree)
  cdn.jsdelivr.net/npm/<pkg>@<ver>/<path>  (file download)

Auth: none (per-IP anonymous; no rate-limit headers observed, but Cloudflare-backed)
Collision: data.jsdelivr.com AND cdn.jsdelivr.net — hosts NOT used by any existing lane.
  Package _names_ overlap with npm_eternal (both walk npm ecosystem), but the HOSTS are
  different (npm=registry.npmjs.org, jsd=data.jsdelivr.com/cdn.jsdelivr.net) so per-IP
  quota is independent. The files differ too: npm_eternal downloads raw source tarballs;
  jsd fetches individual CDN-served built bundles — where accidental hardcoded keys live.
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)

import hashlib, json, re, sqlite3, sys, threading, time
import requests

S = requests.Session()
S.trust_env = False  # bypass local dashscope proxy

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = _os.path.join(BASE, "data", "jsdelivr_eternal.log")
CURSOR_FILE = _os.path.join(BASE, "data", "jsdelivr_cursor.txt")
SEEN_FILE = _os.path.join(BASE, "data", "jsdelivr_seen.txt")
N_THREADS = 4
SLEEP = 90
MAX_ITEMS = 200       # packages per cycle; --once uses min(200,5)
MAX_FILES_PKG = 20    # files to download per package (--once: 5)
FILE_CAP = 500_000    # skip files larger than this
STATS_PAGE_LIMIT = 100  # max pages for stats pagination
ONCE = "--once" in sys.argv  # used to cap files per package in --once mode

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

UA = {"User-Agent": "keyfarm-jsdelivr/1.0"}

# Small text file extensions likely to contain hardcoded keys in built bundles
# Dotfiles like .env, .npmrc have no extension after leading dot.
# os.path.splitext('.env') -> ('.env', '').  We match by full basename
# after splitting on the LAST dot.
DOTFILE_NAMES = frozenset((
    ".env", ".envrc", ".npmrc", ".gitignore",
    ".gitattributes", ".eslintrc", ".prettierrc",
))

SCAN_EXT = frozenset((
    ".js", ".json", ".ts", ".mjs", ".cjs",
    ".yml", ".yaml", ".txt", ".toml",
    ".rc", ".config", ".conf", ".ini", ".cfg",
))

DATA_BASE = "https://data.jsdelivr.com/v1"
CDN_BASE = "https://cdn.jsdelivr.net/npm"


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
conn.execute("""
CREATE TABLE IF NOT EXISTS keys (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  hash TEXT UNIQUE,
  val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
  plan TEXT, price TEXT, remaining TEXT,
  found TEXT, repo TEXT)
""")
conn.commit()
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "jsdelivr", repo[:300]))
        total += 1


def load_cursor():
    """Returns (page, pkg_idx) — position in stats pagination."""
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            raw = f.read().strip()
            if raw:
                parts = raw.split()
                return int(parts[0]), int(parts[1])
    except (FileNotFoundError, ValueError, IndexError):
        pass
    return 1, 0


def save_cursor(page, pkg_idx):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(f"{page} {pkg_idx}")


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                s.add(line.rstrip("\n"))
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(url):
    seen.add(url)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(url + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def req_json(url):
    """GET JSON with retry + backoff. Returns decoded list/dict/None."""
    for attempt in range(3):
        try:
            r = S.get(url, headers=UA, timeout=30)
            if r.status_code == 429:
                ra = r.headers.get("Retry-After", "10")
                wait = int(ra) if ra.isdigit() else 10
                log(f"  429 {url[:80]} — sleep {wait}s")
                time.sleep(wait)
                continue
            if r.status_code == 404:
                return None
            if r.status_code != 200:
                log(f"  HTTP {r.status_code} {url[:80]}")
                if r.status_code >= 500:
                    time.sleep([2, 4, 8][attempt])
                    continue
                return None
            return r.json()
        except (requests.ConnectionError, requests.Timeout) as e:
            log(f"  net error: {e} — retry {attempt+1}/3")
            time.sleep([2, 4, 8][attempt])
    return None


def fetch_stats_packages(page):
    """Fetch one page from stats/packages. Returns list of {type,name} or None."""
    url = f"{DATA_BASE}/stats/packages?limit=100&page={page}"
    data = req_json(url)
    if data is None:
        return None
    # Filter to npm type only
    return [p for p in data if p.get("type") == "npm"]


def get_package_info(pkg_name):
    """Get package info (tags, versions). Returns dict or None."""
    url = f"{DATA_BASE}/packages/npm/{pkg_name}"
    return req_json(url)


def get_file_tree(pkg_name, version):
    """Get file tree for pkg@version. Returns list of files or None."""
    url = f"{DATA_BASE}/packages/npm/{pkg_name}@{version}"
    data = req_json(url)
    if data is None:
        return None
    return data.get("files")


def collect_text_files(files, prefix=""):
    """Recursively walk file tree, yield (path, size) for small text files."""
    if not files:
        return
    for entry in files:
        name = entry.get("name", "")
        full = prefix + name
        if entry.get("type") == "directory":
            yield from collect_text_files(entry.get("files", []),
                                          prefix=full + "/")
        elif entry.get("type") == "file":
            _, ext = _os.path.splitext(name)
            ext = ext.lower()
            if ext in SCAN_EXT or name in DOTFILE_NAMES:
                size = entry.get("size", FILE_CAP + 1)
                if size <= FILE_CAP:
                    yield full, size


def fetch_and_scan(pkg_name, version, path, size):
    """Download file from CDN, decode, scan_text, db_add. Returns key count."""
    seen_key = f"{pkg_name}@{version}/{path}"
    if seen_key in seen:
        return 0
    url = f"{CDN_BASE}/{pkg_name}@{version}/{path}"
    try:
        # cap-before-download: check Content-Length first
        head = S.head(url, headers=UA, timeout=15)
        if head.status_code != 200:
            return 0
        cl = head.headers.get("Content-Length")
        if cl and int(cl) > FILE_CAP:
            return 0

        r = S.get(url, headers=UA, timeout=60, stream=True)
        if r.status_code != 200:
            return 0

        # Check content length from actual response
        cl2 = r.headers.get("Content-Length")
        if cl2 and int(cl2) > FILE_CAP:
            return 0

        # Read in chunks, accumulate into list to avoid O(n^2) buf+=chunk
        chunks = []
        total_read = 0
        for chunk in r.iter_content(65536):
            chunks.append(chunk)
            total_read += len(chunk)
            if total_read > FILE_CAP:
                return 0
        raw = b"".join(chunks)
        if len(raw) > FILE_CAP:
            return 0
    except Exception as e:
        log(f"  download fail {url[:100]}: {e}")
        return 0

    text = raw.decode("utf-8", "replace")
    hits = scan_text(text)
    if not hits:
        mark_seen(seen_key)
        return 0

    n = 0
    repo = f"{pkg_name}@{version}/{path}"
    for prov, key in hits:
        if is_junk(key):
            continue
        db_add(key, prov, repo)
        n += 1
    if n:
        mark_seen(seen_key)
    return n


def process_package(pkg_name, max_files):
    """Process one npm package: get info, tree, download+scan text files.
    Returns (processed_files, keys_found, fail)."""
    info = get_package_info(pkg_name)
    if info is None:
        return 0, 0, 1

    version = (info.get("tags") or {}).get("latest")
    if not version:
        # fallback: first version in versions array
        versions = info.get("versions") or []
        if versions:
            version = versions[0].get("version")
    if not version:
        return 0, 0, 1

    files = get_file_tree(pkg_name, version)
    if files is None:
        return 0, 0, 1

    text_files = []
    for path, size in collect_text_files(files):
        text_files.append((path, size))

    n_files = 0
    n_keys = 0
    fails = 0

    # Process small files first
    text_files.sort(key=lambda x: x[1])
    for path, size in text_files[:max_files]:
        k = fetch_and_scan(pkg_name, version, path, size)
        if k == 0:
            # Only count as fail if file not already seen
            sk = f"{pkg_name}@{version}/{path}"
            if sk not in seen:
                fails += 1
        else:
            n_keys += k
        n_files += 1

    return n_files, n_keys, fails


def cycle(max_items=MAX_ITEMS):
    global total, seen
    page, pkg_idx = load_cursor()
    processed = 0
    keys_found = 0
    pkg_idx_start = pkg_idx

    log(f"cycle start — page={page} pkg_idx={pkg_idx} max_pkgs={max_items}")

    while processed < max_items:
        pkgs = fetch_stats_packages(page)
        if pkgs is None:
            log(f"  stats page {page} failed, resetting to page 1")
            page, pkg_idx = 1, 0
            continue
        if not pkgs:
            # no more pages — wrap around
            log("  no more pages, resetting")
            page, pkg_idx = 1, 0
            # Try the next page
            continue

        while pkg_idx < len(pkgs) and processed < max_items:
            pkg = pkgs[pkg_idx]
            pkg_name = pkg.get("name", "")
            if not pkg_name:
                pkg_idx += 1
                continue

            log(f"  [{processed+1}] {pkg_name}")
            max_files = 5 if ONCE else MAX_FILES_PKG
            n_f, n_k, n_fail = process_package(pkg_name, max_files)
            log(f"    files={n_f} keys={n_k} fails={n_fail}")
            processed += 1
            pkg_idx += 1
            keys_found += n_k

            # Save cursor after each package
            save_cursor(page, pkg_idx)

        if pkg_idx >= len(pkgs):
            page += 1
            pkg_idx = 0

        if page > STATS_PAGE_LIMIT:
            log(f"  reached page limit ({STATS_PAGE_LIMIT}), wrapping")
            page, pkg_idx = 1, 0
            save_cursor(page, pkg_idx)
            break

    save_cursor(page, pkg_idx)
    log(f"cycle done — pkgs={processed} keys={keys_found} total={total}")


def main():
    log("=== jsdelivr_eternal started ===")
    if ONCE:
        log("  --once mode: single bounded cycle")
        mx = min(MAX_ITEMS, 5)
        cycle(max_items=mx)
        log("jsdelivr_eternal --once complete")
        return

    while True:
        cycle()
        log(f"  sleep {SLEEP}s")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()