"""ETERNAL VS Code Marketplace: walk all VS Code extensions via gallery API,
download VSIX, unzip in memory, scan bundled files for API keys/config secrets.

Endpoint: POST https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery
  (anonymous, per-IP rate limit, 150K+ extensions)
Auth: none
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))

import hashlib, io, re, sqlite3, sys, threading, time, zipfile
import requests

S = requests.Session()
S.trust_env = False

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = _os.path.join(BASE, "data", "vscode_eternal.log")
CURSOR_FILE = _os.path.join(BASE, "data", "vscode_cursor.txt")
SEEN_FILE = _os.path.join(BASE, "data", "vscode_seen.txt")
N_THREADS = 3
SLEEP = 90
MAX_ITEMS = 30
VSIX_CAP = 20 * 1024 * 1024  # 20 MB max VSIX download
FILE_CAP = 500_000
PAGE_SIZE = 50

_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

# File extensions to scan inside VSIX
SCAN_EXT = (".js", ".json", ".ts", ".env", ".yaml", ".yml", ".xml",
            ".html", ".txt", ".cfg", ".conf", ".properties", ".ini",
            ".sh", ".bat", ".ps1", ".py", ".rb", ".php")

# Path segments inside VSIX to skip
SKIP_DIRS = ("/node_modules/", "/.git/", "/test/", "/tests/")

# Very long single-line JS (minified bundle) — likely noise but may still
# contain keys; we scan them but skip scanning rows > 100 chars content
MINIFIED_LINE_THRESHOLD = 5000

UA = {"User-Agent": "keyfarm-vscode/1.0"}
GALLERY_URL = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery"
GALLERY_HEADERS = {"Accept": "application/json;api-version=7.2-preview.1", **UA}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
conn.execute("""
CREATE TABLE IF NOT EXISTS keys (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  hash TEXT UNIQUE,
  val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
  plan TEXT, price TEXT, remaining TEXT,
  found TEXT, repo TEXT)
""")
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "vscode", repo))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            return int(f.read().strip() or 0)
    except (FileNotFoundError, ValueError):
        return 0


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(str(c))


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def fetch_page(page):
    """Fetch one page of extensions from the gallery API."""
    body = {
        "filters": [{
            "criteria": [
                {"filterType": 8, "value": "Microsoft.VisualStudio.Code"}
            ],
            "pageNumber": page,
            "pageSize": PAGE_SIZE
        }],
        "flags": 0x206  # IncludeVersions | IncludeFiles | IncludeVersionProperties
    }
    for attempt in range(3):
        try:
            r = S.post(GALLERY_URL, json=body, headers=GALLERY_HEADERS,
                       timeout=30)
            if r.status_code == 429:
                retry = int(r.headers.get("Retry-After", 10))
                log(f"  429 on page {page}, sleeping {retry}s")
                time.sleep(retry)
                continue
            r.raise_for_status()
            data = r.json()
            exts = (data.get("results") or [{}])[0].get("extensions") or []
            return exts
        except requests.exceptions.RequestException as e:
            if attempt < 2:
                time.sleep(2 ** (attempt + 1))
            else:
                log(f"  page {page} fail after 3 retries: {e}")
                return None
    return None


def download_vsix(ext_id, vsix_url):
    """Download VSIX with cap-before-download via HEAD then stream.
    Returns bytes or None on fail/oversize.
    """
    for attempt in range(3):
        try:
            head = S.head(vsix_url, headers=UA, timeout=15)
            cl = int(head.headers.get("Content-Length", 0) or 0)
            if cl > VSIX_CAP:
                log(f"  {ext_id}: Content-Length {cl} > {VSIX_CAP}, skip")
                return None

            if cl > 0:
                r = S.get(vsix_url, headers=UA, timeout=120, stream=True)
                r.raise_for_status()
                chunks = []
                downloaded = 0
                for chunk in r.iter_content(chunk_size=65536):
                    downloaded += len(chunk)
                    if downloaded > VSIX_CAP:
                        log(f"  {ext_id}: stream exceeded cap")
                        return None
                    chunks.append(chunk)
                return b"".join(chunks)
            else:
                # Content-Length 0 or missing — download directly with cap
                r = S.get(vsix_url, headers=UA, timeout=120)
                r.raise_for_status()
                if len(r.content) > VSIX_CAP:
                    log(f"  {ext_id}: body {len(r.content)} > cap")
                    return None
                return r.content
        except requests.exceptions.RequestException as e:
            if attempt < 2:
                time.sleep(2 ** (attempt + 1))
            else:
                log(f"  {ext_id}: VSIX download fail: {e}")
                return None
    return None


def scan_vsix(raw, ext_key):
    """Unzip VSIX in memory, scan text files for keys.
    Returns number of keys found.
    """
    try:
        z = zipfile.ZipFile(io.BytesIO(raw))
    except zipfile.BadZipFile:
        log(f"  {ext_key}: bad zip")
        return 0

    n = 0
    for info in z.infolist():
        fn = info.filename
        # Skip node_modules, .git, test dirs
        if any(sd in fn for sd in SKIP_DIRS):
            continue
        # Only scan relevant extensions
        if not fn.endswith(SCAN_EXT):
            continue
        # Skip huge files (but check member size first — don't read then slice)
        if info.file_size > FILE_CAP:
            continue
        # Skip minified single-line bundles (high noise)
        if info.file_size > MINIFIED_LINE_THRESHOLD:
            # Could still be worth checking the first few bytes for key patterns
            # But full scan of minified JS is very noisy
            pass

        try:
            content = z.read(info.filename)
        except Exception:
            continue

        # Try decode
        try:
            text = content.decode("utf-8", "replace")
        except Exception:
            continue

        # For long single-line content, take first 200KB to avoid noise
        # but keep scanning for patterns
        if len(text) > FILE_CAP:
            text = text[:FILE_CAP]

        hits = scan_text(text)
        for prov, key in hits:
            if not is_junk(key):
                # Use extension key as repo identifier
                db_add(key, prov, ext_key)
                n += 1
    return n


def work(ext):
    """Process one extension: download VSIX, unzip, scan. Returns key count."""
    pub = ext.get("publisher", {}).get("publisherName", "?")
    name = ext.get("extensionName", "?")
    ver_obj = (ext.get("versions") or [{}])[0]
    ver = ver_obj.get("version", "?")
    ext_key = f"{pub}/{name}/{ver}"
    # extract a shorter ext_ident for repo column (max 300 chars)
    repo_slug = f"{pub}/{name}"

    if ext_key in seen:
        return 0

    fb = ver_obj.get("fallbackAssetUri") or ver_obj.get("assetUri")
    if not fb:
        log(f"  {ext_key}: no assetUri, skip")
        mark_seen(ext_key)
        return 0

    vsix_url = f"{fb}/Microsoft.VisualStudio.Services.VSIXPackage"

    raw = download_vsix(ext_key, vsix_url)
    if raw is None:
        mark_seen(ext_key)
        return 0

    n = scan_vsix(raw, repo_slug)
    if n > 0:
        log(f"  {ext_key}: +{n} keys")
    mark_seen(ext_key)
    return n


def cycle(max_items=MAX_ITEMS):
    """Run one bounded cycle: walk pages from cursor."""
    global total, seen

    page = load_cursor()
    if page < 1:
        page = 1
    processed = 0
    fail = 0

    log(f"cycle start: page={page}, max_items={max_items}")

    while processed < max_items:
        exts = fetch_page(page)
        if exts is None:
            log(f"  page {page} returned no extensions, wrap to page 1")
            page = 1
            save_cursor(page)
            continue
        if not exts:
            log(f"  page {page} empty, wrap to page 1")
            page = 1
            save_cursor(page)
            continue

        per_page = min(len(exts), max_items - processed)
        for i in range(per_page):
            try:
                n = work(exts[i])
                if n > 0:
                    processed += 1
                else:
                    processed += 1  # still counts as processed
            except Exception as e:
                log(f"  error processing {exts[i].get('extensionName','?')}: {e}")
                fail += 1
                processed += 1

            if processed >= max_items:
                break

        page += 1
        save_cursor(page)
        # Brief pause between pages
        time.sleep(1)

    log(f"cycle done: page={page} processed={processed} fail={fail} +{total} keys")


def main():
    log("vscode_eternal started")
    if "--once" in sys.argv:
        cycle(max_items=min(MAX_ITEMS, 5))
        log("--once complete")
        return
    while True:
        cycle()
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()