"""ETERNAL crates.io hunt: registry search + .crate tarball unpack + grep."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import re
import sqlite3
import tarfile
import time
import urllib.parse
import urllib.error
import urllib.request

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "crates_eternal.log")
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
UA["Accept"] = "application/json"

QUERIES = [
    "openai", "anthropic", "claude", "dashscope", "openrouter", "deepseek",
    "groq", "zhipu", "glm", "moonshot", "kimi", "together", "fireworks",
    "replicate", "nvidia nim", "siliconflow", "telegram bot", "llm wrapper",
    "ai sdk", "chatbot", "gpt wrapper", "agent framework", "voice ai",
    "tts", "embedding", "api key", "secret", "config env",
]

SCAN_EXT = (".rs", ".toml", ".json", ".md", ".env", ".yml", ".yaml", ".txt")

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, seen, key, prov, repo):
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "crates", repo))
    return 1


def scan_tarball(url, cur, seen, pkg):
    n = 0
    try:
        req = urllib.request.Request(url, headers=UA)
        with urllib.request.urlopen(req, timeout=40) as r:
            blob = r.read()
    except Exception as e:
        log(f"  crate {pkg} err: {str(e)[:50]}")
        return 0
    if len(blob) > 5_000_000:
        return 0
    try:
        tf = tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz")
        members = tf.getmembers()
    except Exception as e:
        log(f"  untar {pkg} err: {str(e)[:50]}")
        return 0
    for m in members:
        if m.size > 600_000:
            continue
        name = m.name.lower()
        if not name.endswith(SCAN_EXT) and "/src/" not in name:
            if not name.endswith(SCAN_EXT):
                continue
        try:
            fdata = tf.extractfile(m)
            if fdata is None:
                continue
            data = fdata.read().decode(errors="replace")
        except Exception:
            continue
        for prov, rx in PATTERNS:
            for key in rx.findall(data):
                if isinstance(key, tuple):
                    key = key[-1]
                hit = db_add(cur, seen, key, prov, f"crates:{pkg}")
                if hit:
                    n += 1
                    log(f"  + {prov} {key[:24]}... crates:{pkg}/{m.name}")
    return n


def cycle(db, qi):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total = 0
    q = QUERIES[qi % len(QUERIES)]
    qi += 1
    try:
        req = urllib.request.Request(
            f"https://crates.io/api/v1/crates?q={urllib.parse.quote(q)}&per_page=50",
            headers=UA)
        with urllib.request.urlopen(req, timeout=30) as r:
            d = json.loads(r.read())
        crates = d.get("crates", [])
        log(f"query {q!r}: {len(crates)} crates")
        for item in crates[:25]:
            name = item.get("name", "")
            ver = item.get("max_version", "") or item.get("newest_version", "")
            if not name or not ver:
                continue
            tb = f"https://static.crates.io/crates/{name}/{name}-{ver}.crate"
            try:
                total += scan_tarball(tb, cur, seen, name)
            except Exception as e:
                log(f"  crate {name} err: {str(e)[:40]}")
            time.sleep(1)
    except urllib.error.HTTPError as e:
        log(f"crates.io HTTP {e.code} for {q!r}; sleeping 5min")
        db.commit()
        time.sleep(300)
    except Exception as e:
        log(f"cycle err: {str(e)[:80]}")
    db.commit()
    return total, qi


def main():
    log(f"=== crates eternal START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    qi = 0
    while True:
        try:
            n, qi = cycle(db, qi)
            log(f"CYCLE done: +{n}; resting 3min")
        except Exception as e:
            log(f"cycle error (continuing): {e}")
        time.sleep(180)


if __name__ == "__main__":
    main()