keyhunter-parser/npm_eternal.py

"""ETERNAL deep NPM hunt: registry search (size 250) + full tarball unpack + grep.
24.4% conversion source — crank volume. Logs npm_eternal.log."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import re
import sqlite3
import tarfile
import time
import urllib.parse
import urllib.request

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "npm_eternal.log")
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

QUERIES = [
    "openai api key", "anthropic api key", "claude api", "dashscope",
    "openrouter", "deepseek api", "groq api", "zhipu glm", "glm-4",
    "kimi moonshot", "moonshot api", "together ai api", "fireworks ai",
    "replicate api", "nvidia nim", "siliconflow", "ai agent key",
    "telegram bot api", "discord bot ai", "llm wrapper", "ai sdk key",
    "chatbot api", "gpt wrapper", "agent framework", "voice ai",
    "tts api key", "embedding api",
]

SCAN_EXT = (".js", ".ts", ".json", ".md", ".env", ".yml", ".yaml", ".mjs", ".cjs", ".txt")

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, seen, key, prov, repo):
    if isinstance(key, tuple):
        key = key[0]
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "npm", repo))
    return 1


def scan_tarball(url, cur, seen, pkg):
    n = 0
    try:
        req = urllib.request.Request(url, headers=UA)
        with urllib.request.urlopen(req, timeout=30) as r:
            blob = r.read()
    except Exception as e:
        log(f"  tarball {pkg} err: {str(e)[:50]}")
        return 0
    if len(blob) > 3_000_000:
        return 0
    try:
        tf = tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz")
        members = tf.getmembers()
    except Exception as e:
        log(f"  untar {pkg} err: {str(e)[:50]}")
        return 0
    for m in members:
        if m.size > 400_000:
            continue
        name = m.name.lower()
        if not name.endswith(SCAN_EXT):
            continue
        if "node_modules/" in name:
            continue
        try:
            fdata = tf.extractfile(m)
            if fdata is None:
                continue
            data = fdata.read().decode(errors="replace")
        except Exception:
            continue
        for prov, rx in PATTERNS:
            for key in rx.findall(data):
                hit = db_add(cur, seen, key, prov, f"npm:{pkg}")
                if hit:
                    n += 1
                    log(f"  + {prov} {key[:24]}... npm:{pkg}/{m.name}")
    return n


def cycle(db, qi):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total = 0
    q = QUERIES[qi % len(QUERIES)]
    qi += 1
    try:
        req = urllib.request.Request(
            f"https://registry.npmjs.org/-/v1/search?text={urllib.parse.quote(q)}&size=250",
            headers=UA)
        with urllib.request.urlopen(req, timeout=20) as r:
            d = json.loads(r.read())
        objects = d.get("objects", [])
        log(f"query {q!r}: {len(objects)} packages")
        for item in objects[:40]:
            pkg = item.get("package", {})
            name = pkg.get("name", "")
            ver = pkg.get("version", "")
            if not name or name.startswith("@types/"):
                continue
            try:
                req = urllib.request.Request(
                    f"https://registry.npmjs.org/{name}", headers=UA)
                with urllib.request.urlopen(req, timeout=15) as r:
                    meta = json.loads(r.read())
                tarball = (meta.get("versions", {}).get(ver, {})
                           .get("dist", {}).get("tarball", ""))
                if not tarball and meta.get("dist-tags"):
                    latest = meta["dist-tags"].get("latest")
                    tarball = (meta.get("versions", {}).get(latest, {})
                               .get("dist", {}).get("tarball", ""))
                if tarball:
                    total += scan_tarball(tarball, cur, seen, name)
            except Exception as e:
                log(f"  meta {name} err: {str(e)[:50]}")
            time.sleep(1)
    except Exception as e:
        log(f"cycle err: {e}")
    db.commit()
    return total, qi


def main():
    log(f"=== npm eternal deep START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    qi = 0
    while True:
        try:
            n, qi = cycle(db, qi)
            log(f"CYCLE done: +{n}; resting 3min")
        except Exception as e:
            log(f"cycle error (continuing): {e}")
        time.sleep(180)


if __name__ == "__main__":
    main()