keyhunter-parser/packagist_eternal.py

"""ETERNAL Packagist harvest: search.json + p2 metadata dist.zip scan.
Lane: packagist. Polite sleep, 25 packages per cycle."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import re
import sqlite3
import sys
import time
import urllib.parse
import urllib.request
import zipfile

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "packagist_eternal.log")

QUERIES = ["openai", "anthropic", "claude", "llm", "chatgpt", "gemini",
           "mistral", "groq", "deepseek", "openrouter", "api",
           "ai", "gpt", "sdk", "client", "secrets"]

SCAN_EXT = (".php", ".env", ".json", ".yml", ".yaml", ".md", ".xml",
            ".dist", ".txt", ".js", ".ts", ".py")
MAX_FILE = 2 * 1024 * 1024
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")

seen = set()


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, key, prov, repo):
    if isinstance(key, tuple):
        key = key[0]
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "packagist", repo))
    return 1


def fetch(url, binary=False):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=60) as r:
        blob = r.read()
    return blob if binary else blob.decode("utf-8", "replace")


def scan_zip(blob, cur, pkg):
    n = 0
    try:
        zf = zipfile.ZipFile(io.BytesIO(blob))
    except Exception:
        return 0
    for info in zf.infolist():
        name = info.filename.lower()
        if info.file_size > MAX_FILE:
            continue
        if not (name.endswith(SCAN_EXT) or ".env" in name):
            continue
        try:
            text = zf.read(info).decode("utf-8", "replace")
        except Exception:
            continue
        for prov, rx in PATTERNS:
            for m in rx.findall(text):
                n += db_add(cur, m, prov, f"{pkg}:{info.filename}")
    return n


def scan_package(pkg, cur):
    try:
        meta = json.loads(fetch(
            "https://repo.packagist.org/p2/" + pkg + ".json"))
    except Exception as e:
        log(f"  meta {pkg} err: {str(e)[:60]}")
        return 0
    versions = meta.get("packages", {}).get(pkg, [])
    if not versions:
        return 0
    dist = versions[0].get("dist", {})
    url = dist.get("url")
    if not url or dist.get("type") != "zip":
        return 0
    try:
        blob = fetch(url, binary=True)
    except Exception as e:
        log(f"  zip {pkg} err: {str(e)[:60]}")
        return 0
    return scan_zip(blob, cur, pkg)


def cycle(db):
    cur = db.cursor()
    total = 0
    for q in QUERIES:
        n = 0
        url = ("https://packagist.org/search.json?q=" +
               urllib.parse.quote(q))
        try:
            data = json.loads(fetch(url))
        except Exception as e:
            log(f"  search '{q}' err: {str(e)[:60]}")
            continue
        for p in data.get("results", [])[:25]:
            n += scan_package(p.get("name", ""), cur)
            db.commit()
            time.sleep(1.0)
        total += n
        log(f"  '{q}': +{n} new")
    return total


def main():
    log(f"=== packagist eternal START pid={__import__('os').getpid()} "
        f"queries={len(QUERIES)}")
    try:
        con = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
        cur = con.cursor()
        cur.execute("SELECT val FROM keys WHERE found='packagist'")
        for r in cur.fetchall():
            seen.add(r[0])
        con.close()
    except Exception:
        pass
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    while True:
        try:
            n = cycle(db)
            log(f"CYCLE done +{n}")
        except Exception as e:
            log(f"cycle err: {repr(e)[:100]}")
            time.sleep(30)
        time.sleep(180)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(0)