keyhunter-parser/pypi_eternal.py

"""ETERNAL PyPI hunt: RSS updates.xml (fresh 100 uploads every cycle) +
filtered package download (sdist/wheel) + grep. Logs pypi_eternal.log."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import re
import sqlite3
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zipfile

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "pypi_eternal.log")
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# name filter: AI-ish packages likely to embed keys
NAME_HINTS = ("openai", "anthropic", "claude", "gpt", "llm", "ai", "bot",
              "agent", "chat", "dashscope", "zhipu", "glm", "kimi", "moonshot",
              "deepseek", "groq", "openrouter", "together", "fireworks",
              "replicate", "nvidia", "silicon", "eleven", "voice", "tts",
              "telegram", "discord", "slack", "mcp", "rag", "embed", "vector",
              "langchain", "llama", "diffus", "stable")

SCAN_EXT = (".py", ".json", ".md", ".txt", ".env", ".yml", ".yaml", ".cfg",
            ".toml", ".ini", ".js", ".ts")

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, seen, key, prov, repo):
    if isinstance(key, tuple):
        key = key[0]
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "pypi", repo))
    return 1


def grep_blob(blob, cur, seen, pkg, fname):
    n = 0
    try:
        data = blob.decode(errors="replace")
    except Exception:
        return 0
    if len(data) > 400_000:
        return 0
    for prov, rx in PATTERNS:
        for key in rx.findall(data):
            hit = db_add(cur, seen, key, prov, f"pypi:{pkg}")
            if hit:
                n += 1
                log(f"  + {prov} {key[:24]}... pypi:{pkg}/{fname}")
    return n


def scan_pkg(name, cur, seen):
    n = 0
    try:
        req = urllib.request.Request(f"https://pypi.org/pypi/{name}/json", headers=UA)
        with urllib.request.urlopen(req, timeout=15) as r:
            meta = json.loads(r.read())
    except Exception as e:
        log(f"  meta {name} err: {str(e)[:50]}")
        return 0
    urls = (meta.get("urls") or [])[:3]
    for u in urls:
        url = u.get("url", "")
        kind = u.get("packagetype", "")
        if not url:
            continue
        try:
            req = urllib.request.Request(url, headers=UA)
            with urllib.request.urlopen(req, timeout=30) as r:
                blob = r.read()
        except Exception:
            continue
        if len(blob) > 3_000_000:
            continue
        if kind == "bdist_wheel" and url.endswith(".whl"):
            try:
                zf = zipfile.ZipFile(io.BytesIO(blob))
                for m in zf.namelist():
                    if not m.lower().endswith(SCAN_EXT):
                        continue
                    n += grep_blob(zf.read(m), cur, seen, name, m)
            except Exception:
                pass
        else:
            import tarfile
            try:
                tf = tarfile.open(fileobj=io.BytesIO(blob))
                for m in tf.getmembers():
                    if m.size > 400_000 or not m.isfile():
                        continue
                    if not m.name.lower().endswith(SCAN_EXT):
                        continue
                    fdata = tf.extractfile(m)
                    if fdata is None:
                        continue
                    n += grep_blob(fdata.read(), cur, seen, name, m.name)
            except Exception:
                pass
        time.sleep(0.5)
    return n


def cycle(db):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total = 0
    try:
        req = urllib.request.Request("https://pypi.org/rss/updates.xml", headers=UA)
        with urllib.request.urlopen(req, timeout=20) as r:
            tree = ET.fromstring(r.read())
        titles = [i.findtext("title", "") for i in tree.findall(".//item")]
        cand = []
        for t in titles:
            name = t.split()[0] if t else ""
            name = name.rsplit(" ", 1)[0].strip()
            if name and any(h in name.lower() for h in NAME_HINTS) and len(name) < 60:
                cand.append(name)
        cand = list(dict.fromkeys(cand))[:25]
        log(f"rss: {len(titles)} updates, {len(cand)} candidates")
        for name in cand:
            total += scan_pkg(name, cur, seen)
            time.sleep(1)
    except Exception as e:
        log(f"cycle err: {e}")
    db.commit()
    return total


def main():
    log(f"=== pypi eternal START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    while True:
        try:
            n = cycle(db)
            log(f"CYCLE done: +{n}; resting 10min")
        except Exception as e:
            log(f"cycle error (continuing): {e}")
        time.sleep(600)


if __name__ == "__main__":
    main()