KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
200
rubygems_eternal.py
Normal file
200
rubygems_eternal.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""ETERNAL rubygems hunt: search + .gem unpack (tar -> data.tar.gz -> members) + grep."""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import time
|
||||
import random
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "rubygems_eternal.log")
|
||||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "application/json"}
|
||||
|
||||
QUERIES = [
|
||||
"openai", "anthropic", "claude", "dashscope", "openrouter", "deepseek",
|
||||
"groq", "zhipu", "glm", "moonshot", "kimi", "together", "fireworks",
|
||||
"replicate", "nvidia", "siliconflow", "telegram bot", "llm", "ai sdk",
|
||||
"chatbot", "gpt", "agent", "voice", "tts", "embedding", "api key",
|
||||
"secret", "config env", "slack api", "discord bot",
|
||||
]
|
||||
|
||||
SCAN_EXT = (".rb", ".json", ".md", ".env", ".yml", ".yaml", ".txt", ".rake")
|
||||
|
||||
from key_patterns import PATTERNS
|
||||
|
||||
GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
|
||||
"fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
|
||||
"aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
|
||||
"changeme", "deadbeef", "fffffff", "0000000")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
def db_add(cur, seen, key, prov, repo):
|
||||
if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
|
||||
return 0
|
||||
seen.add(key)
|
||||
cur.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
|
||||
"rubygems", repo))
|
||||
return 1
|
||||
|
||||
|
||||
def _rubygems_fetch(url, timeout=30, max_retries=3, fallback_to_metadata=False):
|
||||
"""Fetch URL with retries, jitter, and optional gems.json fallback on search 403."""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=UA)
|
||||
req.add_header("Accept-Encoding", "identity")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 403 and fallback_to_metadata and attempt == 0:
|
||||
alt_url = "https://rubygems.org/api/v1/gems.json"
|
||||
log(f" HTTP 403 on search, falling back to {alt_url}")
|
||||
try:
|
||||
req2 = urllib.request.Request(alt_url, headers=UA)
|
||||
req2.add_header("Accept-Encoding", "identity")
|
||||
with urllib.request.urlopen(req2, timeout=timeout) as r2:
|
||||
return r2.read()
|
||||
except Exception:
|
||||
log(f" metadata fallback also failed, continuing retry")
|
||||
delay = 5 + random.uniform(1, 3)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if e.code in (403, 429) or e.code >= 500:
|
||||
if attempt < max_retries - 1:
|
||||
delay = 5 * (2 ** attempt) + random.uniform(1, 3)
|
||||
log(f" HTTP {e.code}, retry {attempt+2}/{max_retries} in {delay:.0f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
log(f" HTTP {e.code} non-retryable")
|
||||
return None
|
||||
except Exception as e:
|
||||
log(f" fetch error: {str(e)[:60]}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(5 + random.uniform(1, 3))
|
||||
continue
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def scan_tarball(url, cur, seen, pkg):
|
||||
n = 0
|
||||
blob = _rubygems_fetch(url, timeout=40)
|
||||
if blob is None:
|
||||
log(f" gem {pkg} err: failed after retries")
|
||||
return 0
|
||||
if len(blob) > 5_000_000:
|
||||
return 0
|
||||
try:
|
||||
outer = tarfile.open(fileobj=io.BytesIO(blob), mode="r:")
|
||||
data_member = None
|
||||
for m in outer.getmembers():
|
||||
if m.name == "data.tar.gz":
|
||||
data_member = m
|
||||
break
|
||||
if data_member is None:
|
||||
return 0
|
||||
inner_blob = outer.extractfile(data_member).read()
|
||||
tf = tarfile.open(fileobj=io.BytesIO(inner_blob), mode="r:gz")
|
||||
members = tf.getmembers()
|
||||
except Exception as e:
|
||||
log(f" ungz {pkg} err: {str(e)[:50]}")
|
||||
return 0
|
||||
for m in members:
|
||||
if m.size > 600_000:
|
||||
continue
|
||||
name = m.name.lower()
|
||||
if not name.endswith(SCAN_EXT):
|
||||
continue
|
||||
try:
|
||||
fdata = tf.extractfile(m)
|
||||
if fdata is None:
|
||||
continue
|
||||
data = fdata.read().decode(errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
for prov, rx in PATTERNS:
|
||||
for key in rx.findall(data):
|
||||
if isinstance(key, tuple):
|
||||
key = key[-1]
|
||||
hit = db_add(cur, seen, key, prov, f"gem:{pkg}")
|
||||
if hit:
|
||||
n += 1
|
||||
log(f" + {prov} {key[:24]}... gem:{pkg}/{m.name}")
|
||||
return n
|
||||
|
||||
|
||||
def cycle(db, qi):
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT val FROM keys")
|
||||
seen = {r[0] for r in cur.fetchall()}
|
||||
total = 0
|
||||
q = QUERIES[qi % len(QUERIES)]
|
||||
qi += 1
|
||||
try:
|
||||
data = _rubygems_fetch(
|
||||
f"https://rubygems.org/api/v1/search.json?query={urllib.parse.quote(q)}",
|
||||
timeout=30, fallback_to_metadata=True)
|
||||
if data is None:
|
||||
log(f"rubygems fetch failed for {q!r}; sleeping 5min")
|
||||
db.commit()
|
||||
time.sleep(300)
|
||||
return total, qi
|
||||
gems = json.loads(data)
|
||||
log(f"query {q!r}: {len(gems)} gems")
|
||||
for item in gems[:25]:
|
||||
name = item.get("name", "")
|
||||
ver = item.get("version", "")
|
||||
if not name or not ver:
|
||||
continue
|
||||
dl = f"https://rubygems.org/downloads/{name}-{ver}.gem"
|
||||
try:
|
||||
total += scan_tarball(dl, cur, seen, name)
|
||||
except Exception as e:
|
||||
log(f" gem {name} err: {str(e)[:40]}")
|
||||
time.sleep(random.uniform(3, 5))
|
||||
except urllib.error.HTTPError as e:
|
||||
log(f"rubygems HTTP {e.code} for {q!r}; sleeping 5min")
|
||||
db.commit()
|
||||
time.sleep(300)
|
||||
except Exception as e:
|
||||
log(f"cycle err: {str(e)[:80]}")
|
||||
db.commit()
|
||||
return total, qi
|
||||
|
||||
|
||||
def main():
|
||||
log(f"=== rubygems eternal START pid={__import__('os').getpid()} ===")
|
||||
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
||||
qi = 0
|
||||
while True:
|
||||
try:
|
||||
n, qi = cycle(db, qi)
|
||||
log(f"CYCLE done: +{n}; resting 3min")
|
||||
except Exception as e:
|
||||
log(f"cycle error (continuing): {e}")
|
||||
time.sleep(180)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user