KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
387
jsdelivr_eternal.py
Normal file
387
jsdelivr_eternal.py
Normal file
@@ -0,0 +1,387 @@
|
||||
"""ETERNAL jsDelivr CDN: walk built npm package files for hardcoded secrets.
|
||||
|
||||
Endpoint(s):
|
||||
data.jsdelivr.com/v1/stats/packages (paginated, max 100 pages×100)
|
||||
data.jsdelivr.com/v1/packages/npm/<pkg> (tags + versions)
|
||||
data.jsdelivr.com/v1/packages/npm/<pkg>@<ver> (file tree)
|
||||
cdn.jsdelivr.net/npm/<pkg>@<ver>/<path> (file download)
|
||||
|
||||
Auth: none (per-IP anonymous; no rate-limit headers observed, but Cloudflare-backed)
|
||||
Collision: data.jsdelivr.com AND cdn.jsdelivr.net — hosts NOT used by any existing lane.
|
||||
Package _names_ overlap with npm_eternal (both walk npm ecosystem), but the HOSTS are
|
||||
different (npm=registry.npmjs.org, jsd=data.jsdelivr.com/cdn.jsdelivr.net) so per-IP
|
||||
quota is independent. The files differ too: npm_eternal downloads raw source tarballs;
|
||||
jsd fetches individual CDN-served built bundles — where accidental hardcoded keys live.
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib, json, re, sqlite3, sys, threading, time
|
||||
import requests
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False # bypass local dashscope proxy
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "jsdelivr_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "jsdelivr_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "jsdelivr_seen.txt")
|
||||
N_THREADS = 4
|
||||
SLEEP = 90
|
||||
MAX_ITEMS = 200 # packages per cycle; --once uses min(200,5)
|
||||
MAX_FILES_PKG = 20 # files to download per package (--once: 5)
|
||||
FILE_CAP = 500_000 # skip files larger than this
|
||||
STATS_PAGE_LIMIT = 100 # max pages for stats pagination
|
||||
ONCE = "--once" in sys.argv # used to cap files per package in --once mode
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-jsdelivr/1.0"}
|
||||
|
||||
# Small text file extensions likely to contain hardcoded keys in built bundles
|
||||
# Dotfiles like .env, .npmrc have no extension after leading dot.
|
||||
# os.path.splitext('.env') -> ('.env', ''). We match by full basename
|
||||
# after splitting on the LAST dot.
|
||||
DOTFILE_NAMES = frozenset((
|
||||
".env", ".envrc", ".npmrc", ".gitignore",
|
||||
".gitattributes", ".eslintrc", ".prettierrc",
|
||||
))
|
||||
|
||||
SCAN_EXT = frozenset((
|
||||
".js", ".json", ".ts", ".mjs", ".cjs",
|
||||
".yml", ".yaml", ".txt", ".toml",
|
||||
".rc", ".config", ".conf", ".ini", ".cfg",
|
||||
))
|
||||
|
||||
DATA_BASE = "https://data.jsdelivr.com/v1"
|
||||
CDN_BASE = "https://cdn.jsdelivr.net/npm"
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE,
|
||||
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||||
plan TEXT, price TEXT, remaining TEXT,
|
||||
found TEXT, repo TEXT)
|
||||
""")
|
||||
conn.commit()
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "jsdelivr", repo[:300]))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
"""Returns (page, pkg_idx) — position in stats pagination."""
|
||||
try:
|
||||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||||
raw = f.read().strip()
|
||||
if raw:
|
||||
parts = raw.split()
|
||||
return int(parts[0]), int(parts[1])
|
||||
except (FileNotFoundError, ValueError, IndexError):
|
||||
pass
|
||||
return 1, 0
|
||||
|
||||
|
||||
def save_cursor(page, pkg_idx):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(f"{page} {pkg_idx}")
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
s.add(line.rstrip("\n"))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(url):
|
||||
seen.add(url)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(url + "\n")
|
||||
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def req_json(url):
|
||||
"""GET JSON with retry + backoff. Returns decoded list/dict/None."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = S.get(url, headers=UA, timeout=30)
|
||||
if r.status_code == 429:
|
||||
ra = r.headers.get("Retry-After", "10")
|
||||
wait = int(ra) if ra.isdigit() else 10
|
||||
log(f" 429 {url[:80]} — sleep {wait}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
if r.status_code == 404:
|
||||
return None
|
||||
if r.status_code != 200:
|
||||
log(f" HTTP {r.status_code} {url[:80]}")
|
||||
if r.status_code >= 500:
|
||||
time.sleep([2, 4, 8][attempt])
|
||||
continue
|
||||
return None
|
||||
return r.json()
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
log(f" net error: {e} — retry {attempt+1}/3")
|
||||
time.sleep([2, 4, 8][attempt])
|
||||
return None
|
||||
|
||||
|
||||
def fetch_stats_packages(page):
|
||||
"""Fetch one page from stats/packages. Returns list of {type,name} or None."""
|
||||
url = f"{DATA_BASE}/stats/packages?limit=100&page={page}"
|
||||
data = req_json(url)
|
||||
if data is None:
|
||||
return None
|
||||
# Filter to npm type only
|
||||
return [p for p in data if p.get("type") == "npm"]
|
||||
|
||||
|
||||
def get_package_info(pkg_name):
|
||||
"""Get package info (tags, versions). Returns dict or None."""
|
||||
url = f"{DATA_BASE}/packages/npm/{pkg_name}"
|
||||
return req_json(url)
|
||||
|
||||
|
||||
def get_file_tree(pkg_name, version):
|
||||
"""Get file tree for pkg@version. Returns list of files or None."""
|
||||
url = f"{DATA_BASE}/packages/npm/{pkg_name}@{version}"
|
||||
data = req_json(url)
|
||||
if data is None:
|
||||
return None
|
||||
return data.get("files")
|
||||
|
||||
|
||||
def collect_text_files(files, prefix=""):
|
||||
"""Recursively walk file tree, yield (path, size) for small text files."""
|
||||
if not files:
|
||||
return
|
||||
for entry in files:
|
||||
name = entry.get("name", "")
|
||||
full = prefix + name
|
||||
if entry.get("type") == "directory":
|
||||
yield from collect_text_files(entry.get("files", []),
|
||||
prefix=full + "/")
|
||||
elif entry.get("type") == "file":
|
||||
_, ext = _os.path.splitext(name)
|
||||
ext = ext.lower()
|
||||
if ext in SCAN_EXT or name in DOTFILE_NAMES:
|
||||
size = entry.get("size", FILE_CAP + 1)
|
||||
if size <= FILE_CAP:
|
||||
yield full, size
|
||||
|
||||
|
||||
def fetch_and_scan(pkg_name, version, path, size):
|
||||
"""Download file from CDN, decode, scan_text, db_add. Returns key count."""
|
||||
seen_key = f"{pkg_name}@{version}/{path}"
|
||||
if seen_key in seen:
|
||||
return 0
|
||||
url = f"{CDN_BASE}/{pkg_name}@{version}/{path}"
|
||||
try:
|
||||
# cap-before-download: check Content-Length first
|
||||
head = S.head(url, headers=UA, timeout=15)
|
||||
if head.status_code != 200:
|
||||
return 0
|
||||
cl = head.headers.get("Content-Length")
|
||||
if cl and int(cl) > FILE_CAP:
|
||||
return 0
|
||||
|
||||
r = S.get(url, headers=UA, timeout=60, stream=True)
|
||||
if r.status_code != 200:
|
||||
return 0
|
||||
|
||||
# Check content length from actual response
|
||||
cl2 = r.headers.get("Content-Length")
|
||||
if cl2 and int(cl2) > FILE_CAP:
|
||||
return 0
|
||||
|
||||
# Read in chunks, accumulate into list to avoid O(n^2) buf+=chunk
|
||||
chunks = []
|
||||
total_read = 0
|
||||
for chunk in r.iter_content(65536):
|
||||
chunks.append(chunk)
|
||||
total_read += len(chunk)
|
||||
if total_read > FILE_CAP:
|
||||
return 0
|
||||
raw = b"".join(chunks)
|
||||
if len(raw) > FILE_CAP:
|
||||
return 0
|
||||
except Exception as e:
|
||||
log(f" download fail {url[:100]}: {e}")
|
||||
return 0
|
||||
|
||||
text = raw.decode("utf-8", "replace")
|
||||
hits = scan_text(text)
|
||||
if not hits:
|
||||
mark_seen(seen_key)
|
||||
return 0
|
||||
|
||||
n = 0
|
||||
repo = f"{pkg_name}@{version}/{path}"
|
||||
for prov, key in hits:
|
||||
if is_junk(key):
|
||||
continue
|
||||
db_add(key, prov, repo)
|
||||
n += 1
|
||||
if n:
|
||||
mark_seen(seen_key)
|
||||
return n
|
||||
|
||||
|
||||
def process_package(pkg_name, max_files):
|
||||
"""Process one npm package: get info, tree, download+scan text files.
|
||||
Returns (processed_files, keys_found, fail)."""
|
||||
info = get_package_info(pkg_name)
|
||||
if info is None:
|
||||
return 0, 0, 1
|
||||
|
||||
version = (info.get("tags") or {}).get("latest")
|
||||
if not version:
|
||||
# fallback: first version in versions array
|
||||
versions = info.get("versions") or []
|
||||
if versions:
|
||||
version = versions[0].get("version")
|
||||
if not version:
|
||||
return 0, 0, 1
|
||||
|
||||
files = get_file_tree(pkg_name, version)
|
||||
if files is None:
|
||||
return 0, 0, 1
|
||||
|
||||
text_files = []
|
||||
for path, size in collect_text_files(files):
|
||||
text_files.append((path, size))
|
||||
|
||||
n_files = 0
|
||||
n_keys = 0
|
||||
fails = 0
|
||||
|
||||
# Process small files first
|
||||
text_files.sort(key=lambda x: x[1])
|
||||
for path, size in text_files[:max_files]:
|
||||
k = fetch_and_scan(pkg_name, version, path, size)
|
||||
if k == 0:
|
||||
# Only count as fail if file not already seen
|
||||
sk = f"{pkg_name}@{version}/{path}"
|
||||
if sk not in seen:
|
||||
fails += 1
|
||||
else:
|
||||
n_keys += k
|
||||
n_files += 1
|
||||
|
||||
return n_files, n_keys, fails
|
||||
|
||||
|
||||
def cycle(max_items=MAX_ITEMS):
|
||||
global total, seen
|
||||
page, pkg_idx = load_cursor()
|
||||
processed = 0
|
||||
keys_found = 0
|
||||
pkg_idx_start = pkg_idx
|
||||
|
||||
log(f"cycle start — page={page} pkg_idx={pkg_idx} max_pkgs={max_items}")
|
||||
|
||||
while processed < max_items:
|
||||
pkgs = fetch_stats_packages(page)
|
||||
if pkgs is None:
|
||||
log(f" stats page {page} failed, resetting to page 1")
|
||||
page, pkg_idx = 1, 0
|
||||
continue
|
||||
if not pkgs:
|
||||
# no more pages — wrap around
|
||||
log(" no more pages, resetting")
|
||||
page, pkg_idx = 1, 0
|
||||
# Try the next page
|
||||
continue
|
||||
|
||||
while pkg_idx < len(pkgs) and processed < max_items:
|
||||
pkg = pkgs[pkg_idx]
|
||||
pkg_name = pkg.get("name", "")
|
||||
if not pkg_name:
|
||||
pkg_idx += 1
|
||||
continue
|
||||
|
||||
log(f" [{processed+1}] {pkg_name}")
|
||||
max_files = 5 if ONCE else MAX_FILES_PKG
|
||||
n_f, n_k, n_fail = process_package(pkg_name, max_files)
|
||||
log(f" files={n_f} keys={n_k} fails={n_fail}")
|
||||
processed += 1
|
||||
pkg_idx += 1
|
||||
keys_found += n_k
|
||||
|
||||
# Save cursor after each package
|
||||
save_cursor(page, pkg_idx)
|
||||
|
||||
if pkg_idx >= len(pkgs):
|
||||
page += 1
|
||||
pkg_idx = 0
|
||||
|
||||
if page > STATS_PAGE_LIMIT:
|
||||
log(f" reached page limit ({STATS_PAGE_LIMIT}), wrapping")
|
||||
page, pkg_idx = 1, 0
|
||||
save_cursor(page, pkg_idx)
|
||||
break
|
||||
|
||||
save_cursor(page, pkg_idx)
|
||||
log(f"cycle done — pkgs={processed} keys={keys_found} total={total}")
|
||||
|
||||
|
||||
def main():
|
||||
log("=== jsdelivr_eternal started ===")
|
||||
if ONCE:
|
||||
log(" --once mode: single bounded cycle")
|
||||
mx = min(MAX_ITEMS, 5)
|
||||
cycle(max_items=mx)
|
||||
log("jsdelivr_eternal --once complete")
|
||||
return
|
||||
|
||||
while True:
|
||||
cycle()
|
||||
log(f" sleep {SLEEP}s")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user