387 lines
12 KiB
Python
387 lines
12 KiB
Python
"""ETERNAL jsDelivr CDN: walk built npm package files for hardcoded secrets.
|
||
|
||
Endpoint(s):
|
||
data.jsdelivr.com/v1/stats/packages (paginated, max 100 pages×100)
|
||
data.jsdelivr.com/v1/packages/npm/<pkg> (tags + versions)
|
||
data.jsdelivr.com/v1/packages/npm/<pkg>@<ver> (file tree)
|
||
cdn.jsdelivr.net/npm/<pkg>@<ver>/<path> (file download)
|
||
|
||
Auth: none (per-IP anonymous; no rate-limit headers observed, but Cloudflare-backed)
|
||
Collision: data.jsdelivr.com AND cdn.jsdelivr.net — hosts NOT used by any existing lane.
|
||
Package _names_ overlap with npm_eternal (both walk npm ecosystem), but the HOSTS are
|
||
different (npm=registry.npmjs.org, jsd=data.jsdelivr.com/cdn.jsdelivr.net) so per-IP
|
||
quota is independent. The files differ too: npm_eternal downloads raw source tarballs;
|
||
jsd fetches individual CDN-served built bundles — where accidental hardcoded keys live.
|
||
"""
|
||
import os as _os
|
||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||
|
||
import hashlib, json, re, sqlite3, sys, threading, time
|
||
import requests
|
||
|
||
S = requests.Session()
|
||
S.trust_env = False # bypass local dashscope proxy
|
||
|
||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||
LOG = _os.path.join(BASE, "data", "jsdelivr_eternal.log")
|
||
CURSOR_FILE = _os.path.join(BASE, "data", "jsdelivr_cursor.txt")
|
||
SEEN_FILE = _os.path.join(BASE, "data", "jsdelivr_seen.txt")
|
||
N_THREADS = 4
|
||
SLEEP = 90
|
||
MAX_ITEMS = 200 # packages per cycle; --once uses min(200,5)
|
||
MAX_FILES_PKG = 20 # files to download per package (--once: 5)
|
||
FILE_CAP = 500_000 # skip files larger than this
|
||
STATS_PAGE_LIMIT = 100 # max pages for stats pagination
|
||
ONCE = "--once" in sys.argv # used to cap files per package in --once mode
|
||
|
||
from key_patterns import scan_text
|
||
|
||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||
|
||
UA = {"User-Agent": "keyfarm-jsdelivr/1.0"}
|
||
|
||
# Small text file extensions likely to contain hardcoded keys in built bundles
|
||
# Dotfiles like .env, .npmrc have no extension after leading dot.
|
||
# os.path.splitext('.env') -> ('.env', ''). We match by full basename
|
||
# after splitting on the LAST dot.
|
||
DOTFILE_NAMES = frozenset((
|
||
".env", ".envrc", ".npmrc", ".gitignore",
|
||
".gitattributes", ".eslintrc", ".prettierrc",
|
||
))
|
||
|
||
SCAN_EXT = frozenset((
|
||
".js", ".json", ".ts", ".mjs", ".cjs",
|
||
".yml", ".yaml", ".txt", ".toml",
|
||
".rc", ".config", ".conf", ".ini", ".cfg",
|
||
))
|
||
|
||
DATA_BASE = "https://data.jsdelivr.com/v1"
|
||
CDN_BASE = "https://cdn.jsdelivr.net/npm"
|
||
|
||
|
||
def log(msg):
|
||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||
with open(LOG, "a", encoding="utf-8") as f:
|
||
f.write(line)
|
||
|
||
|
||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||
isolation_level=None)
|
||
conn.execute("""
|
||
CREATE TABLE IF NOT EXISTS keys (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
hash TEXT UNIQUE,
|
||
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||
plan TEXT, price TEXT, remaining TEXT,
|
||
found TEXT, repo TEXT)
|
||
""")
|
||
conn.commit()
|
||
lock = threading.Lock()
|
||
total = 0
|
||
|
||
|
||
def db_add(key, prov, repo):
|
||
global total
|
||
with lock:
|
||
conn.execute(
|
||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||
"VALUES (?,?,?,?,?,?) "
|
||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||
"found=excluded.found, repo=excluded.repo "
|
||
"WHERE keys.status IN ('DEAD','ERR')",
|
||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||
key, prov, "NEW", "jsdelivr", repo[:300]))
|
||
total += 1
|
||
|
||
|
||
def load_cursor():
|
||
"""Returns (page, pkg_idx) — position in stats pagination."""
|
||
try:
|
||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||
raw = f.read().strip()
|
||
if raw:
|
||
parts = raw.split()
|
||
return int(parts[0]), int(parts[1])
|
||
except (FileNotFoundError, ValueError, IndexError):
|
||
pass
|
||
return 1, 0
|
||
|
||
|
||
def save_cursor(page, pkg_idx):
|
||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||
f.write(f"{page} {pkg_idx}")
|
||
|
||
|
||
def load_seen():
|
||
s = set()
|
||
try:
|
||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||
for line in f:
|
||
s.add(line.rstrip("\n"))
|
||
except FileNotFoundError:
|
||
pass
|
||
return s
|
||
|
||
|
||
seen = load_seen()
|
||
|
||
|
||
def mark_seen(url):
|
||
seen.add(url)
|
||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||
f.write(url + "\n")
|
||
|
||
|
||
def is_junk(v):
|
||
if any(g in v for g in GARBAGE):
|
||
return True
|
||
if len(set(v)) < 6:
|
||
return True
|
||
return False
|
||
|
||
|
||
def req_json(url):
|
||
"""GET JSON with retry + backoff. Returns decoded list/dict/None."""
|
||
for attempt in range(3):
|
||
try:
|
||
r = S.get(url, headers=UA, timeout=30)
|
||
if r.status_code == 429:
|
||
ra = r.headers.get("Retry-After", "10")
|
||
wait = int(ra) if ra.isdigit() else 10
|
||
log(f" 429 {url[:80]} — sleep {wait}s")
|
||
time.sleep(wait)
|
||
continue
|
||
if r.status_code == 404:
|
||
return None
|
||
if r.status_code != 200:
|
||
log(f" HTTP {r.status_code} {url[:80]}")
|
||
if r.status_code >= 500:
|
||
time.sleep([2, 4, 8][attempt])
|
||
continue
|
||
return None
|
||
return r.json()
|
||
except (requests.ConnectionError, requests.Timeout) as e:
|
||
log(f" net error: {e} — retry {attempt+1}/3")
|
||
time.sleep([2, 4, 8][attempt])
|
||
return None
|
||
|
||
|
||
def fetch_stats_packages(page):
|
||
"""Fetch one page from stats/packages. Returns list of {type,name} or None."""
|
||
url = f"{DATA_BASE}/stats/packages?limit=100&page={page}"
|
||
data = req_json(url)
|
||
if data is None:
|
||
return None
|
||
# Filter to npm type only
|
||
return [p for p in data if p.get("type") == "npm"]
|
||
|
||
|
||
def get_package_info(pkg_name):
|
||
"""Get package info (tags, versions). Returns dict or None."""
|
||
url = f"{DATA_BASE}/packages/npm/{pkg_name}"
|
||
return req_json(url)
|
||
|
||
|
||
def get_file_tree(pkg_name, version):
|
||
"""Get file tree for pkg@version. Returns list of files or None."""
|
||
url = f"{DATA_BASE}/packages/npm/{pkg_name}@{version}"
|
||
data = req_json(url)
|
||
if data is None:
|
||
return None
|
||
return data.get("files")
|
||
|
||
|
||
def collect_text_files(files, prefix=""):
|
||
"""Recursively walk file tree, yield (path, size) for small text files."""
|
||
if not files:
|
||
return
|
||
for entry in files:
|
||
name = entry.get("name", "")
|
||
full = prefix + name
|
||
if entry.get("type") == "directory":
|
||
yield from collect_text_files(entry.get("files", []),
|
||
prefix=full + "/")
|
||
elif entry.get("type") == "file":
|
||
_, ext = _os.path.splitext(name)
|
||
ext = ext.lower()
|
||
if ext in SCAN_EXT or name in DOTFILE_NAMES:
|
||
size = entry.get("size", FILE_CAP + 1)
|
||
if size <= FILE_CAP:
|
||
yield full, size
|
||
|
||
|
||
def fetch_and_scan(pkg_name, version, path, size):
|
||
"""Download file from CDN, decode, scan_text, db_add. Returns key count."""
|
||
seen_key = f"{pkg_name}@{version}/{path}"
|
||
if seen_key in seen:
|
||
return 0
|
||
url = f"{CDN_BASE}/{pkg_name}@{version}/{path}"
|
||
try:
|
||
# cap-before-download: check Content-Length first
|
||
head = S.head(url, headers=UA, timeout=15)
|
||
if head.status_code != 200:
|
||
return 0
|
||
cl = head.headers.get("Content-Length")
|
||
if cl and int(cl) > FILE_CAP:
|
||
return 0
|
||
|
||
r = S.get(url, headers=UA, timeout=60, stream=True)
|
||
if r.status_code != 200:
|
||
return 0
|
||
|
||
# Check content length from actual response
|
||
cl2 = r.headers.get("Content-Length")
|
||
if cl2 and int(cl2) > FILE_CAP:
|
||
return 0
|
||
|
||
# Read in chunks, accumulate into list to avoid O(n^2) buf+=chunk
|
||
chunks = []
|
||
total_read = 0
|
||
for chunk in r.iter_content(65536):
|
||
chunks.append(chunk)
|
||
total_read += len(chunk)
|
||
if total_read > FILE_CAP:
|
||
return 0
|
||
raw = b"".join(chunks)
|
||
if len(raw) > FILE_CAP:
|
||
return 0
|
||
except Exception as e:
|
||
log(f" download fail {url[:100]}: {e}")
|
||
return 0
|
||
|
||
text = raw.decode("utf-8", "replace")
|
||
hits = scan_text(text)
|
||
if not hits:
|
||
mark_seen(seen_key)
|
||
return 0
|
||
|
||
n = 0
|
||
repo = f"{pkg_name}@{version}/{path}"
|
||
for prov, key in hits:
|
||
if is_junk(key):
|
||
continue
|
||
db_add(key, prov, repo)
|
||
n += 1
|
||
if n:
|
||
mark_seen(seen_key)
|
||
return n
|
||
|
||
|
||
def process_package(pkg_name, max_files):
|
||
"""Process one npm package: get info, tree, download+scan text files.
|
||
Returns (processed_files, keys_found, fail)."""
|
||
info = get_package_info(pkg_name)
|
||
if info is None:
|
||
return 0, 0, 1
|
||
|
||
version = (info.get("tags") or {}).get("latest")
|
||
if not version:
|
||
# fallback: first version in versions array
|
||
versions = info.get("versions") or []
|
||
if versions:
|
||
version = versions[0].get("version")
|
||
if not version:
|
||
return 0, 0, 1
|
||
|
||
files = get_file_tree(pkg_name, version)
|
||
if files is None:
|
||
return 0, 0, 1
|
||
|
||
text_files = []
|
||
for path, size in collect_text_files(files):
|
||
text_files.append((path, size))
|
||
|
||
n_files = 0
|
||
n_keys = 0
|
||
fails = 0
|
||
|
||
# Process small files first
|
||
text_files.sort(key=lambda x: x[1])
|
||
for path, size in text_files[:max_files]:
|
||
k = fetch_and_scan(pkg_name, version, path, size)
|
||
if k == 0:
|
||
# Only count as fail if file not already seen
|
||
sk = f"{pkg_name}@{version}/{path}"
|
||
if sk not in seen:
|
||
fails += 1
|
||
else:
|
||
n_keys += k
|
||
n_files += 1
|
||
|
||
return n_files, n_keys, fails
|
||
|
||
|
||
def cycle(max_items=MAX_ITEMS):
|
||
global total, seen
|
||
page, pkg_idx = load_cursor()
|
||
processed = 0
|
||
keys_found = 0
|
||
pkg_idx_start = pkg_idx
|
||
|
||
log(f"cycle start — page={page} pkg_idx={pkg_idx} max_pkgs={max_items}")
|
||
|
||
while processed < max_items:
|
||
pkgs = fetch_stats_packages(page)
|
||
if pkgs is None:
|
||
log(f" stats page {page} failed, resetting to page 1")
|
||
page, pkg_idx = 1, 0
|
||
continue
|
||
if not pkgs:
|
||
# no more pages — wrap around
|
||
log(" no more pages, resetting")
|
||
page, pkg_idx = 1, 0
|
||
# Try the next page
|
||
continue
|
||
|
||
while pkg_idx < len(pkgs) and processed < max_items:
|
||
pkg = pkgs[pkg_idx]
|
||
pkg_name = pkg.get("name", "")
|
||
if not pkg_name:
|
||
pkg_idx += 1
|
||
continue
|
||
|
||
log(f" [{processed+1}] {pkg_name}")
|
||
max_files = 5 if ONCE else MAX_FILES_PKG
|
||
n_f, n_k, n_fail = process_package(pkg_name, max_files)
|
||
log(f" files={n_f} keys={n_k} fails={n_fail}")
|
||
processed += 1
|
||
pkg_idx += 1
|
||
keys_found += n_k
|
||
|
||
# Save cursor after each package
|
||
save_cursor(page, pkg_idx)
|
||
|
||
if pkg_idx >= len(pkgs):
|
||
page += 1
|
||
pkg_idx = 0
|
||
|
||
if page > STATS_PAGE_LIMIT:
|
||
log(f" reached page limit ({STATS_PAGE_LIMIT}), wrapping")
|
||
page, pkg_idx = 1, 0
|
||
save_cursor(page, pkg_idx)
|
||
break
|
||
|
||
save_cursor(page, pkg_idx)
|
||
log(f"cycle done — pkgs={processed} keys={keys_found} total={total}")
|
||
|
||
|
||
def main():
|
||
log("=== jsdelivr_eternal started ===")
|
||
if ONCE:
|
||
log(" --once mode: single bounded cycle")
|
||
mx = min(MAX_ITEMS, 5)
|
||
cycle(max_items=mx)
|
||
log("jsdelivr_eternal --once complete")
|
||
return
|
||
|
||
while True:
|
||
cycle()
|
||
log(f" sleep {SLEEP}s")
|
||
time.sleep(SLEEP)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |