431 lines
13 KiB
Plaintext
431 lines
13 KiB
Plaintext
"""ETERNAL Docker Hub hunt: search walk + registry v2 config/layer scan.
|
|
|
|
Endpoint(s):
|
|
- https://hub.docker.com/v2/search/repositories/?query=...
|
|
- https://hub.docker.com/v2/repositories/<ns>/<img>/tags
|
|
- https://auth.docker.io/token?service=registry.docker.io&scope=...
|
|
- https://registry-1.docker.io/v2/<ns>/<img>/manifests/<tag>
|
|
- https://registry-1.docker.io/v2/<ns>/<img>/blobs/<digest>
|
|
Auth: none (anonymous pull via registry token)
|
|
"""
|
|
import os as _os
|
|
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
|
|
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
|
LOG = _os.path.join(BASE, "data", "dockerhub_eternal.log")
|
|
CURSOR_FILE = _os.path.join(BASE, "data", "dockerhub_cursor.txt")
|
|
SEEN_FILE = _os.path.join(BASE, "data", "dockerhub_seen.txt")
|
|
N_THREADS = 4
|
|
SLEEP = 120
|
|
MAX_ITEMS = 200
|
|
MAX_TAGS = 3 # tags per image (--once → 1)
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef", "00000000",
|
|
"ffffffff", "deadbeef", "testtest")
|
|
|
|
SCAN_EXT = (".env", ".txt", ".cfg", ".conf", ".yml", ".yaml",
|
|
".json", ".xml", ".ini", ".toml", ".properties",
|
|
".sh", ".bash", ".ps1", ".py")
|
|
|
|
UA = {"User-Agent": "keyfarm-dockerhub/1.0"}
|
|
S = requests.Session()
|
|
S.trust_env = False
|
|
|
|
REGISTRY_UA = {
|
|
"User-Agent": "keyfarm-dockerhub/1.0",
|
|
"Accept": "application/vnd.oci.image.index.v1+json,"
|
|
"application/vnd.docker.distribution.manifest.v2+json,"
|
|
"application/vnd.oci.image.manifest.v1+json",
|
|
}
|
|
|
|
REGISTRY_UA_BLOB = {"User-Agent": "keyfarm-dockerhub/1.0"}
|
|
|
|
# Search queries to find images likely to carry secrets in env / config
|
|
QUERIES = [
|
|
"api", "token", "bot", "config", "secret", "deploy",
|
|
"credentials", "backend", "openai", "app",
|
|
]
|
|
|
|
FILE_CAP = 500_000 # text files inside layer
|
|
LAYER_CAP = 20 * 1024 * 1024 # 20 MB for layer tarball
|
|
CONFIG_FILE_SIZES = (1_000, 50_000) # only scan .env between these sizes
|
|
DOCKER_HUB_RATE = 100 # hub.docker.com SEARCH API /hr budget
|
|
PULL_QUOTA = 90 # registry-1 anon pull budget per 6h (limit ~100)
|
|
PULL_WINDOW = 6 * 3600
|
|
PULL_STATE = _os.path.join(BASE, "data", "dockerhub_pulls.txt")
|
|
_pull_lock = threading.Lock()
|
|
|
|
|
|
def is_junk(v):
|
|
if any(g in v for g in GARBAGE):
|
|
return True
|
|
if len(set(v)) < 6:
|
|
return True
|
|
return False
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
with lock:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "dockerhub", repo[:300]))
|
|
total += 1
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
s.add(line)
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def _pull_ok():
|
|
"""Persistent 6h-window counter for registry-1 pulls (state file, shared
|
|
across processes/restarts). Anonymous Docker Hub pull quota is ~100 per
|
|
6 HOURS per IP — not 180/hr (that is the hub search API)."""
|
|
with _pull_lock:
|
|
now = int(time.time())
|
|
ws, cnt = now, 0
|
|
try:
|
|
parts = open(PULL_STATE, encoding="utf-8").read().split()
|
|
if len(parts) == 2:
|
|
ws, cnt = int(parts[0]), int(parts[1])
|
|
except Exception:
|
|
pass
|
|
if now - ws >= PULL_WINDOW:
|
|
ws, cnt = now, 0
|
|
if cnt >= PULL_QUOTA:
|
|
return False
|
|
cnt += 1
|
|
with open(PULL_STATE, "w", encoding="utf-8") as f:
|
|
f.write(str(ws) + " " + str(cnt))
|
|
return True
|
|
|
|
|
|
def _req(url, headers=None, timeout=15):
|
|
"""Single GET with retry; registry-1 pulls are quota-gated."""
|
|
if url.startswith("https://registry-1.docker.io") and not _pull_ok():
|
|
log(" pull quota exhausted (6h window) - skipping")
|
|
return None
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = S.get(url, headers=headers or UA, timeout=timeout)
|
|
if r.status_code == 429:
|
|
ra = r.headers.get("Retry-After", "60")
|
|
s = int(ra) if ra.isdigit() else 60
|
|
log(f" 429 on {url[:80]}, sleep {s}s")
|
|
time.sleep(s)
|
|
continue
|
|
return r
|
|
except requests.RequestException:
|
|
if attempt == 3:
|
|
log(f" req-err {url[:80]}")
|
|
return None
|
|
time.sleep(4)
|
|
return None
|
|
|
|
|
|
def get_registry_token(repo):
|
|
"""Obtain anonymous bearer token for registry access."""
|
|
url = ("https://auth.docker.io/token"
|
|
f"?service=registry.docker.io&scope=repository:{repo}:pull")
|
|
r = _req(url, timeout=15)
|
|
if r and r.status_code == 200:
|
|
return r.json().get("token")
|
|
return None
|
|
|
|
|
|
def get_manifest(repo, tag, token):
|
|
"""Fetch manifest JSON (resolves OCI image index to amd64/linux)."""
|
|
url = f"https://registry-1.docker.io/v2/{repo}/manifests/{tag}"
|
|
r = _req(url, headers={**REGISTRY_UA, "Authorization": f"Bearer {token}"},
|
|
timeout=20)
|
|
if not r or r.status_code != 200:
|
|
return None
|
|
man = r.json()
|
|
mt = man.get("mediaType", "")
|
|
if "index" in mt or "list" in mt:
|
|
for m in man.get("manifests", []):
|
|
plat = m.get("platform", {})
|
|
if plat.get("architecture") == "amd64" and plat.get("os") == "linux":
|
|
return get_manifest(repo, m["digest"], token)
|
|
# fallback: first manifest entry
|
|
if man.get("manifests"):
|
|
return get_manifest(repo, man["manifests"][0]["digest"], token)
|
|
return None
|
|
return man
|
|
|
|
|
|
def get_config_blob(repo, digest, token):
|
|
"""Fetch image config blob by digest."""
|
|
url = f"https://registry-1.docker.io/v2/{repo}/blobs/{digest}"
|
|
r = _req(url, headers={**REGISTRY_UA_BLOB, "Authorization": f"Bearer {token}"},
|
|
timeout=20)
|
|
if r and r.status_code == 200:
|
|
return r.json()
|
|
return None
|
|
|
|
|
|
def get_layer_blob(repo, digest, token):
|
|
"""Fetch layer blob (gzipped tarball)."""
|
|
url = f"https://registry-1.docker.io/v2/{repo}/blobs/{digest}"
|
|
r = _req(url, headers={**REGISTRY_UA_BLOB, "Authorization": f"Bearer {token}"},
|
|
timeout=60)
|
|
if r and r.status_code == 200 and len(r.content) <= LAYER_CAP:
|
|
return r.content
|
|
return None
|
|
|
|
|
|
def scan_config_env(cfg, repo_tag):
|
|
"""Scan config's Env entries for keys."""
|
|
n = 0
|
|
env_list = (cfg.get("config") or {}).get("Env") or []
|
|
for entry in env_list:
|
|
if "=" not in entry:
|
|
continue
|
|
val = entry.split("=", 1)[1]
|
|
if is_junk(val) or len(val) < 30:
|
|
continue
|
|
for prov, key in scan_text(val):
|
|
if is_junk(key) or len(key) < 30:
|
|
continue
|
|
db_add(key, prov, repo_tag)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def scan_env_whole_text(cfg, repo_tag):
|
|
"""Join entire Env block and scan as single text for ENV_RE patterns."""
|
|
env_list = (cfg.get("config") or {}).get("Env") or []
|
|
if not env_list:
|
|
return 0
|
|
text = "\n".join(env_list)
|
|
n = 0
|
|
for prov, key in scan_text(text):
|
|
if is_junk(key) or len(key) < 30:
|
|
continue
|
|
db_add(key, prov, repo_tag)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def scan_layer(layer_data, repo_tag):
|
|
"""Extract .env/config files from layer tarball and scan."""
|
|
n = 0
|
|
try:
|
|
with gzip.GzipFile(fileobj=io.BytesIO(layer_data)) as gf:
|
|
with tarfile.TarFile(fileobj=gf, mode="r") as tf:
|
|
for member in tf.getmembers():
|
|
if member.isdir() or not member.isfile():
|
|
continue
|
|
fn = member.name.lower()
|
|
if not any(fn.endswith(e) for e in SCAN_EXT):
|
|
continue
|
|
if member.size > FILE_CAP or member.size == 0:
|
|
continue
|
|
try:
|
|
data = tf.extractfile(member)
|
|
if data is None:
|
|
continue
|
|
text = data.read().decode("utf-8", errors="replace")
|
|
except Exception:
|
|
continue
|
|
for prov, key in scan_text(text):
|
|
if is_junk(key) or len(key) < 30:
|
|
continue
|
|
db_add(key, prov, repo_tag)
|
|
n += 1
|
|
except Exception:
|
|
pass
|
|
return n
|
|
|
|
|
|
def search_images(query, page=1):
|
|
"""Search Docker Hub for images matching query."""
|
|
url = ("https://hub.docker.com/v2/search/repositories/"
|
|
f"?query={query}&page_size=30&page={page}")
|
|
r = _req(url, headers=UA, timeout=20)
|
|
if r and r.status_code == 200:
|
|
return r.json().get("results", [])
|
|
return []
|
|
|
|
|
|
def get_tags(repo):
|
|
"""Get tags (+ digests) for a repo."""
|
|
url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=25"
|
|
r = _req(url, headers=UA, timeout=15)
|
|
if r and r.status_code == 200:
|
|
return r.json().get("results", [])
|
|
return []
|
|
|
|
|
|
def process_image(repo):
|
|
"""Full scan flow for one image: tags → manifest → config env → layer."""
|
|
tags = get_tags(repo)
|
|
if not tags:
|
|
return 0
|
|
|
|
n = 0
|
|
ttl = 0
|
|
for tag_obj in tags:
|
|
if ttl >= MAX_TAGS:
|
|
break
|
|
tag = tag_obj.get("name")
|
|
if not tag:
|
|
continue
|
|
repo_tag = f"docker:{repo}:{tag}"
|
|
if repo_tag in seen:
|
|
continue
|
|
mark_seen(repo_tag)
|
|
|
|
# Skip images without a digest (can't pull from registry)
|
|
digest = tag_obj.get("digest")
|
|
if not digest:
|
|
continue
|
|
|
|
ttl += 1
|
|
|
|
token = get_registry_token(repo)
|
|
if not token:
|
|
continue
|
|
|
|
man = get_manifest(repo, tag, token)
|
|
if not man:
|
|
continue
|
|
|
|
cfg_digest = (man.get("config") or {}).get("digest")
|
|
if not cfg_digest:
|
|
continue
|
|
|
|
cfg = get_config_blob(repo, cfg_digest, token)
|
|
if not cfg:
|
|
continue
|
|
|
|
n += scan_env_whole_text(cfg, repo_tag)
|
|
|
|
# Try first layer if small enough
|
|
layers = man.get("layers") or []
|
|
if layers:
|
|
first = layers[0]
|
|
layer_digest = first.get("digest", "")
|
|
layer_size = first.get("size", 0)
|
|
if layer_digest and layer_size <= LAYER_CAP and layer_size > 0:
|
|
layer_data = get_layer_blob(repo, layer_digest, token)
|
|
if layer_data and len(layer_data) <= LAYER_CAP:
|
|
n += scan_layer(layer_data, repo_tag)
|
|
return n
|
|
|
|
|
|
def cycle():
|
|
global total, seen
|
|
total = 0
|
|
fail = 0
|
|
qidx = 0
|
|
|
|
# Round-robin through queries across cycles using cursor
|
|
cursor_path = CURSOR_FILE
|
|
try:
|
|
with open(cursor_path, encoding="utf-8") as f:
|
|
raw = f.read().strip()
|
|
if raw:
|
|
qidx, page = (int(x) for x in raw.split(":", 1))
|
|
else:
|
|
qidx, page = 0, 1
|
|
except (FileNotFoundError, ValueError):
|
|
qidx, page = 0, 1
|
|
|
|
query = QUERIES[qidx % len(QUERIES)]
|
|
log(f"cycle query={query} page={page}")
|
|
|
|
images = search_images(query, page)
|
|
if not images:
|
|
# No results or page exhausted → next query, reset page
|
|
qidx = (qidx + 1) % len(QUERIES)
|
|
page = 1
|
|
images = search_images(QUERIES[qidx], page)
|
|
|
|
for img in images[:MAX_ITEMS]:
|
|
repo = img.get("repo_name")
|
|
if not repo:
|
|
continue
|
|
try:
|
|
n = process_image(repo)
|
|
if n > 0:
|
|
log(f" {repo} +{n} keys")
|
|
except Exception:
|
|
fail += 1
|
|
|
|
with open(cursor_path, "w", encoding="utf-8") as f:
|
|
next_page = page + 1
|
|
f.write(f"{qidx}:{next_page}")
|
|
|
|
log(f"cycle done query={QUERIES[qidx]} page={page} "
|
|
f"images={len(images)} fail={fail} +{total} keys")
|
|
|
|
|
|
def main():
|
|
log("=== dockerhub_eternal START ===")
|
|
if "--once" in sys.argv:
|
|
global MAX_ITEMS, MAX_TAGS
|
|
MAX_ITEMS = min(MAX_ITEMS, 5)
|
|
MAX_TAGS = 1
|
|
while True:
|
|
cycle()
|
|
if "--once" in sys.argv:
|
|
break
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |