"""ETERNAL Docker Hub hunt: search walk + registry v2 config/layer scan.

Endpoint(s):
- https://hub.docker.com/v2/search/repositories/?query=...
- https://hub.docker.com/v2/repositories/<ns>/<img>/tags
- https://auth.docker.io/token?service=registry.docker.io&scope=...
- https://registry-1.docker.io/v2/<ns>/<img>/manifests/<tag>
- https://registry-1.docker.io/v2/<ns>/<img>/blobs/<digest>
Auth: none (anonymous pull via registry token)
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))


import gzip
import hashlib
import io
import json
import re
import sqlite3
import sys
import tarfile
import threading
import time

import requests

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = _os.path.join(BASE, "data", "dockerhub_eternal.log")
CURSOR_FILE = _os.path.join(BASE, "data", "dockerhub_cursor.txt")
SEEN_FILE = _os.path.join(BASE, "data", "dockerhub_seen.txt")
N_THREADS = 4
SLEEP = 120
MAX_ITEMS = 200
MAX_TAGS = 3              # tags per image (--once → 1)

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef", "00000000",
           "ffffffff", "deadbeef", "testtest")

SCAN_EXT = (".env", ".txt", ".cfg", ".conf", ".yml", ".yaml",
            ".json", ".xml", ".ini", ".toml", ".properties",
            ".sh", ".bash", ".ps1", ".py")

UA = {"User-Agent": "keyfarm-dockerhub/1.0"}
S = requests.Session()
S.trust_env = False

REGISTRY_UA = {
    "User-Agent": "keyfarm-dockerhub/1.0",
    "Accept": "application/vnd.oci.image.index.v1+json,"
              "application/vnd.docker.distribution.manifest.v2+json,"
              "application/vnd.oci.image.manifest.v1+json",
}

REGISTRY_UA_BLOB = {"User-Agent": "keyfarm-dockerhub/1.0"}

# Search queries to find images likely to carry secrets in env / config
QUERIES = [
    "api", "token", "bot", "config", "secret", "deploy",
    "credentials", "backend", "openai", "app",
]

FILE_CAP = 500_000      # text files inside layer
LAYER_CAP = 20 * 1024 * 1024  # 20 MB for layer tarball
CONFIG_FILE_SIZES = (1_000, 50_000)  # only scan .env between these sizes
DOCKER_HUB_RATE = 100   # hub.docker.com SEARCH API /hr budget
PULL_QUOTA = 90         # registry-1 anon pull budget per 6h (limit ~100)
PULL_WINDOW = 6 * 3600
PULL_STATE = _os.path.join(BASE, "data", "dockerhub_pulls.txt")
_pull_lock = threading.Lock()


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "dockerhub", repo[:300]))
        total += 1


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def _pull_ok():
    """Persistent 6h-window counter for registry-1 pulls (state file, shared
    across processes/restarts). Anonymous Docker Hub pull quota is ~100 per
    6 HOURS per IP — not 180/hr (that is the hub search API)."""
    with _pull_lock:
        now = int(time.time())
        ws, cnt = now, 0
        try:
            parts = open(PULL_STATE, encoding="utf-8").read().split()
            if len(parts) == 2:
                ws, cnt = int(parts[0]), int(parts[1])
        except Exception:
            pass
        if now - ws >= PULL_WINDOW:
            ws, cnt = now, 0
        if cnt >= PULL_QUOTA:
            return False
        cnt += 1
        with open(PULL_STATE, "w", encoding="utf-8") as f:
            f.write(str(ws) + " " + str(cnt))
        return True


def _req(url, headers=None, timeout=15):
    """Single GET with retry; registry-1 pulls are quota-gated."""
    if url.startswith("https://registry-1.docker.io") and not _pull_ok():
        log("  pull quota exhausted (6h window) - skipping")
        return None
    for attempt in (1, 2, 3):
        try:
            r = S.get(url, headers=headers or UA, timeout=timeout)
            if r.status_code == 429:
                ra = r.headers.get("Retry-After", "60")
                s = int(ra) if ra.isdigit() else 60
                log(f"  429 on {url[:80]}, sleep {s}s")
                time.sleep(s)
                continue
            return r
        except requests.RequestException:
            if attempt == 3:
                log(f"  req-err {url[:80]}")
                return None
            time.sleep(4)
    return None


def get_registry_token(repo):
    """Obtain anonymous bearer token for registry access."""
    url = ("https://auth.docker.io/token"
           f"?service=registry.docker.io&scope=repository:{repo}:pull")
    r = _req(url, timeout=15)
    if r and r.status_code == 200:
        return r.json().get("token")
    return None


def get_manifest(repo, tag, token):
    """Fetch manifest JSON (resolves OCI image index to amd64/linux)."""
    url = f"https://registry-1.docker.io/v2/{repo}/manifests/{tag}"
    r = _req(url, headers={**REGISTRY_UA, "Authorization": f"Bearer {token}"},
             timeout=20)
    if not r or r.status_code != 200:
        return None
    man = r.json()
    mt = man.get("mediaType", "")
    if "index" in mt or "list" in mt:
        for m in man.get("manifests", []):
            plat = m.get("platform", {})
            if plat.get("architecture") == "amd64" and plat.get("os") == "linux":
                return get_manifest(repo, m["digest"], token)
        # fallback: first manifest entry
        if man.get("manifests"):
            return get_manifest(repo, man["manifests"][0]["digest"], token)
        return None
    return man


def get_config_blob(repo, digest, token):
    """Fetch image config blob by digest."""
    url = f"https://registry-1.docker.io/v2/{repo}/blobs/{digest}"
    r = _req(url, headers={**REGISTRY_UA_BLOB, "Authorization": f"Bearer {token}"},
             timeout=20)
    if r and r.status_code == 200:
        return r.json()
    return None


def get_layer_blob(repo, digest, token):
    """Fetch layer blob (gzipped tarball)."""
    url = f"https://registry-1.docker.io/v2/{repo}/blobs/{digest}"
    r = _req(url, headers={**REGISTRY_UA_BLOB, "Authorization": f"Bearer {token}"},
             timeout=60)
    if r and r.status_code == 200 and len(r.content) <= LAYER_CAP:
        return r.content
    return None


def scan_config_env(cfg, repo_tag):
    """Scan config's Env entries for keys."""
    n = 0
    env_list = (cfg.get("config") or {}).get("Env") or []
    for entry in env_list:
        if "=" not in entry:
            continue
        val = entry.split("=", 1)[1]
        if is_junk(val) or len(val) < 30:
            continue
        for prov, key in scan_text(val):
            if is_junk(key) or len(key) < 30:
                continue
            db_add(key, prov, repo_tag)
            n += 1
    return n


def scan_env_whole_text(cfg, repo_tag):
    """Join entire Env block and scan as single text for ENV_RE patterns."""
    env_list = (cfg.get("config") or {}).get("Env") or []
    if not env_list:
        return 0
    text = "\n".join(env_list)
    n = 0
    for prov, key in scan_text(text):
        if is_junk(key) or len(key) < 30:
            continue
        db_add(key, prov, repo_tag)
        n += 1
    return n


def scan_layer(layer_data, repo_tag):
    """Extract .env/config files from layer tarball and scan."""
    n = 0
    try:
        with gzip.GzipFile(fileobj=io.BytesIO(layer_data)) as gf:
            with tarfile.TarFile(fileobj=gf, mode="r") as tf:
                for member in tf.getmembers():
                    if member.isdir() or not member.isfile():
                        continue
                    fn = member.name.lower()
                    if not any(fn.endswith(e) for e in SCAN_EXT):
                        continue
                    if member.size > FILE_CAP or member.size == 0:
                        continue
                    try:
                        data = tf.extractfile(member)
                        if data is None:
                            continue
                        text = data.read().decode("utf-8", errors="replace")
                    except Exception:
                        continue
                    for prov, key in scan_text(text):
                        if is_junk(key) or len(key) < 30:
                            continue
                        db_add(key, prov, repo_tag)
                        n += 1
    except Exception:
        pass
    return n


def search_images(query, page=1):
    """Search Docker Hub for images matching query."""
    url = ("https://hub.docker.com/v2/search/repositories/"
           f"?query={query}&page_size=30&page={page}")
    r = _req(url, headers=UA, timeout=20)
    if r and r.status_code == 200:
        return r.json().get("results", [])
    return []


def get_tags(repo):
    """Get tags (+ digests) for a repo."""
    url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=25"
    r = _req(url, headers=UA, timeout=15)
    if r and r.status_code == 200:
        return r.json().get("results", [])
    return []


def process_image(repo):
    """Full scan flow for one image: tags → manifest → config env → layer."""
    tags = get_tags(repo)
    if not tags:
        return 0

    n = 0
    ttl = 0
    for tag_obj in tags:
        if ttl >= MAX_TAGS:
            break
        tag = tag_obj.get("name")
        if not tag:
            continue
        repo_tag = f"docker:{repo}:{tag}"
        if repo_tag in seen:
            continue
        mark_seen(repo_tag)

        # Skip images without a digest (can't pull from registry)
        digest = tag_obj.get("digest")
        if not digest:
            continue

        ttl += 1

        token = get_registry_token(repo)
        if not token:
            continue

        man = get_manifest(repo, tag, token)
        if not man:
            continue

        cfg_digest = (man.get("config") or {}).get("digest")
        if not cfg_digest:
            continue

        cfg = get_config_blob(repo, cfg_digest, token)
        if not cfg:
            continue

        n += scan_env_whole_text(cfg, repo_tag)

        # Try first layer if small enough
        layers = man.get("layers") or []
        if layers:
            first = layers[0]
            layer_digest = first.get("digest", "")
            layer_size = first.get("size", 0)
            if layer_digest and layer_size <= LAYER_CAP and layer_size > 0:
                layer_data = get_layer_blob(repo, layer_digest, token)
                if layer_data and len(layer_data) <= LAYER_CAP:
                    n += scan_layer(layer_data, repo_tag)
    return n


def cycle():
    global total, seen
    total = 0
    fail = 0
    qidx = 0

    # Round-robin through queries across cycles using cursor
    cursor_path = CURSOR_FILE
    try:
        with open(cursor_path, encoding="utf-8") as f:
            raw = f.read().strip()
            if raw:
                qidx, page = (int(x) for x in raw.split(":", 1))
            else:
                qidx, page = 0, 1
    except (FileNotFoundError, ValueError):
        qidx, page = 0, 1

    query = QUERIES[qidx % len(QUERIES)]
    log(f"cycle query={query} page={page}")

    images = search_images(query, page)
    if not images:
        # No results or page exhausted → next query, reset page
        qidx = (qidx + 1) % len(QUERIES)
        page = 1
        images = search_images(QUERIES[qidx], page)

    for img in images[:MAX_ITEMS]:
        repo = img.get("repo_name")
        if not repo:
            continue
        try:
            n = process_image(repo)
            if n > 0:
                log(f"  {repo} +{n} keys")
        except Exception:
            fail += 1

    with open(cursor_path, "w", encoding="utf-8") as f:
        next_page = page + 1
        f.write(f"{qidx}:{next_page}")

    log(f"cycle done query={QUERIES[qidx]} page={page} "
        f"images={len(images)} fail={fail} +{total} keys")


def main():
    log("=== dockerhub_eternal START ===")
    if "--once" in sys.argv:
        global MAX_ITEMS, MAX_TAGS
        MAX_ITEMS = min(MAX_ITEMS, 5)
        MAX_TAGS = 1
    while True:
        cycle()
        if "--once" in sys.argv:
            break
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()