"""ETERNAL Wayback Machine: CDX prefix crawl over GitHub orgs with leak history.

Walks Internet Archive CDX with matchType=prefix over raw.githubusercontent.com/<ORG>/
for orgs mined from keys.db (top by historical key count, cached in
data/wayback_orgs.txt). Server-side filter original:.*/\.env + statuscode:200
(broad regex filters → CDX 503, proven live 2026-10-19; matchType=domain —
dead, timeouts). Fetches raw archived content, scans for keys.
pastebin removed: CDX does not index pastebin content. gist: TODO if needed.

Endpoint(s): http://web.archive.org/cdx/search/cdx (CDX index API)
             https://web.archive.org/web/<ts>id_/<url> (raw archived content)
Auth: none
Flags: --once (single small cycle), --remine (rebuild data/wayback_orgs.txt)
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)

import hashlib, re, sqlite3, sys, threading, time
import requests

S = requests.Session()
S.trust_env = False  # contract 4: bypass local dashscope proxy

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = _os.path.join(BASE, "data", "wayback_eternal.log")
CURSOR_FILE = _os.path.join(BASE, "data", "wayback_cursor.txt")
SEEN_FILE = _os.path.join(BASE, "data", "wayback_seen.txt")
N_THREADS = 1
SLEEP = 90
MAX_ITEMS = 200
CDX_LIMIT = 5000
FILE_CAP = 500_000

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

UA = {"User-Agent": "keyfarm-wayback/1.0"}

ORGS_FILE = _os.path.join(BASE, "data", "wayback_orgs.txt")
# Server-side CDX filters: narrow single-pattern each (wide alternation → 503,
# proven 2026-10-19). Rotated per org cycle; all pass the is_config_url gate.
CDX_FILTERS = (
    "original:.*/\\.env",
    "original:.*/\\.npmrc",
    "original:.*/\\.netrc",
    "original:.*/\\.git-credentials",
    "original:.*credentials\\.json",
    "original:.*/config\\.json",
)
# Fallback if keys.db mining fails
SEED_ORGS = ["dpangestuw", "moleway", "sse-casb-lab"]

# Config-like filenames that may harbour secrets / tokens
CONFIG_NAMES = frozenset([
    ".env", ".env.example", ".env.production", ".env.local",
    ".env.dev", ".env.staging", ".envrc", ".env.template", ".env.sample",
    "credentials", "credentials.json", "credentials.yml", "credentials.yaml",
    "config.js", "config.json",
    "application.properties",
    ".npmrc",
    "id_rsa", "id_rsa.pub", "id_ed25519", "id_ecdsa",
    "secret", "secrets", "secrets.json", "secrets.yml", "secrets.yaml",
    ".netrc", ".git-credentials", ".dockercfg",
    "composer.json",
])

# Match a URL path's final segment against CONFIG_NAMES and also catch
# paths like /foo/.env or /foo/credentials.json via substring check.
_CONFIG_RE = re.compile(
    r'/(?:'
    r'\.env(?:\.\w+)?|'
    r'credentials(?:\.\w+)?|'
    r'config\.(?:js|json)|'
    r'application\.properties|'
    r'\.npmrc|'
    r'id_(?:rsa|ed25519|ecdsa)(?:\.pub)?|'
    r'secret(?:s)?(?:\.\w+)?|'
    r'\.netrc|'
    r'\.git-credentials|'
    r'\.dockercfg|'
    r'composer\.json'
    r')(?:\?.*)?$',
    re.IGNORECASE
)


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
conn.execute("""
    CREATE TABLE IF NOT EXISTS keys (
      id INTEGER PRIMARY KEY AUTOINCREMENT,
      hash TEXT UNIQUE,
      val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
      plan TEXT, price TEXT, remaining TEXT,
      found TEXT, repo TEXT)
""")
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "wayback", repo[:300]))
        total += 1


def load_cursor():
    """Returns (domain_idx, offset)."""
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            raw = f.read().strip()
            if raw:
                parts = raw.split()
                return int(parts[0]), int(parts[1])
    except (FileNotFoundError, ValueError, IndexError):
        pass
    return 0, 0


def save_cursor(domain_idx, offset):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(f"{domain_idx} {offset}")


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(url):
    seen.add(url)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(url + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def is_config_url(original_url):
    """Check if URL's path ends with a known config filename."""
    path = original_url.split("?")[0].rstrip("/")
    return bool(_CONFIG_RE.search(path))


_SKIP_PREFIXES = ("gist:", "hf:", "codeberg:", "gitea:", "gl:", "bb:", "sf:")
_ORG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,38}$")


def _clean_org(repo):
    """keys.repo column value -> GitHub org/user (lowercase) or None."""
    if not repo:
        return None
    r = repo.strip()
    if r.startswith(_SKIP_PREFIXES):
        return None
    if r.startswith("issue:"):
        r = r[6:]
    if r.startswith("go:github.com/"):
        r = r[14:]
    elif r.startswith("github.com/"):
        r = r[11:]
    org = r.split("/")[0].split("#")[0].lower()
    return org if _ORG_RE.match(org) else None


def mine_orgs(limit=3000):
    """Top GitHub orgs by historical key count from keys.db (read-only)."""
    counts = {}
    try:
        ro = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=30)
        cur = ro.execute(
            "SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != ''")
        for (repo,) in cur:
            o = _clean_org(repo)
            if o:
                counts[o] = counts.get(o, 0) + 1
        ro.close()
    except Exception as e:
        log(f"mine_orgs DB error: {type(e).__name__} {e}")
    ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:limit]
    return [o for o, _ in ranked] or list(SEED_ORGS)


def load_orgs():
    """Cached org list; mines keys.db and writes ORGS_FILE on first run."""
    try:
        with open(ORGS_FILE, encoding="utf-8") as f:
            orgs = [l.strip().lower() for l in f if l.strip()]
        if orgs:
            return orgs
    except FileNotFoundError:
        pass
    orgs = mine_orgs()
    try:
        with open(ORGS_FILE, "w", encoding="utf-8") as f:
            f.write("\n".join(orgs) + "\n")
    except OSError as e:
        log(f"orgs file write error: {e}")
    return orgs


def fetch_cdx(prefix, offset, limit, flt):
    """Fetch one CDX page. Returns list of (timestamp, original_url) or None."""
    url = (
        "http://web.archive.org/cdx/search/cdx"
        f"?url={prefix}&matchType=prefix"
        f"&filter={flt}&filter=statuscode:200"
        f"&collapse=urlkey"
        f"&limit={limit}&offset={offset}"
    )
    for attempt in (1, 2, 3):
        try:
            r = S.get(url, headers=UA, timeout=60)
            if r.status_code == 429:
                retry = int(r.headers.get("Retry-After", 60))
                log(f"  cdx 429, retry-after {retry}s")
                time.sleep(retry)
                continue
            if r.status_code == 503:
                log(f"  cdx 503 (offline), backing off")
                time.sleep(30)
                continue
            if r.status_code != 200:
                raise RuntimeError(f"http {r.status_code}")
            lines = r.text.strip().split("\n")
            out = []
            for line in lines:
                line = line.strip()
                if not line or line.startswith("<"):
                    continue
                parts = line.split()
                if len(parts) < 6:
                    continue
                ts = parts[1]
                orig = parts[2]
                if orig and ts:
                    out.append((ts, orig))
            return out
        except requests.RequestException as e:
            if attempt == 3:
                log(f"  cdx-err {prefix} offset={offset}: {e}")
                return None
            time.sleep(8 * attempt)
        except Exception as e:
            if attempt == 3:
                log(f"  cdx-err {prefix}: {e}")
                return None
            time.sleep(8)
    return None


def fetch_archive(timestamp, original_url):
    """Fetch raw archived content from Wayback Machine."""
    url = f"https://web.archive.org/web/{timestamp}id_/{original_url}"
    for attempt in (1, 2, 3):
        try:
            r = S.get(url, headers=UA, timeout=30, stream=True)
            if r.status_code == 429:
                r.close()
                retry = int(r.headers.get("Retry-After", 30))
                log(f"  archive 429, retry-after {retry}s")
                time.sleep(retry)
                continue
            if r.status_code == 404:
                r.close()
                return None
            if r.status_code != 200:
                r.close()
                raise RuntimeError(f"http {r.status_code}")
            chunks = []
            size = 0
            for chunk in r.iter_content(65536):
                size += len(chunk)
                if size > FILE_CAP:
                    r.close()
                    return None
                chunks.append(chunk)
            r.close()
            return b"".join(chunks)
        except requests.RequestException as e:
            if attempt == 3:
                return None
            time.sleep(8 * attempt)
        except Exception:
            if attempt == 3:
                return None
            time.sleep(8)
    return None


def process_content(raw_bytes, original_url):
    """Decode, scan_text, filter junk, db_add. Returns key count."""
    text = raw_bytes.decode("utf-8", "replace")
    n = 0
    for prov, key in scan_text(text):
        if is_junk(key) or len(key) < 30:
            continue
        db_add(key, prov, original_url)
        n += 1
    return n


def cycle(orgs, max_items=MAX_ITEMS):
    global total, seen
    total = 0
    org_idx, offset = load_cursor()

    org = orgs[org_idx % len(orgs)]
    flt = CDX_FILTERS[org_idx % len(CDX_FILTERS)]
    prefix = f"raw.githubusercontent.com/{org}/"
    log(f"cycle {prefix} offset={offset} filter={flt.split(':')[-1]}")

    rows = fetch_cdx(prefix, offset, CDX_LIMIT, flt)
    if rows is None:
        log(f"  cdx-fail {prefix}, rotating")
        save_cursor((org_idx + 1) % len(orgs), 0)
        return

    if not rows:
        log(f"  cdx-empty {prefix}, rotating")
        save_cursor((org_idx + 1) % len(orgs), 0)
        return

    processed = 0
    config_items = [(ts, orig) for ts, orig in rows if is_config_url(orig)]

    config_items.sort(key=lambda x: x[0])  # oldest first for stability
    log(f"  cdx rows={len(rows)} config={len(config_items)}")

    for ts, orig in config_items:
        if processed >= max_items:
            break
        if orig in seen:
            continue

        mark_seen(orig)
        raw = fetch_archive(ts, orig)
        if raw is None:
            continue
        n = process_content(raw, orig)
        processed += 1

    # Advance cursor: even if we hit max_items early, advance offset so
    # next cycle doesn't re-scan the same rows.
    new_offset = offset + CDX_LIMIT
    if new_offset > len(rows) + offset or len(rows) < CDX_LIMIT:
        # Exhausted this org's batch or hit end; rotate
        save_cursor((org_idx + 1) % len(orgs), 0)
    else:
        save_cursor(org_idx, new_offset)

    log(f"  +{total} keys processed={processed} / {len(config_items)} configs")


def main():
    log("wayback_eternal started (prefix mode)")
    if "--remine" in sys.argv:
        try:
            _os.remove(ORGS_FILE)
        except OSError:
            pass
    orgs = load_orgs()
    log(f"orgs loaded: {len(orgs)}")
    if "--once" in sys.argv:
        cycle(orgs, max_items=min(MAX_ITEMS, 5))
        log("wayback_eternal --once done")
        return

    while True:
        try:
            cycle(orgs)
        except Exception as e:
            log(f"cycle-exception {type(e).__name__} {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()