"""ETERNAL GitLab public CI artifacts: project list -> pipelines -> jobs -> traces + zips.

Endpoint(s):
  - api/v4/projects?visibility=public (200, no auth)
  - api/v4/projects/{pid}/pipelines (200, no auth)
  - api/v4/projects/{pid}/pipelines/{plid}/jobs (200, no auth)
  - /path/-/jobs/{jid}/raw (200, no auth — raw trace)
  - /path/-/jobs/{jid}/artifacts/download (302->CDN, some projects — zip)
Auth: none (public GitLab.com data)
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)

import hashlib, io, json, re, sqlite3, sys, threading, time, zipfile
import requests

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = _os.path.join(BASE, "data", "gitlab_artifacts_eternal.log")
CURSOR_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_cursor.txt")
SEEN_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_seen.txt")
N_THREADS = 4
SLEEP = 120
MAX_ITEMS = 200
FILE_CAP = 500_000           # max bytes to scan per text file
ARCHIVE_CAP = 20 * 1024 * 1024  # max zip to download (20MB)
TRACE_CAP = 2 * 1024 * 1024    # max trace to scan (2MB)
ARTIFACT_CAP = 20 * 1024 * 1024

from key_patterns import scan_text

GARBAGE = (
    "xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
    "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
    "abcdefghijklmnopqrst", "1234567890abcdef", "0000000", "fffffff",
)

SCAN_EXT = (
    ".env", ".json", ".yml", ".yaml", ".txt", ".cfg", ".conf", ".config",
    ".ini", ".py", ".sh", ".xml", ".properties", ".toml", ".yaml", ".md",
    ".js", ".ts", ".html",
)

FOUND_TAG = "gitlab-artifacts"

UA = {"User-Agent": "Mozilla/5.0"}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
  plan TEXT, price TEXT, remaining TEXT,
  found TEXT, repo TEXT)""")
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    if any(g in key for g in GARBAGE) or len(set(key)) < 6:
        return
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    repo_trunc = (repo or "")[:300]
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (h, key, prov, "NEW", FOUND_TAG, repo_trunc))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE) as f:
            return int(f.read().strip())
    except (FileNotFoundError, ValueError):
        return 0


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(str(c))


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for ln in f:
                s.add(ln.strip())
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def scan_text_chunk(text, repo):
    """Scan text for keys, feed to db_add."""
    n = 0
    for prov, key in scan_text(text):
        if is_junk(key):
            continue
        db_add(key, prov, repo)
        n += 1
    return n


# ── HTTP helpers ──────────────────────────────────────────

S = requests.Session()
S.trust_env = False  # bypass local dashscope proxy
S.headers.update(UA)


def _get(url, timeout=30, stream=False):
    """GET with retry and backoff."""
    for wait in (2, 4, 8):
        try:
            r = S.get(url, timeout=timeout, stream=stream)
            if r.status_code == 429 or r.status_code == 403:
                ra = r.headers.get("Retry-After")
                if ra and ra.isdigit():
                    time.sleep(int(ra))
                else:
                    time.sleep(30)
                continue
            if r.status_code == 200 or r.status_code == 302:
                return r
            if r.status_code >= 500:
                time.sleep(wait)
                continue
            return None  # 4xx that isn't 429=non-retryable
        except (requests.ConnectionError, requests.Timeout):
            time.sleep(wait)
    return None


# ── Scanning ──────────────────────────────────────────────

def scan_trace(text, repo):
    """Scan CI trace text for keys."""
    return scan_text_chunk(text, repo)


def scan_zip(content, repo):
    """Download and scan zip members (text files only)."""
    n = 0
    try:
        zf = zipfile.ZipFile(io.BytesIO(content))
        for name in zf.namelist():
            if not any(name.endswith(ex) for ex in SCAN_EXT):
                continue
            try:
                info = zf.getinfo(name)
                # zip-bomb guard: check UNCOMPRESSED size from metadata BEFORE
                # reading; zf.read() would fully decompress into RAM (Linux
                # OOM-killer SIGKILLs, no MemoryError raised).
                if info.file_size == 0 or info.file_size > FILE_CAP:
                    continue
                data = zf.open(name).read(FILE_CAP + 1)
                text = data.decode("utf-8", "replace")[:FILE_CAP]
                for prov, key in scan_text(text):
                    if is_junk(key):
                        continue
                    db_add(key, prov, f"{repo}#{name}")
                    n += 1
            except (zipfile.BadZipFile, MemoryError):
                continue
    except (zipfile.BadZipFile, MemoryError):
        pass
    return n


def _read_capped(r, cap):
    """Stream-read r up to cap bytes; None if oversized. Always closes r."""
    if r is None:
        return None
    chunks = []
    size = 0
    try:
        for chunk in r.iter_content(65536):
            size += len(chunk)
            if size > cap:
                return None
            chunks.append(chunk)
    finally:
        r.close()
    return b"".join(chunks)


def process_job(pid, path, jid, jname):
    """Fetch trace + artifacts for one job, return key counts."""
    nk = 0
    repo = f"{path}/job/{jid}"

    # raw trace (streamed, capped)
    trace_url = f"https://gitlab.com/{path}/-/jobs/{jid}/raw"
    r = _get(trace_url, timeout=30, stream=True)
    if r and r.status_code == 200:
        raw = _read_capped(r, TRACE_CAP)
        if raw:
            nk += scan_trace(raw.decode("utf-8", "replace"), repo)
    elif r is not None:
        r.close()

    # artifacts zip (streamed, capped)
    art_url = f"https://gitlab.com/{path}/-/jobs/{jid}/artifacts/download"
    r2 = _get(art_url, timeout=60, stream=True)
    if r2 and r2.status_code in (200, 302):
        if r2.status_code == 302:
            loc = r2.headers.get("location", "")
            r2.close()
            if loc:
                r3 = _get(loc, timeout=60, stream=True)
                if r3 and r3.status_code == 200:
                    payload = _read_capped(r3, ARCHIVE_CAP)
                    if payload:
                        nk += scan_zip(payload, repo)
                elif r3 is not None:
                    r3.close()
        else:
            payload = _read_capped(r2, ARCHIVE_CAP)
            if payload:
                nk += scan_zip(payload, repo)
    elif r2 is not None:
        r2.close()

    return nk


def process_project(pid, path):
    """Fetch pipelines → jobs for a single project, return key count."""
    nk = 0
    # recent pipelines
    pl_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines?per_page=5"
    r = _get(pl_url)
    if not r or r.status_code != 200:
        return 0
    try:
        pipelines = r.json()
    except json.JSONDecodeError:
        return 0
    if not isinstance(pipelines, list):
        return 0

    for pl in pipelines[:3]:  # at most 3 pipelines per cycle
        plid = pl.get("id")
        if not plid:
            continue
        # jobs for this pipeline
        jobs_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines/{plid}/jobs?per_page=15"
        r2 = _get(jobs_url)
        if not r2 or r2.status_code != 200:
            continue
        try:
            jobs = r2.json()
        except json.JSONDecodeError:
            continue
        if not isinstance(jobs, list):
            continue

        for j in jobs:
            jid = j.get("id")
            if not jid:
                continue
            jname = j.get("name", "")
            skey = f"{pid}/{jid}"
            if skey in seen:
                continue
            nk += process_job(pid, path, jid, jname)
            mark_seen(skey)

    return nk


def cycle(once=False):
    """One full pass: discover projects, scan their CI artifacts."""
    global total
    n_max = MAX_ITEMS if not once else min(MAX_ITEMS, 5)
    cursor = load_cursor()
    page = 1
    projects_seen = 0
    fail = 0
    keys_found = 0

    while projects_seen < n_max:
        url = (f"https://gitlab.com/api/v4/projects"
               f"?visibility=public&order_by=last_activity_at"
               f"&per_page=20&page={page}")
        r = _get(url)
        if not r or r.status_code != 200:
            fail += 1
            break
        try:
            projects = r.json()
        except json.JSONDecodeError:
            fail += 1
            break
        if not isinstance(projects, list) or not projects:
            break

        for p in projects:
            pid = p.get("id")
            path = p.get("path_with_namespace")
            if not pid or not path:
                continue
            if pid <= cursor:
                continue
            if projects_seen >= n_max:
                break

            nk = process_project(pid, path)
            if nk is None:
                fail += 1
                continue
            keys_found += nk
            projects_seen += 1
            save_cursor(pid)

            if once and projects_seen >= 5:
                break

        page += 1
        if once and page > 2:
            break

    log(f"cycle projects={projects_seen} keys={keys_found} fail={fail} +{total}")
    return projects_seen, keys_found, fail


def main():
    log("gitlab_artifacts_eternal started")
    once = "--once" in sys.argv
    if once:
        init_total = total
        n, nk, f = cycle(once=True)
        print(f"OK scanned={n} keys={nk} fail={f} db_keys={total - init_total}")
        sys.exit(0 if n > 0 else 1)
    while True:
        try:
            cycle()
        except Exception as exc:
            log(f"cycle crash: {exc}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()