"""ETERNAL Clojars firehose: repo root group-walk + /api/groups + jar scan.

Anonymous. Root HTML listing (https://repo.clojars.org/) = ~11k group dirs
(fetched once per cycle). Cursor walks GROUPS_PER_CYCLE groups; per group
GET /api/groups/{g} (list of {jar_name, latest_version}); per unseen g:a
fetch jar from repo.clojars.org (cap 8MB), zipfile scan. seen=g:a.
mark_seen ONLY on success.
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import re
import sqlite3
import sys
import threading
import time
import zipfile

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "data/clojars_eternal.log")
CURSOR_FILE = os.path.join(KF_BASE, "data/clojars_cursor.txt")
SEEN_FILE = os.path.join(KF_BASE, "data/clojars_seen.txt")
N_THREADS = 6
GROUPS_PER_CYCLE = 40
JAR_CAP = 8 * 1024 * 1024
FILE_CAP = 500_000
SLEEP = 90

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

SCAN_EXT = (".clj", ".cljs", ".cljc", ".edn", ".json", ".yaml", ".yml",
            ".properties", ".xml", ".env", ".txt", ".cfg")

UA = {"User-Agent": "keyfarm-clojars/1.0"}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "clojars", repo))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            return int(f.read().strip() or 0)
    except (FileNotFoundError, ValueError):
        return 0


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(str(c))


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def work(g, a, v):
    """Fetch jar, scan. -1 on final fail."""
    n = 0
    gp = g.replace(".", "/")
    url = f"https://repo.clojars.org/{gp}/{a}/{v}/{a}-{v}.jar"
    for attempt in (1, 2, 3):
        try:
            r = requests.get(url, headers=UA, timeout=40)
            if r.status_code == 404:
                return 0
            if r.status_code != 200:
                raise RuntimeError(f"http {r.status_code}")
            if len(r.content) > JAR_CAP:
                return 0
            zf = zipfile.ZipFile(io.BytesIO(r.content))
            for m in zf.infolist():
                if m.is_dir() or m.file_size > FILE_CAP:
                    continue
                fn = m.filename.lower()
                if not any(fn.endswith(e) for e in SCAN_EXT):
                    continue
                data = zf.read(m)
                if b"\x00" in data[:8192]:
                    continue  # binary archive member — no real text key has NUL
                text = data.decode("utf-8", errors="ignore")
                for prov, key in scan_text(text):
                    if isinstance(key, tuple):
                        key = key[0]
                    if any(g in key for g in GARBAGE) or len(key) < 30:
                        continue
                    db_add(key, prov, f"clojars:{g}:{a}")
                    n += 1
            return n
        except Exception:
            if attempt == 3:
                log(f"  jar-err {g}:{a}")
                return -1
            time.sleep(8)
    return -1


def group_artifacts(g):
    """GET /api/groups/{g} -> [(a, v), ...]; -1 on final fail."""
    for attempt in (1, 2, 3):
        try:
            r = requests.get(f"https://clojars.org/api/groups/{g}",
                             headers=UA, timeout=30)
            if r.status_code == 404:
                return []
            if r.status_code != 200:
                raise RuntimeError(f"http {r.status_code}")
            out = []
            for it in r.json():
                a = it.get("jar_name")
                v = it.get("latest_version")
                if a and v:
                    out.append((a, v))
            return out
        except Exception:
            if attempt == 3:
                log(f"  group-err {g}")
                return None
            time.sleep(8)
    return None


def cycle():
    global total
    total = 0
    # root group listing (1.4MB html)
    for attempt in (1, 2, 3):
        try:
            r = requests.get("https://repo.clojars.org/", headers=UA,
                             timeout=60)
            if r.status_code != 200:
                raise RuntimeError(f"http {r.status_code}")
            break
        except Exception as e:
            if attempt == 3:
                log(f"root-err {type(e).__name__}")
                return
            time.sleep(8)
    groups = re.findall(r'href="([^"/]+)/"', r.text)
    groups = [g for g in groups if g not in (".meta",)]
    if not groups:
        log("root-err empty")
        return

    idx = load_cursor() % len(groups)
    items = []
    for _ in range(GROUPS_PER_CYCLE):
        g = groups[idx]
        idx = (idx + 1) % len(groups)
        arts = group_artifacts(g)
        if arts is None:
            continue          # group retried next rotation
        for a, v in arts:
            sid = f"{g}:{a}"
            if sid not in seen:
                items.append((g, a, v))
    save_cursor(idx)

    ok = fail = 0
    from concurrent.futures import ThreadPoolExecutor, as_completed
    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futs = {ex.submit(work, g, a, v): (g, a) for g, a, v in items}
        for fut in as_completed(futs):
            g, a = futs[fut]
            try:
                res = fut.result()
            except Exception:
                res = -1
            if res == -1:
                fail += 1          # NOT marked seen: retried next cycle
            else:
                ok += 1
                mark_seen(f"{g}:{a}")

    log(f"cycle: groups={len(groups)} idx={idx} items={len(items)} ok={ok} "
        f"fail={fail} +{total} keys")


def main():
    log("clojars_eternal started")
    if "--once" in sys.argv:
        cycle()
        return
    while True:
        try:
            cycle()
        except Exception as e:
            log(f"cycle-err {type(e).__name__}: {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()