"""Deep repo scan: for each repo with hit history, pull full git tree,
fetch config-ish files, grep PATTERNS. Catches files beyond search paging."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import sqlite3
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.insert(0, KF_BASE)
import requests  # noqa: E402
from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
N_THREADS = 16
S = requests.Session()
S.headers.update({"User-Agent": "keyfarm"})

# file-name filters: high-yield config carriers
HOT_NAMES = (".env", "config", "secret", "token", "apikey", "api_key",
             "credential", "mcp.json", "settings", "keys.json", "auth",
             "docker-compose", "dockerfile", ".npmrc", "application.yml",
             "application.properties", "claude", "vercel", "netlify",
             "readme", "example", "sample", "demo", "backup", "notes",
             ".bak", "history", "scratch", "tmp", "old", "copy")
HOT_EXT = (".env", ".json", ".yml", ".yaml", ".toml", ".ini", ".cfg",
           ".properties", ".ipynb", ".sh", ".md", ".txt", ".pem.key")
MAX_FILES_PER_REPO = 60


def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)


def load_repos():
    db = sqlite3.connect(DB, timeout=60)
    c = db.cursor()
    c.execute("SELECT repo FROM keys WHERE repo LIKE '_%/_%' "
              "AND found IN ('gh-eternal','gh-dork2','retro-scan','legacy-waves')")
    repos = set()
    for (r,) in c.fetchall():
        if r.count("/") >= 1 and len(r) < 150 and "…" not in r:
            repo = "/".join(r.split("/")[:2])
            if repo.isascii() and not repo.endswith(".git"):
                repos.add(repo)
    db.close()
    return sorted(repos)


def repo_tree_files(repo, tok):
    """Full recursive tree of repo via API. Returns list of candidate paths."""
    try:
        r = S.get(
            f"https://api.github.com/repos/{repo}/git/trees/HEAD?recursive=1",
            headers={"Authorization": f"Bearer {tok}"},
            timeout=20)
        if r.status_code == 409:  # empty repo
            return []
        if r.status_code != 200:
            return None
        paths = []
        for t in r.json().get("tree", []):
            if t.get("type") != "blob":
                continue
            p = t.get("path", "")
            low = p.lower()
            base = low.rsplit("/", 1)[-1]
            if any(h in low for h in HOT_NAMES) or base.endswith(HOT_EXT):
                if not any(x in low for x in ("node_modules/", "vendor/",
                                              "dist/", "build/", ".min.")):
                    paths.append(p)
        return paths[:MAX_FILES_PER_REPO]
    except Exception:
        return None


def fetch_and_scan(repo, path, tok):
    try:
        r = S.get(f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
                  headers={"Authorization": f"Bearer {tok}"},
                  timeout=20)
        if r.status_code != 200 or len(r.text) > 2_000_000:
            return []
        found = []
        for prov, rx in PATTERNS:
            for key in rx.findall(r.text):
                if isinstance(key, tuple):
                    key = key[0]
                if any(g in key for g in GARBAGE) or len(key) < 30:
                    continue
                found.append((prov, key))
        return found
    except Exception:
        return []


def main(threads=N_THREADS):
    repos = load_repos()
    log(f"repos: {len(repos)} to deep-scan")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
    cur = db.cursor()
    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
    new_total = 0
    lock = __import__("threading").Lock()

    def work(repo):
        nonlocal new_total
        tok = POOL.tokens[hash(repo) % len(POOL.tokens)]
        paths = repo_tree_files(repo, tok)
        if not paths:
            return 0
        n = 0
        for p in paths:
            for prov, key in fetch_and_scan(repo, p, tok):
                if key in seen:
                    continue
                with lock:
                    seen.add(key)
                    cur.execute(
                        "INSERT OR IGNORE INTO keys "
                        "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
                        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
                         "NEW", "deep-scan", f"{repo}/{p}"))
                    new_total += 1
                n += 1
        return n

    done = 0
    with ThreadPoolExecutor(max_workers=threads) as ex:
        futs = {ex.submit(work, r): r for r in repos}
        for fut in as_completed(futs):
            done += 1
            if done % 50 == 0:
                with lock:
                    db.commit()
                log(f"{done}/{len(repos)} repos, +{new_total} new")
    with lock:
        db.commit()
    log(f"done: +{new_total} new keys")
    db.close()


if __name__ == "__main__":
    main()