"""Retro full-file scan: all (repo, path) hit-pairs already in DB.
Cycle parser fetches full files only for current-cycle hits (runtime dedup);
this one-shot re-harvests every historical hit with the complete file body."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import json
import re
import sqlite3
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.insert(0, KF_BASE)
from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
N_THREADS = 16


def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)


def load_targets():
    db = sqlite3.connect(DB, timeout=60)
    c = db.cursor()
    c.execute("SELECT repo FROM keys WHERE repo LIKE '%/%' "
              "AND found IN ('gh-eternal','gh-dork2','legacy-waves','wiki')")
    pairs = set()
    for (r,) in c.fetchall():
        if r.count("/") >= 1 and len(r) < 200 and "…" not in r:
            parts = r.split("/")
            repo = "/".join(parts[:2])
            path = "/".join(parts[2:])
            if repo and path and not path.startswith("commit"):
                pairs.add((repo, path))
    db.close()
    return list(pairs)


def fetch_and_scan(pair, tok):
    repo, path = pair
    try:
        import requests
        r = requests.get(
            f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
            headers={"Authorization": f"Bearer {tok}",
                     "User-Agent": "keyfarm"},
            timeout=20)
        if r.status_code != 200 or len(r.text) > 2_000_000:
            return pair, None
        found = []
        for prov, rx in PATTERNS:
            for key in rx.findall(r.text):
                if isinstance(key, tuple):
                    key = key[0]
                if any(g in key for g in GARBAGE) or len(key) < 30:
                    continue
                found.append((prov, key))
        return pair, found
    except Exception:
        return pair, None


def main():
    pairs = load_targets()
    log(f"targets: {len(pairs)} files")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
    cur = db.cursor()
    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
    new_total = 0
    lock = __import__("threading").Lock()

    def work(pair):
        nonlocal new_total
        tok = POOL.tokens[hash(pair[0]) % len(POOL.tokens)]
        pair, found = fetch_and_scan(pair, tok)
        n = 0
        if found:
            for prov, key in found:
                if key in seen:
                    continue
                with lock:
                    seen.add(key)
                    cur.execute(
                        "INSERT OR IGNORE INTO keys "
                        "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
                        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
                         "NEW", "retro-scan", f"{pair[0]}/{pair[1]}"))
                    new_total += 1
                n += 1
        return n

    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futs = {ex.submit(work, p): p for p in pairs}
        done = 0
        for fut in as_completed(futs):
            done += 1
            if done % 200 == 0:
                db.commit()
                log(f"{done}/{len(pairs)} scanned, +{new_total} new")
    db.commit()
    log(f"done: +{new_total} new keys")
    db.close()


if __name__ == "__main__":
    main()