"""Scan GitHub wikis of repo-hits: code search does NOT index wiki pages.
Wiki = separate git repo {owner}/{repo}.wiki.git — clone, grep PATTERNS."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import shutil
import sqlite3
import subprocess
import sys
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.insert(0, KF_BASE)
from gh_eternal import PATTERNS, GARBAGE, POOL  # noqa: E402

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "wiki_scan.log")


def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)


def wiki_urls_from_db(limit):
    db = sqlite3.connect(DB, timeout=60)
    c = db.cursor()
    c.execute("SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != '' "
              "GROUP BY repo ORDER BY count(*) DESC LIMIT ?", (limit,))
    repos = []
    for (r,) in c.fetchall():
        repo = r.split("/")[0] + "/" + r.split("/")[1] if "/" in r else ""
        if repo and repo not in repos:
            repos.append(repo)
    db.close()
    return repos


def clone_and_scan(repo):
    """Clone wiki repo shallow, grep PATTERNS over all files."""
    tok = POOL.tokens[0] if POOL.tokens else ""
    url = f"https://x-access-token:{tok}@github.com/{repo}.wiki.git"
    tmp = tempfile.mkdtemp(prefix="wiki_")
    found = []
    try:
        r = subprocess.run(
            ["git", "clone", "--depth", "1", "-q", url, tmp],
            capture_output=True, timeout=30)
        if r.returncode != 0:
            return repo, "no-wiki", []
        import re
        texts = []
        import pathlib
        for f in pathlib.Path(tmp).rglob("*"):
            if f.is_file() and f.stat().st_size < 1_000_000:
                try:
                    texts.append(f.read_text(errors="ignore"))
                except Exception:
                    pass
        blob = "\n".join(texts)
        for prov, rx in PATTERNS:
            for key in rx.findall(blob):
                if isinstance(key, tuple):
                    key = key[0]
                if any(g in key for g in GARBAGE) or len(key) < 30:
                    continue
                found.append((prov, key))
        return repo, f"scanned ({len(texts)} files)", found
    except subprocess.TimeoutExpired:
        return repo, "timeout", []
    except Exception as e:
        return repo, f"error {e!r}", []
    finally:
        shutil.rmtree(tmp, ignore_errors=True)


def main(limit, threads):
    repos = wiki_urls_from_db(limit)
    log(f"start: {len(repos)} repos to wiki-scan, {threads} threads")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
    cur = db.cursor()
    seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
    new_total = 0
    lock_cur = __import__("threading").Lock()

    def scan(repo):
        nonlocal new_total
        repo, status, found = clone_and_scan(repo)
        n = 0
        for prov, key in found:
            if key in seen:
                continue
            with lock_cur:
                seen.add(key)
                import hashlib
                cur.execute(
                    "INSERT OR IGNORE INTO keys "
                    "(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
                    (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
                     "NEW", "wiki-scan", f"{repo}/wiki"))
                db.commit()
                new_total += 1
            n += 1
        if n or "scanned" in status:
            log(f"{repo}: {status}, +{n} new")
        return n

    with ThreadPoolExecutor(max_workers=threads) as ex:
        futs = [ex.submit(scan, r) for r in repos]
        for _ in as_completed(futs):
            pass
    log(f"done: +{new_total} new keys")
    db.close()


if __name__ == "__main__":
    limit = int(sys.argv[1]) if len(sys.argv) > 1 else 300
    threads = int(sys.argv[2]) if len(sys.argv) > 2 else 8
    main(limit, threads)