"""ETERNAL StackExchange hunt: search/advanced with body, strip HTML, grep keys.
Free API: 300 req/day without token. sort=creation desc -> newest first, has_more paging."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import json
import re
import sqlite3
import time
import urllib.parse
import urllib.error
import urllib.request

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "se_eternal.log")
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
API = "https://api.stackexchange.com/2.3/search/advanced"

QUERIES = [
    "sk-ant-api03", "sk-proj-", "anthropic api key", "OPENAI_API_KEY",
    "sk-or-v1", "gsk_", "openrouter api key", "dashscope api key",
    "deepseek api key", "zhipu api key", "moonshot api key", "kimi",
    "together api key", "together.ai api key", "replicate api token",
    "elevenlabs api key", "groq api key", "perplexity api key", "pplx-",
    "stability api key", "nvidia api key", "nvapi-", "twitter api key",
    "telegram bot token 123456", "github token pasted", "aws secret",
    "google api key exposed", "api key in code", "secret key hardcoded",
]

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, seen, key, prov, repo):
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "se", repo))
    return 1


def strip_html(s):
    s = re.sub(r"<pre.*?</pre>", " ", s, flags=re.S | re.I)
    s = re.sub(r"<code.*?</code>", " ", s, flags=re.S | re.I)
    s = re.sub(r"<[^>]+>", " ", s)
    return s


SITES = ["stackoverflow", "superuser", "serverfault", "webapps",
         "wordpress", "softwareengineering", "crypto", "devops",
         "unix", "askubuntu"]
_site_i = 0


def fetch_page(query, page):
    global _site_i
    site = SITES[_site_i % len(SITES)]
    _site_i += 1
    params = {
        "q": query, "site": site, "order": "desc",
        "sort": "creation", "pagesize": "100", "page": str(page),
        "filter": "withbody",
    }
    url = API + "?" + urllib.parse.urlencode(params)
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.loads(r.read())


def cycle(db, qi):
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    seen = {r[0] for r in cur.fetchall()}
    total = 0
    q = QUERIES[qi % len(QUERIES)]
    qi += 1
    n_questions = 0
    try:
        data = fetch_page(q, 1)
        quota = data.get("quota_remaining", 300)
        backoff = data.get("backoff", 0)
        items = data.get("items", [])
        log(f"query {q!r}: {len(items)} items, quota={quota}")
        for it in items:
            for key in it.get("tags", []):
                pass
            qid = it.get("question_id", "")
            body = strip_html(it.get("body", "") or "")
            title = it.get("title", "") or ""
            text = body + " " + strip_html(title)
            n_questions += 1
            for prov, rx in PATTERNS:
                for key in rx.findall(text):
                    if isinstance(key, tuple):
                        key = key[-1]
                    hit = db_add(cur, seen, key, prov, f"se:{qid}")
                    if hit:
                        total += 1
                        log(f"  + {prov} {key[:24]}... se:{qid}")
        if backoff:
            log(f"backoff {backoff}s from API")
            time.sleep(backoff)
        if quota < 50:
            log("quota low, sleeping 30min")
            db.commit()
            time.sleep(1800)
    except urllib.error.HTTPError as e:
        log(f"HTTPError {e.code} for {q!r}; throttle -> sleep 5min")
        db.commit()
        time.sleep(300)
    except Exception as e:
        log(f"cycle err: {str(e)[:80]}")
    db.commit()
    return total, qi


def main():
    log(f"=== se eternal START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    qi = 0
    while True:
        try:
            n, qi = cycle(db, qi)
            log(f"CYCLE done: +{n}; resting 3min")
        except Exception as e:
            log(f"cycle error (continuing): {e}")
        time.sleep(180)


if __name__ == "__main__":
    main()