"""ETERNAL Hacker News harvest: hn.algolia.com search API.
Scans story_text + comment_text + title for leaked API keys.
No quota; polite sleep. Lane: hn."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import re
import sqlite3
import sys
import time
import urllib.request
import urllib.parse
import urllib.error

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "hn_eternal.log")

QUERIES = [
    "sk-ant-api03", "sk-proj", "sk-or-v1", "gsk_", "tgp_",
    "pplx-", "nvapi-", "r8_", "xai-",
    "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY",
    "leaked api key", "leaked keys github", "api key in .env",
    "accidentally pushed keys", "openai key leaked",
    "anthropic key leaked", "hardcoded api key",
]

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
           "changeme", "deadbeef", "fffffff", "0000000")

seen = set()
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


def db_add(cur, key, prov, repo):
    if isinstance(key, tuple):
        key = key[0]
    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
        return 0
    seen.add(key)
    cur.execute(
        "INSERT INTO keys (hash, val, prov, status, found, repo) "
        "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
         "hn", repo))
    return 1


def fetch_json(url):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.loads(r.read().decode("utf-8", "replace"))


def scan_text(cur, text, src):
    n = 0
    for prov, rx in PATTERNS:
        for m in rx.findall(text or ""):
            n += db_add(cur, m, prov, src)
    return n


def cycle(db):
    cur = db.cursor()
    total = 0
    for q in QUERIES:
        n = 0
        for ep in ("search", "search_by_date"):
            url = ("https://hn.algolia.com/api/v1/" + ep +
                   "?query=" + urllib.parse.quote(q) + "&hitsPerPage=50")
            try:
                data = fetch_json(url)
            except Exception as e:
                log(f"  {q} {ep} err: {str(e)[:60]}")
                continue
            for h in data.get("hits", []):
                text = " ".join(filter(None, [
                    h.get("title"), h.get("story_text"),
                    h.get("comment_text"), h.get("url")]))
                n += scan_text(cur, text, f"hn:{h.get('objectID')}")
            time.sleep(1.2)
        db.commit()
        total += n
        log(f"  '{q}': +{n} new")
    return total


def main():
    log(f"=== hn eternal START pid={__import__('os').getpid()} "
        f"queries={len(QUERIES)}")
    try:
        con = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
        cur = con.cursor()
        cur.execute("SELECT val FROM keys WHERE found='hn'")
        for r in cur.fetchall():
            seen.add(r[0])
        con.close()
    except Exception:
        pass
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    while True:
        try:
            n = cycle(db)
            log(f"CYCLE done +{n}")
        except Exception as e:
            log(f"cycle err: {repr(e)[:100]}")
            time.sleep(30)
        time.sleep(90)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(0)