keyhunter-parser/stackex_eternal.py

"""ETERNAL StackExchange key-dork lane: search/advanced + withbody scan.

Anonymous quota ~300/day. One query per SLEEP (300s = 288/day). Dork
rotation + page cursor. body HTML scanned via key_patterns.scan_text.
mark_seen ONLY on success.
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import sqlite3
import sys
import threading
import time
from urllib.parse import quote

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "data/stackex_eternal.log")
CURSOR_FILE = os.path.join(KF_BASE, "data/stackex_cursor.txt")
SEEN_FILE = os.path.join(KF_BASE, "data/stackex_seen.txt")
PAGES_PER_QUERY = 3
SLEEP = 300

from key_patterns import scan_text

DORKS = [
    "ANTHROPIC_API_KEY", "sk-ant-api03", "sk-proj-", "OPENAI_API_KEY",
    "ghp_", "gho_", "xoxb-", "AIza", "MISTRAL_API_KEY", "GROQ_API_KEY",
    "HF_TOKEN", "glpat-", "dop_v1_", "whsec_", "xkeysib-", "napi_",
    "tvly-", "dckr_pat_", "squ_", "glc_", "re_[A-Za-z0-9]", "shpat_",
    "sq0atp-", "rubygems_", "lsv2_pt", "sentry dsn", "SECRET_KEY",
    "PINECONE_API_KEY", "ELEVENLABS", "DEEPSEEK", "TOGETHER_API_KEY",
    "FIREWORKS", "OPENROUTER", "PERPLEXITY", "GEMINI_API_KEY",
    "GOOGLE_API_KEY", "AWS_SECRET_ACCESS_KEY", "STRIPE_API_KEY",
    "SENDGRID", "TWILIO_AUTH_TOKEN", "MAPBOX", "MAILGUN",
]

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

UA = {"User-Agent": "keyfarm-research/1.0"}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "stackex", repo))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            return int(f.read().strip() or 0)
    except Exception:
        return 0


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(str(c))


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                s.add(line.strip())
    except Exception:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def scan_body(body, qid, n):
    for prov, key in scan_text(body):
        if isinstance(key, tuple):
            key = key[0]
        if any(g in key for g in GARBAGE) or len(key) < 30:
            continue
        db_add(key, prov, f"so:{qid}")
        n += 1
    return n


def cycle():
    global total
    total = 0
    di = load_cursor() % len(DORKS)
    dork = DORKS[di]
    n_items = n_keys = 0
    for page in range(1, PAGES_PER_QUERY + 1):
        url = ("https://api.stackexchange.com/2.3/search/advanced"
               f"?order=desc&sort=activity&q={quote(dork)}"
               f"&site=stackoverflow&filter=withbody&pagesize=100&page={page}")
        d = None
        for attempt in (1, 2):
            try:
                r = requests.get(url, headers=UA, timeout=25)
                if r.status_code != 200:
                    raise RuntimeError(f"http {r.status_code}")
                d = r.json()
                break
            except Exception as e:
                if attempt == 2:
                    log(f"query-err dork={dork} page={page} "
                        f"{type(e).__name__}")
                    return
                time.sleep(5)
        if not d:
            break
        items = d.get("items") or []
        if not items:
            break
        for it in items:
            qid = it.get("question_id")
            sid = f"q{qid}"
            if qid is None or sid in seen:
                continue
            n_items += 1
            body = it.get("body") or ""
            n_keys += scan_body(body, qid, 0)
            # answers not fetched here; question body only (backoff budget)
            mark_seen(sid)
        if not d.get("has_more"):
            break

    save_cursor(load_cursor() + 1)
    log(f"cycle: dork={dork} items={n_items} +{n_keys} keys "
        f"({total} total inserts)")


def main():
    log("stackex_eternal started")
    if "--once" in sys.argv:
        cycle()
        return
    while True:
        try:
            cycle()
        except Exception as e:
            log(f"cycle-err {type(e).__name__}: {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()