KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

stackex_eternal.py

@@ -0,0 +1,177 @@
+"""ETERNAL StackExchange key-dork lane: search/advanced + withbody scan.
+
+Anonymous quota ~300/day. One query per SLEEP (300s = 288/day). Dork
+rotation + page cursor. body HTML scanned via key_patterns.scan_text.
+mark_seen ONLY on success.
+"""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import sqlite3
+import sys
+import threading
+import time
+from urllib.parse import quote
+
+import requests
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.path.join(KF_BASE, "data/stackex_eternal.log")
+CURSOR_FILE = os.path.join(KF_BASE, "data/stackex_cursor.txt")
+SEEN_FILE = os.path.join(KF_BASE, "data/stackex_seen.txt")
+PAGES_PER_QUERY = 3
+SLEEP = 300
+
+from key_patterns import scan_text
+
+DORKS = [
+    "ANTHROPIC_API_KEY", "sk-ant-api03", "sk-proj-", "OPENAI_API_KEY",
+    "ghp_", "gho_", "xoxb-", "AIza", "MISTRAL_API_KEY", "GROQ_API_KEY",
+    "HF_TOKEN", "glpat-", "dop_v1_", "whsec_", "xkeysib-", "napi_",
+    "tvly-", "dckr_pat_", "squ_", "glc_", "re_[A-Za-z0-9]", "shpat_",
+    "sq0atp-", "rubygems_", "lsv2_pt", "sentry dsn", "SECRET_KEY",
+    "PINECONE_API_KEY", "ELEVENLABS", "DEEPSEEK", "TOGETHER_API_KEY",
+    "FIREWORKS", "OPENROUTER", "PERPLEXITY", "GEMINI_API_KEY",
+    "GOOGLE_API_KEY", "AWS_SECRET_ACCESS_KEY", "STRIPE_API_KEY",
+    "SENDGRID", "TWILIO_AUTH_TOKEN", "MAPBOX", "MAILGUN",
+]
+
+GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
+           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
+           "abcdefghijklmnopqrst", "1234567890abcdef")
+
+UA = {"User-Agent": "keyfarm-research/1.0"}
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    with open(LOG, "a", encoding="utf-8") as f:
+        f.write(line)
+
+
+conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
+                       isolation_level=None)
+lock = threading.Lock()
+total = 0
+
+
+def db_add(key, prov, repo):
+    global total
+    with lock:
+        conn.execute(
+            "INSERT INTO keys (hash, val, prov, status, found, repo) "
+            "VALUES (?,?,?,?,?,?) "
+            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
+            "found=excluded.found, repo=excluded.repo "
+            "WHERE keys.status IN ('DEAD','ERR')",
+            (hashlib.sha1(key.encode()).hexdigest()[:16],
+             key, prov, "NEW", "stackex", repo))
+        total += 1
+
+
+def load_cursor():
+    try:
+        with open(CURSOR_FILE, encoding="utf-8") as f:
+            return int(f.read().strip() or 0)
+    except Exception:
+        return 0
+
+
+def save_cursor(c):
+    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
+        f.write(str(c))
+
+
+def load_seen():
+    s = set()
+    try:
+        with open(SEEN_FILE, encoding="utf-8") as f:
+            for line in f:
+                s.add(line.strip())
+    except Exception:
+        pass
+    return s
+
+
+seen = load_seen()
+
+
+def mark_seen(p):
+    seen.add(p)
+    with open(SEEN_FILE, "a", encoding="utf-8") as f:
+        f.write(p + "\n")
+
+
+def scan_body(body, qid, n):
+    for prov, key in scan_text(body):
+        if isinstance(key, tuple):
+            key = key[0]
+        if any(g in key for g in GARBAGE) or len(key) < 30:
+            continue
+        db_add(key, prov, f"so:{qid}")
+        n += 1
+    return n
+
+
+def cycle():
+    global total
+    total = 0
+    di = load_cursor() % len(DORKS)
+    dork = DORKS[di]
+    n_items = n_keys = 0
+    for page in range(1, PAGES_PER_QUERY + 1):
+        url = ("https://api.stackexchange.com/2.3/search/advanced"
+               f"?order=desc&sort=activity&q={quote(dork)}"
+               f"&site=stackoverflow&filter=withbody&pagesize=100&page={page}")
+        d = None
+        for attempt in (1, 2):
+            try:
+                r = requests.get(url, headers=UA, timeout=25)
+                if r.status_code != 200:
+                    raise RuntimeError(f"http {r.status_code}")
+                d = r.json()
+                break
+            except Exception as e:
+                if attempt == 2:
+                    log(f"query-err dork={dork} page={page} "
+                        f"{type(e).__name__}")
+                    return
+                time.sleep(5)
+        if not d:
+            break
+        items = d.get("items") or []
+        if not items:
+            break
+        for it in items:
+            qid = it.get("question_id")
+            sid = f"q{qid}"
+            if qid is None or sid in seen:
+                continue
+            n_items += 1
+            body = it.get("body") or ""
+            n_keys += scan_body(body, qid, 0)
+            # answers not fetched here; question body only (backoff budget)
+            mark_seen(sid)
+        if not d.get("has_more"):
+            break
+
+    save_cursor(load_cursor() + 1)
+    log(f"cycle: dork={dork} items={n_items} +{n_keys} keys "
+        f"({total} total inserts)")
+
+
+def main():
+    log("stackex_eternal started")
+    if "--once" in sys.argv:
+        cycle()
+        return
+    while True:
+        try:
+            cycle()
+        except Exception as e:
+            log(f"cycle-err {type(e).__name__}: {e}")
+        time.sleep(SLEEP)
+
+
+if __name__ == "__main__":
+    main()