KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

se_eternal.py

@@ -0,0 +1,149 @@
+"""ETERNAL StackExchange hunt: search/advanced with body, strip HTML, grep keys.
+Free API: 300 req/day without token. sort=creation desc -> newest first, has_more paging."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import hashlib
+import json
+import re
+import sqlite3
+import time
+import urllib.parse
+import urllib.error
+import urllib.request
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.path.join(KF_BASE, "se_eternal.log")
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
+API = "https://api.stackexchange.com/2.3/search/advanced"
+
+QUERIES = [
+    "sk-ant-api03", "sk-proj-", "anthropic api key", "OPENAI_API_KEY",
+    "sk-or-v1", "gsk_", "openrouter api key", "dashscope api key",
+    "deepseek api key", "zhipu api key", "moonshot api key", "kimi",
+    "together api key", "together.ai api key", "replicate api token",
+    "elevenlabs api key", "groq api key", "perplexity api key", "pplx-",
+    "stability api key", "nvidia api key", "nvapi-", "twitter api key",
+    "telegram bot token 123456", "github token pasted", "aws secret",
+    "google api key exposed", "api key in code", "secret key hardcoded",
+]
+
+from key_patterns import PATTERNS
+
+GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
+           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
+           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
+           "changeme", "deadbeef", "fffffff", "0000000")
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    print(line, flush=True)
+    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
+        f.write(line)
+
+
+def db_add(cur, seen, key, prov, repo):
+    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
+        return 0
+    seen.add(key)
+    cur.execute(
+        "INSERT INTO keys (hash, val, prov, status, found, repo) "
+        "VALUES (?,?,?,?,?,?) "
+        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+        "WHERE keys.status IN ('DEAD','ERR')",
+        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
+         "se", repo))
+    return 1
+
+
+def strip_html(s):
+    s = re.sub(r"<pre.*?</pre>", " ", s, flags=re.S | re.I)
+    s = re.sub(r"<code.*?</code>", " ", s, flags=re.S | re.I)
+    s = re.sub(r"<[^>]+>", " ", s)
+    return s
+
+
+SITES = ["stackoverflow", "superuser", "serverfault", "webapps",
+         "wordpress", "softwareengineering", "crypto", "devops",
+         "unix", "askubuntu"]
+_site_i = 0
+
+
+def fetch_page(query, page):
+    global _site_i
+    site = SITES[_site_i % len(SITES)]
+    _site_i += 1
+    params = {
+        "q": query, "site": site, "order": "desc",
+        "sort": "creation", "pagesize": "100", "page": str(page),
+        "filter": "withbody",
+    }
+    url = API + "?" + urllib.parse.urlencode(params)
+    req = urllib.request.Request(url, headers=UA)
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.loads(r.read())
+
+
+def cycle(db, qi):
+    cur = db.cursor()
+    cur.execute("SELECT val FROM keys")
+    seen = {r[0] for r in cur.fetchall()}
+    total = 0
+    q = QUERIES[qi % len(QUERIES)]
+    qi += 1
+    n_questions = 0
+    try:
+        data = fetch_page(q, 1)
+        quota = data.get("quota_remaining", 300)
+        backoff = data.get("backoff", 0)
+        items = data.get("items", [])
+        log(f"query {q!r}: {len(items)} items, quota={quota}")
+        for it in items:
+            for key in it.get("tags", []):
+                pass
+            qid = it.get("question_id", "")
+            body = strip_html(it.get("body", "") or "")
+            title = it.get("title", "") or ""
+            text = body + " " + strip_html(title)
+            n_questions += 1
+            for prov, rx in PATTERNS:
+                for key in rx.findall(text):
+                    if isinstance(key, tuple):
+                        key = key[-1]
+                    hit = db_add(cur, seen, key, prov, f"se:{qid}")
+                    if hit:
+                        total += 1
+                        log(f"  + {prov} {key[:24]}... se:{qid}")
+        if backoff:
+            log(f"backoff {backoff}s from API")
+            time.sleep(backoff)
+        if quota < 50:
+            log("quota low, sleeping 30min")
+            db.commit()
+            time.sleep(1800)
+    except urllib.error.HTTPError as e:
+        log(f"HTTPError {e.code} for {q!r}; throttle -> sleep 5min")
+        db.commit()
+        time.sleep(300)
+    except Exception as e:
+        log(f"cycle err: {str(e)[:80]}")
+    db.commit()
+    return total, qi
+
+
+def main():
+    log(f"=== se eternal START pid={__import__('os').getpid()} ===")
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+    qi = 0
+    while True:
+        try:
+            n, qi = cycle(db, qi)
+            log(f"CYCLE done: +{n}; resting 3min")
+        except Exception as e:
+            log(f"cycle error (continuing): {e}")
+        time.sleep(180)
+
+
+if __name__ == "__main__":
+    main()