KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

hn_eternal.py

@@ -0,0 +1,130 @@
+"""ETERNAL Hacker News harvest: hn.algolia.com search API.
+Scans story_text + comment_text + title for leaked API keys.
+No quota; polite sleep. Lane: hn."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import hashlib
+import io
+import json
+import re
+import sqlite3
+import sys
+import time
+import urllib.request
+import urllib.parse
+import urllib.error
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+LOG = os.path.join(KF_BASE, "hn_eternal.log")
+
+QUERIES = [
+    "sk-ant-api03", "sk-proj", "sk-or-v1", "gsk_", "tgp_",
+    "pplx-", "nvapi-", "r8_", "xai-",
+    "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY",
+    "leaked api key", "leaked keys github", "api key in .env",
+    "accidentally pushed keys", "openai key leaked",
+    "anthropic key leaked", "hardcoded api key",
+]
+
+from key_patterns import PATTERNS
+
+GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
+           "fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
+           "aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
+           "changeme", "deadbeef", "fffffff", "0000000")
+
+seen = set()
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    print(line, flush=True)
+    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
+        f.write(line)
+
+
+def db_add(cur, key, prov, repo):
+    if isinstance(key, tuple):
+        key = key[0]
+    if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
+        return 0
+    seen.add(key)
+    cur.execute(
+        "INSERT INTO keys (hash, val, prov, status, found, repo) "
+        "VALUES (?,?,?,?,?,?) "
+        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+        "WHERE keys.status IN ('DEAD','ERR')",
+        (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
+         "hn", repo))
+    return 1
+
+
+def fetch_json(url):
+    req = urllib.request.Request(url, headers=UA)
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.loads(r.read().decode("utf-8", "replace"))
+
+
+def scan_text(cur, text, src):
+    n = 0
+    for prov, rx in PATTERNS:
+        for m in rx.findall(text or ""):
+            n += db_add(cur, m, prov, src)
+    return n
+
+
+def cycle(db):
+    cur = db.cursor()
+    total = 0
+    for q in QUERIES:
+        n = 0
+        for ep in ("search", "search_by_date"):
+            url = ("https://hn.algolia.com/api/v1/" + ep +
+                   "?query=" + urllib.parse.quote(q) + "&hitsPerPage=50")
+            try:
+                data = fetch_json(url)
+            except Exception as e:
+                log(f"  {q} {ep} err: {str(e)[:60]}")
+                continue
+            for h in data.get("hits", []):
+                text = " ".join(filter(None, [
+                    h.get("title"), h.get("story_text"),
+                    h.get("comment_text"), h.get("url")]))
+                n += scan_text(cur, text, f"hn:{h.get('objectID')}")
+            time.sleep(1.2)
+        db.commit()
+        total += n
+        log(f"  '{q}': +{n} new")
+    return total
+
+
+def main():
+    log(f"=== hn eternal START pid={__import__('os').getpid()} "
+        f"queries={len(QUERIES)}")
+    try:
+        con = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+        cur = con.cursor()
+        cur.execute("SELECT val FROM keys WHERE found='hn'")
+        for r in cur.fetchall():
+            seen.add(r[0])
+        con.close()
+    except Exception:
+        pass
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+    while True:
+        try:
+            n = cycle(db)
+            log(f"CYCLE done +{n}")
+        except Exception as e:
+            log(f"cycle err: {repr(e)[:100]}")
+            time.sleep(30)
+        time.sleep(90)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(0)