keyhunter-parser/codeberg_eternal.py

"""Codeberg eternal lane — /api/v1/repos/search?sort=updated, tree walk,
raw fetch interesting files, regex keys → keys.db."""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import re
import sqlite3
import threading
import time
import hashlib
import requests
from requests.adapters import HTTPAdapter, Retry

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data", "keys.db")
REST = 300
WORKERS = 6
PAGES_PER_CYCLE = 30  # 30*50 = 1500 repos/cycle
SEARCHES = [
    "TOGETHER_API_KEY",
    "FIREWORKS_API_KEY",
    "NOVITA_API_KEY",
    "PERPLEXITY_API_KEY",
    "STABILITY_API_KEY",
    "GROQ_API_KEY",
    "NVIDIA_API_KEY",
    "MISTRAL_API_KEY",
    "COHERE_API_KEY",
    "REPLICATE_API_TOKEN",
    "ELEVENLABS_API_KEY",
    "HF_TOKEN",
    "TOGETHER_API_TOKEN",
    "DASHSCOPE_API_KEY",
    "QWEN_API_KEY",
    "TONGYI_QIANWEN_API_KEY",
    "DASHSCOPE_API_TOKEN",
]
_KW = [0]
API = "https://codeberg.org/api/v1"

S = requests.Session()
S.trust_env = False
S.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128"
S.mount("https://", HTTPAdapter(max_retries=Retry(
    total=3, backoff_factor=1.5, status_forcelist=[429, 500, 502, 503])))

INTERESTING = re.compile(
    r'\.env|(^|/)(env|config|settings|credentials|secrets?|tokens?|keys?|auth)'
    r'|\.(ini|cfg|properties|toml|ya?ml|json)$'
    r'|\.bashrc|\.profile|\.netrc', re.I)

from key_patterns import PATTERNS

GARBAGE = re.compile(r'^(pass|test|example|sample|dummy|fake|your|xxx|abc|123|key|token|secret|apikey|placeholder)$', re.I)

db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
db.execute("PRAGMA busy_timeout=120000")
DB_LOCK = threading.Lock()


def log(msg):
    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)


def seen_vals():
    cur = db.cursor()
    cur.execute("SELECT val FROM keys")
    return {r[0] for r in cur.fetchall()}


def insert_key(key, prov, repo):
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    with DB_LOCK:
        try:
            db.execute(
                "INSERT INTO keys (hash, val, prov, status, found, repo) "
                "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
                (h, key, prov, "NEW", "codeberg-eternal", repo))
            db.commit()
        except sqlite3.Error:
            pass


def scan_repo(full_name, branch, seen):
    n = 0
    try:
        r = S.get(f"{API}/repos/{full_name}/git/trees/{branch}",
                  params={"recursive": "true"}, timeout=30)
        if r.status_code != 200:
            return 0
        tree = r.json().get("tree", [])[:500]
    except Exception:
        return 0
    for ent in tree:
        if ent.get("type") != "blob":
            continue
        path = ent.get("path", "")
        if not INTERESTING.search(path) or ent.get("size", 0) > 200_000:
            continue
        try:
            raw = S.get(f"{API}/repos/{full_name}/raw/{path}",
                        params={"ref": branch}, timeout=25)
            if raw.status_code != 200 or len(raw.text) > 300_000:
                continue
        except Exception:
            continue
        for prov, rx in PATTERNS:
            for key in rx.findall(raw.text):
                if isinstance(key, tuple):
                    key = key[0]
                key = key.strip().strip('"\'`.,:')
                if not key or key in seen or GARBAGE.match(key):
                    continue
                insert_key(key, prov, full_name)
                seen.add(key)
                n += 1
                log(f"  +{prov} {key[:12]}… {full_name}")
    return n


def cycle(seen, kw=None):
    total = 0
    lock = threading.Lock()
    idx = [0]

    def fetch_page():
        page = idx[0]
        idx[0] += 1
        params = {"limit": 50, "page": page}
        if kw:
            params["q"] = kw
        else:
            params["sort"] = "updated"
            params["order"] = "desc"
        try:
            r = S.get(f"{API}/repos/search", params=params, timeout=30)
            if r.status_code != 200:
                return []
            return [(v["full_name"], v.get("default_branch") or "main")
                    for v in r.json().get("data", [])
                    if v.get("full_name") and not v.get("archived")]
        except Exception:
            return []

    def work():
        nonlocal total
        while True:
            with lock:
                if idx[0] >= PAGES_PER_CYCLE:
                    return
                repos = fetch_page()
            if not repos:
                return
            for fn, br in repos:
                total += scan_repo(fn, br, seen)
                time.sleep(0.4)

    ths = [threading.Thread(target=work, daemon=True) for _ in range(WORKERS)]
    for t in ths:
        t.start()
    for t in ths:
        t.join()
    return total

def main():
    log(f"=== CODEBERG eternal START threads={WORKERS} ===")
    while True:
        try:
            seen = seen_vals()
            kw = SEARCHES[_KW[0] % len(SEARCHES)]
            _KW[0] += 1
            n = cycle(seen, kw=kw)
            log(f"CYCLE done (kw={kw!r}): +{n} new; resting {REST}s")
        except Exception as e:
            log(f"CYCLE error: {e}; resting {REST}s")
        time.sleep(REST)


if __name__ == "__main__":
    main()