KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
196
grepapp_eternal.py
Normal file
196
grepapp_eternal.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""ETERNAL grep.app hunt: public code-search API, independent rate lane.
|
||||
Cycles queries against https://grep.app/api/search, extracts keys, writes keys.db.
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "grepapp_eternal.log")
|
||||
API = "https://grep.app/api/search"
|
||||
REST = 300 # grep.app rate limits are aggressive
|
||||
TOKENS_FILE = os.environ.get("KF_TOKENS_FILE") or os.path.join(KF_BASE, "gh_tokens.txt")
|
||||
|
||||
# Token pool — passive for grep.app (no auth enforced), rotates as fallback
|
||||
class TokenPool:
|
||||
def __init__(self):
|
||||
self.tokens = []
|
||||
self._i = 0
|
||||
self._lock = threading.Lock()
|
||||
self._load()
|
||||
|
||||
def _load(self):
|
||||
try:
|
||||
with open(TOKENS_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
t = line.strip()
|
||||
if t:
|
||||
self.tokens.append(t)
|
||||
except Exception:
|
||||
pass
|
||||
log(f"TokenPool: {len(self.tokens)} tokens from {TOKENS_FILE}")
|
||||
|
||||
def next(self):
|
||||
with self._lock:
|
||||
if not self.tokens:
|
||||
return ""
|
||||
t = self.tokens[self._i % len(self.tokens)]
|
||||
self._i += 1
|
||||
return t
|
||||
|
||||
|
||||
def _backoff_sleep(attempt):
|
||||
"""Exponential backoff + jitter. 2^attempt * 10-30s, capped 600s."""
|
||||
delay = min(600, (2 ** attempt) * random.uniform(10, 30))
|
||||
log(f"backoff #{attempt}: sleep {delay:.0f}s")
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
QUERIES = [
|
||||
"ANTHROPIC_API_KEY",
|
||||
"sk-ant-api03",
|
||||
"OPENAI_API_KEY",
|
||||
"sk-or-v1",
|
||||
"OPENROUTER_API_KEY",
|
||||
"DASHSCOPE_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"ZHIPU_API_KEY",
|
||||
"ZHIPUAI_API_KEY",
|
||||
"GLM_API_KEY",
|
||||
"KIMI_API_KEY",
|
||||
"MOONSHOT_API_KEY",
|
||||
"REPLICATE_API_TOKEN",
|
||||
"ELEVENLABS_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"TOGETHER_API_KEY",
|
||||
"FIREWORKS_API_KEY",
|
||||
"PERPLEXITY_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"HF_TOKEN",
|
||||
"HUGGINGFACE_API_TOKEN",
|
||||
"SILICONFLOW_API_KEY",
|
||||
"TELEGRAM_BOT_TOKEN",
|
||||
"sk_c434e9eed6b6",
|
||||
"NOVITA_API_KEY",
|
||||
"STABILITY_API_KEY",
|
||||
"NVIDIA_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"COHERE_API_KEY",
|
||||
"TOGETHER_API_TOKEN",
|
||||
"tgp_",
|
||||
"pplx-",
|
||||
"nvapi-",
|
||||
"gsk_",
|
||||
"QWEN_API_KEY",
|
||||
"TONGYI_QIANWEN_API_KEY",
|
||||
"DASHSCOPE_API_TOKEN",
|
||||
"api.dashscope.cn",
|
||||
]
|
||||
|
||||
from key_patterns import PATTERNS
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
|
||||
"fake", "test", "Test", "TEST", "demo", "Demo", "aaaaaaaa", "abcdef",
|
||||
"1234567890", "abcdefghij", "placehol", "Placehol", "changeme",
|
||||
"deadbeef", "fffffff", "0000000000", "INSERT", "REEMPLAZA", "AQUI")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
TOKEN_POOL = TokenPool()
|
||||
|
||||
|
||||
def search(q, page=0):
|
||||
for attempt in range(10):
|
||||
try:
|
||||
r = requests.get(API, params={"q": q, "page": page},
|
||||
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Referer": "https://grep.app/",
|
||||
"Origin": "https://grep.app"}, timeout=30)
|
||||
if r.status_code == 429:
|
||||
log(f" {q!r}: HTTP 429 (attempt {attempt+1}/10)")
|
||||
_backoff_sleep(attempt)
|
||||
if attempt >= 3:
|
||||
TOKEN_POOL.next() # rotate on repeated 429
|
||||
continue
|
||||
if r.status_code != 200:
|
||||
log(f" {q!r}: HTTP {r.status_code}")
|
||||
return []
|
||||
j = r.json()
|
||||
return j.get("hits", {}).get("hits", [])
|
||||
except Exception as e:
|
||||
log(f" {q!r} error (attempt {attempt+1}/10): {e}")
|
||||
_backoff_sleep(attempt)
|
||||
log(f" {q!r}: all retries exhausted")
|
||||
return []
|
||||
|
||||
|
||||
def cycle(db, seen):
|
||||
cur = db.cursor()
|
||||
new_total, by_prov = 0, {}
|
||||
for q in QUERIES:
|
||||
for page in (0,):
|
||||
hits = search(q, page)
|
||||
qn = 0
|
||||
for h in hits:
|
||||
snip = h.get("content", {}).get("snippet", "") or ""
|
||||
repo = h.get("repo", {}).get("raw", "unknown")
|
||||
path = h.get("path", {}).get("raw", "")
|
||||
for prov, rx in PATTERNS:
|
||||
for key in rx.findall(snip):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if key in seen or any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
seen.add(key)
|
||||
cur.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
||||
"NEW", "grepapp", f"{repo}/{path}"))
|
||||
new_total += 1
|
||||
qn += 1
|
||||
by_prov[prov] = by_prov.get(prov, 0) + 1
|
||||
log(f"{q!r} p{page}: {len(hits)} hits, +{qn} new")
|
||||
time.sleep(20)
|
||||
db.commit()
|
||||
return new_total, by_prov
|
||||
|
||||
|
||||
def main():
|
||||
log(f"=== grep.app eternal hunt START pid={__import__('os').getpid()} ===")
|
||||
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
||||
while True:
|
||||
try:
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT val FROM keys")
|
||||
seen = {r[0] for r in cur.fetchall()}
|
||||
n, prov = cycle(db, seen)
|
||||
log(f"CYCLE done: +{n} new {prov}; resting {REST}s")
|
||||
except Exception as e:
|
||||
log(f"CYCLE ERROR: {e!r}")
|
||||
time.sleep(REST)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user