keyhunter-parser/grepapp_eternal.py

"""ETERNAL grep.app hunt: public code-search API, independent rate lane.
Cycles queries against https://grep.app/api/search, extracts keys, writes keys.db.
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import random
import re
import sqlite3
import sys
import threading
import time

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "grepapp_eternal.log")
API = "https://grep.app/api/search"
REST = 300  # grep.app rate limits are aggressive
TOKENS_FILE = os.environ.get("KF_TOKENS_FILE") or os.path.join(KF_BASE, "gh_tokens.txt")

# Token pool — passive for grep.app (no auth enforced), rotates as fallback
class TokenPool:
    def __init__(self):
        self.tokens = []
        self._i = 0
        self._lock = threading.Lock()
        self._load()

    def _load(self):
        try:
            with open(TOKENS_FILE, encoding="utf-8") as f:
                for line in f:
                    t = line.strip()
                    if t:
                        self.tokens.append(t)
        except Exception:
            pass
        log(f"TokenPool: {len(self.tokens)} tokens from {TOKENS_FILE}")

    def next(self):
        with self._lock:
            if not self.tokens:
                return ""
            t = self.tokens[self._i % len(self.tokens)]
            self._i += 1
            return t


def _backoff_sleep(attempt):
    """Exponential backoff + jitter. 2^attempt * 10-30s, capped 600s."""
    delay = min(600, (2 ** attempt) * random.uniform(10, 30))
    log(f"backoff #{attempt}: sleep {delay:.0f}s")
    time.sleep(delay)


QUERIES = [
    "ANTHROPIC_API_KEY",
    "sk-ant-api03",
    "OPENAI_API_KEY",
    "sk-or-v1",
    "OPENROUTER_API_KEY",
    "DASHSCOPE_API_KEY",
    "DEEPSEEK_API_KEY",
    "ZHIPU_API_KEY",
    "ZHIPUAI_API_KEY",
    "GLM_API_KEY",
    "KIMI_API_KEY",
    "MOONSHOT_API_KEY",
    "REPLICATE_API_TOKEN",
    "ELEVENLABS_API_KEY",
    "GROQ_API_KEY",
    "TOGETHER_API_KEY",
    "FIREWORKS_API_KEY",
    "PERPLEXITY_API_KEY",
    "GEMINI_API_KEY",
    "HF_TOKEN",
    "HUGGINGFACE_API_TOKEN",
    "SILICONFLOW_API_KEY",
    "TELEGRAM_BOT_TOKEN",
    "sk_c434e9eed6b6",
    "NOVITA_API_KEY",
    "STABILITY_API_KEY",
    "NVIDIA_API_KEY",
    "MISTRAL_API_KEY",
    "COHERE_API_KEY",
    "TOGETHER_API_TOKEN",
    "tgp_",
    "pplx-",
    "nvapi-",
    "gsk_",
    "QWEN_API_KEY",
    "TONGYI_QIANWEN_API_KEY",
    "DASHSCOPE_API_TOKEN",
    "api.dashscope.cn",
]

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
           "fake", "test", "Test", "TEST", "demo", "Demo", "aaaaaaaa", "abcdef",
           "1234567890", "abcdefghij", "placehol", "Placehol", "changeme",
           "deadbeef", "fffffff", "0000000000", "INSERT", "REEMPLAZA", "AQUI")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


TOKEN_POOL = TokenPool()


def search(q, page=0):
    for attempt in range(10):
        try:
            r = requests.get(API, params={"q": q, "page": page},
                             headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
                             "Accept": "application/json, text/plain, */*",
                             "Referer": "https://grep.app/",
                             "Origin": "https://grep.app"}, timeout=30)
            if r.status_code == 429:
                log(f"  {q!r}: HTTP 429 (attempt {attempt+1}/10)")
                _backoff_sleep(attempt)
                if attempt >= 3:
                    TOKEN_POOL.next()  # rotate on repeated 429
                continue
            if r.status_code != 200:
                log(f"  {q!r}: HTTP {r.status_code}")
                return []
            j = r.json()
            return j.get("hits", {}).get("hits", [])
        except Exception as e:
            log(f"  {q!r} error (attempt {attempt+1}/10): {e}")
            _backoff_sleep(attempt)
    log(f"  {q!r}: all retries exhausted")
    return []


def cycle(db, seen):
    cur = db.cursor()
    new_total, by_prov = 0, {}
    for q in QUERIES:
        for page in (0,):
            hits = search(q, page)
            qn = 0
            for h in hits:
                snip = h.get("content", {}).get("snippet", "") or ""
                repo = h.get("repo", {}).get("raw", "unknown")
                path = h.get("path", {}).get("raw", "")
                for prov, rx in PATTERNS:
                    for key in rx.findall(snip):
                        if isinstance(key, tuple):
                            key = key[0]
                        if key in seen or any(g in key for g in GARBAGE) or len(key) < 30:
                            continue
                        seen.add(key)
                        cur.execute(
                            "INSERT INTO keys (hash, val, prov, status, found, repo) "
                            "VALUES (?,?,?,?,?,?) "
        "ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
        "WHERE keys.status IN ('DEAD','ERR')",
                            (hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
                             "NEW", "grepapp", f"{repo}/{path}"))
                        new_total += 1
                        qn += 1
                        by_prov[prov] = by_prov.get(prov, 0) + 1
            log(f"{q!r} p{page}: {len(hits)} hits, +{qn} new")
            time.sleep(20)
    db.commit()
    return new_total, by_prov


def main():
    log(f"=== grep.app eternal hunt START pid={__import__('os').getpid()} ===")
    db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
    while True:
        try:
            cur = db.cursor()
            cur.execute("SELECT val FROM keys")
            seen = {r[0] for r in cur.fetchall()}
            n, prov = cycle(db, seen)
            log(f"CYCLE done: +{n} new {prov}; resting {REST}s")
        except Exception as e:
            log(f"CYCLE ERROR: {e!r}")
        time.sleep(REST)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(0)