keyhunter-parser/grepapp_farm.py

"""grepapp_farm.py — validate 446 grep.app dashscope hits.
Fetches each file from raw.githubusercontent.com, scans for keys, writes keys.db.
One-shot (not eternal). Run: python grepapp_farm.py
"""
import os, sys, json, time, hashlib, sqlite3, threading
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests

KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
sys.path.insert(0, KF_BASE)

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
INPUT_FILE = os.path.join(KF_BASE, "grepapp_farm_input.json")
LOG_FILE = os.path.join(KF_BASE, "grepapp_farm.log")
N_THREADS = 16
FETCH_TIMEOUT = 15
RATE_BURST = 8
RATE_PAUSE = 2.0

from key_patterns import PATTERNS

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
           "AQUI", "sk-xxxx", "sk-XXXX", "sk-your", "sk-YOUR", "sk-your-",
           "not_a_real", "REPLACE", "replace", "CHANGE", "change", "YOUR_KEY",
           "your_key", "your-key", "MY_KEY", "my_key", "PUT_YOUR", "put_your",
           "PASTE", "paste", "HERE", "here", "PLACEHOLDER", "placeholder",
           "SKIP_ME", "skip_me", "00000000", "12345678", "YOUR_SECRET",
           "your_secret", "*", "•", "xxxxxxxx", "DEADBEEF", "deadbeef")

session = requests.Session()
session.trust_env = False
session.headers["User-Agent"] = "grepapp-farm/1.0"

_burst_lock = threading.Lock()
_burst_count = 0
_burst_ts = time.time()


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}"
    print(line, flush=True)
    try:
        with open(LOG_FILE, "a", encoding="utf-8") as f:
            f.write(line + "\n")
    except:
        pass


def _rate_wait():
    global _burst_count, _burst_ts
    with _burst_lock:
        _burst_count += 1
        if _burst_count >= RATE_BURST:
            elapsed = time.time() - _burst_ts
            if elapsed < RATE_PAUSE:
                time.sleep(RATE_PAUSE - elapsed)
            _burst_count = 0
            _burst_ts = time.time()


def url_for(hit):
    repo = hit["repo"].strip()
    branch = hit["branch"].strip()
    path = hit["path"].strip()
    return f"https://raw.githubusercontent.com/{repo}/{branch}/{path}"


def fetch_and_scan(hit):
    _rate_wait()
    url = url_for(hit)
    repo = hit["repo"]
    try:
        r = session.get(url, timeout=FETCH_TIMEOUT, allow_redirects=True)
        if r.status_code != 200:
            return []
        text = r.text
    except Exception:
        return []

    found = []
    for prov, pat in PATTERNS:
        for m in pat.finditer(text):
            val = m.group(0)
            if len(val) < 6:
                continue
            if any(g in val for g in GARBAGE):
                continue
            digits = sum(c.isdigit() for c in val)
            if digits > len(val) * 0.8 and len(val) < 20:
                continue
            found.append((prov, val.strip()))
    return found


def write_keys(db_conn, hits_found):
    cur = db_conn.cursor()
    n = 0
    for prov, val, repo in hits_found:
        h = hashlib.sha256(val.encode()).hexdigest()
        try:
            cur.execute(
                "INSERT INTO keys (hash, val, prov, status, found, repo) "
                "VALUES (?,?,?,?,?,?) "
                "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
                "found=excluded.found, repo=excluded.repo "
                "WHERE keys.status IN ('DEAD','ERR')",
                (h, val, prov, "NEW", time.strftime("%Y-%m-%d %H:%M"), repo)
            )
            n += 1
        except Exception as e:
            log(f"DB write error: {e}")
    db_conn.commit()
    return n


def main():
    log("=== grepapp_farm START ===")

    if not os.path.exists(INPUT_FILE):
        log(f"ERROR: input file not found: {INPUT_FILE}")
        sys.exit(1)

    with open(INPUT_FILE, encoding="utf-8") as f:
        hits = json.load(f)
    log(f"Loaded {len(hits)} hits from {INPUT_FILE}")

    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
    db.execute("""CREATE TABLE IF NOT EXISTS keys (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
        plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
    db.commit()

    total_keys = 0
    done = 0
    batch = []

    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futures = {ex.submit(fetch_and_scan, h): h for h in hits}
        for f in as_completed(futures):
            hit = futures[f]
            done += 1
            try:
                results = f.result()
            except Exception as e:
                results = []
                log(f"ERR {hit['repo']}/{hit['path']}: {e}")

            for prov, val in results:
                batch.append((prov, val, hit["repo"]))
                total_keys += 1

            if len(batch) >= 20:
                n = write_keys(db, batch)
                if n:
                    log(f"  +{n} keys ({done}/{len(hits)} files)")
                batch = []

            if done % 50 == 0:
                log(f"Progress: {done}/{len(hits)} files, {total_keys} keys")

    if batch:
        n = write_keys(db, batch)
        if n:
            log(f"  +{n} keys (final flush)")

    db.close()
    log(f"=== grepapp_farm DONE: {done} files, {total_keys} keys ===")

    db2 = sqlite3.connect(DB)
    cur = db2.execute("SELECT prov, COUNT(*) FROM keys WHERE status='NEW' GROUP BY prov ORDER BY 2 DESC LIMIT 20")
    log("Top providers in DB:")
    for row in cur.fetchall():
        log(f"  {row[0]}: {row[1]}")
    db2.close()


if __name__ == "__main__":
    main()