KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

grepapp_farm.py

@@ -0,0 +1,181 @@
+"""grepapp_farm.py — validate 446 grep.app dashscope hits.
+Fetches each file from raw.githubusercontent.com, scans for keys, writes keys.db.
+One-shot (not eternal). Run: python grepapp_farm.py
+"""
+import os, sys, json, time, hashlib, sqlite3, threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+
+KF_BASE = os.path.dirname(os.path.abspath(__file__))
+os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
+sys.path.insert(0, KF_BASE)
+
+DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
+INPUT_FILE = os.path.join(KF_BASE, "grepapp_farm_input.json")
+LOG_FILE = os.path.join(KF_BASE, "grepapp_farm.log")
+N_THREADS = 16
+FETCH_TIMEOUT = 15
+RATE_BURST = 8
+RATE_PAUSE = 2.0
+
+from key_patterns import PATTERNS
+
+GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
+           "AQUI", "sk-xxxx", "sk-XXXX", "sk-your", "sk-YOUR", "sk-your-",
+           "not_a_real", "REPLACE", "replace", "CHANGE", "change", "YOUR_KEY",
+           "your_key", "your-key", "MY_KEY", "my_key", "PUT_YOUR", "put_your",
+           "PASTE", "paste", "HERE", "here", "PLACEHOLDER", "placeholder",
+           "SKIP_ME", "skip_me", "00000000", "12345678", "YOUR_SECRET",
+           "your_secret", "*", "•", "xxxxxxxx", "DEADBEEF", "deadbeef")
+
+session = requests.Session()
+session.trust_env = False
+session.headers["User-Agent"] = "grepapp-farm/1.0"
+
+_burst_lock = threading.Lock()
+_burst_count = 0
+_burst_ts = time.time()
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}"
+    print(line, flush=True)
+    try:
+        with open(LOG_FILE, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+    except:
+        pass
+
+
+def _rate_wait():
+    global _burst_count, _burst_ts
+    with _burst_lock:
+        _burst_count += 1
+        if _burst_count >= RATE_BURST:
+            elapsed = time.time() - _burst_ts
+            if elapsed < RATE_PAUSE:
+                time.sleep(RATE_PAUSE - elapsed)
+            _burst_count = 0
+            _burst_ts = time.time()
+
+
+def url_for(hit):
+    repo = hit["repo"].strip()
+    branch = hit["branch"].strip()
+    path = hit["path"].strip()
+    return f"https://raw.githubusercontent.com/{repo}/{branch}/{path}"
+
+
+def fetch_and_scan(hit):
+    _rate_wait()
+    url = url_for(hit)
+    repo = hit["repo"]
+    try:
+        r = session.get(url, timeout=FETCH_TIMEOUT, allow_redirects=True)
+        if r.status_code != 200:
+            return []
+        text = r.text
+    except Exception:
+        return []
+
+    found = []
+    for prov, pat in PATTERNS:
+        for m in pat.finditer(text):
+            val = m.group(0)
+            if len(val) < 6:
+                continue
+            if any(g in val for g in GARBAGE):
+                continue
+            digits = sum(c.isdigit() for c in val)
+            if digits > len(val) * 0.8 and len(val) < 20:
+                continue
+            found.append((prov, val.strip()))
+    return found
+
+
+def write_keys(db_conn, hits_found):
+    cur = db_conn.cursor()
+    n = 0
+    for prov, val, repo in hits_found:
+        h = hashlib.sha256(val.encode()).hexdigest()
+        try:
+            cur.execute(
+                "INSERT INTO keys (hash, val, prov, status, found, repo) "
+                "VALUES (?,?,?,?,?,?) "
+                "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
+                "found=excluded.found, repo=excluded.repo "
+                "WHERE keys.status IN ('DEAD','ERR')",
+                (h, val, prov, "NEW", time.strftime("%Y-%m-%d %H:%M"), repo)
+            )
+            n += 1
+        except Exception as e:
+            log(f"DB write error: {e}")
+    db_conn.commit()
+    return n
+
+
+def main():
+    log("=== grepapp_farm START ===")
+
+    if not os.path.exists(INPUT_FILE):
+        log(f"ERROR: input file not found: {INPUT_FILE}")
+        sys.exit(1)
+
+    with open(INPUT_FILE, encoding="utf-8") as f:
+        hits = json.load(f)
+    log(f"Loaded {len(hits)} hits from {INPUT_FILE}")
+
+    db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
+    db.execute("""CREATE TABLE IF NOT EXISTS keys (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
+        plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
+    db.commit()
+
+    total_keys = 0
+    done = 0
+    batch = []
+
+    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
+        futures = {ex.submit(fetch_and_scan, h): h for h in hits}
+        for f in as_completed(futures):
+            hit = futures[f]
+            done += 1
+            try:
+                results = f.result()
+            except Exception as e:
+                results = []
+                log(f"ERR {hit['repo']}/{hit['path']}: {e}")
+
+            for prov, val in results:
+                batch.append((prov, val, hit["repo"]))
+                total_keys += 1
+
+            if len(batch) >= 20:
+                n = write_keys(db, batch)
+                if n:
+                    log(f"  +{n} keys ({done}/{len(hits)} files)")
+                batch = []
+
+            if done % 50 == 0:
+                log(f"Progress: {done}/{len(hits)} files, {total_keys} keys")
+
+    if batch:
+        n = write_keys(db, batch)
+        if n:
+            log(f"  +{n} keys (final flush)")
+
+    db.close()
+    log(f"=== grepapp_farm DONE: {done} files, {total_keys} keys ===")
+
+    db2 = sqlite3.connect(DB)
+    cur = db2.execute("SELECT prov, COUNT(*) FROM keys WHERE status='NEW' GROUP BY prov ORDER BY 2 DESC LIMIT 20")
+    log("Top providers in DB:")
+    for row in cur.fetchall():
+        log(f"  {row[0]}: {row[1]}")
+    db2.close()
+
+
+if __name__ == "__main__":
+    main()