KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
269
gh_realtime_eternal.py
Normal file
269
gh_realtime_eternal.py
Normal file
@@ -0,0 +1,269 @@
|
||||
"""ETERNAL gh_realtime: GitHub public Events firehose — fresh pushes scanned
|
||||
within seconds of landing (keys get revoked fast; realtime beats dork crawls).
|
||||
|
||||
Endpoint(s): https://api.github.com/events (probed 2026-10-06: 200 unauth),
|
||||
https://api.github.com/repos/{o}/{r}/commits/{sha} (diff, capped)
|
||||
Auth: none | PAT pool (gh_tokens.txt) for 5000/h instead of 60/h
|
||||
"""
|
||||
import os
|
||||
BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(BASE, "data", "keys.db")
|
||||
LOG = os.path.join(BASE, "data", "gh_realtime_eternal.log")
|
||||
CURSOR_FILE = os.path.join(BASE, "data", "gh_realtime_cursor.txt")
|
||||
SEEN_FILE = os.path.join(BASE, "data", "gh_realtime_seen.txt")
|
||||
TOKENS_FILE = os.path.join(BASE, "gh_tokens.txt")
|
||||
N_THREADS = 4 # commit fetches in parallel
|
||||
SLEEP = 75 # events page rotates fast; poll just under a minute
|
||||
MAX_ITEMS = 40 # commits fetched per cycle
|
||||
DIFF_CAP = 400_000 # bytes per commit diff
|
||||
SEEN_CAP = 20_000
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
INTERESTING = re.compile(
|
||||
r'(\.env|\.ya?ml|\.json|\.toml|\.ini|\.cfg|\.conf|\.properties'
|
||||
r'|config|secret|credential|setting|key|token|deploy|docker-compose'
|
||||
r'|application|settings|\.py|\.js|\.ts|\.sh)\b', re.I)
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "placeholder",
|
||||
"CHANGEME", "changeme", "dummy", "DUMMY", "abcdefghijklmnopqrst",
|
||||
"1234567890abcdef")
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False # LANE_CONTRACT §4: local proxy kills long requests
|
||||
lock = threading.Lock()
|
||||
seen = set()
|
||||
stats = {"events": 0, "commits": 0, "keys": 0, "fails": 0}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
try:
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def load_tokens():
|
||||
toks = []
|
||||
try:
|
||||
with open(TOKENS_FILE, encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
t = line.strip()
|
||||
if t.startswith(("ghp_", "github_pat_")) and t not in toks:
|
||||
toks.append(t)
|
||||
except OSError:
|
||||
pass
|
||||
return toks
|
||||
|
||||
|
||||
TOKENS = load_tokens()
|
||||
_tok_i = [0]
|
||||
|
||||
|
||||
def auth_headers():
|
||||
h = {"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "keyfarm-realtime",
|
||||
"X-GitHub-Api-Version": "2022-11-28"}
|
||||
with lock:
|
||||
if TOKENS:
|
||||
t = TOKENS[_tok_i[0] % len(TOKENS)]
|
||||
_tok_i[0] += 1
|
||||
h["Authorization"] = "Bearer " + t
|
||||
return h
|
||||
|
||||
|
||||
def get(url, accept=None, timeout=25):
|
||||
h = auth_headers()
|
||||
if accept:
|
||||
h["Accept"] = accept
|
||||
for attempt, wait in enumerate((2, 4, 8)):
|
||||
try:
|
||||
r = S.get(url, headers=h, timeout=timeout)
|
||||
if r.status_code in (403, 429):
|
||||
ra = int(r.headers.get("Retry-After", "0") or 0)
|
||||
time.sleep(max(ra, wait))
|
||||
continue
|
||||
if r.status_code >= 500 and attempt < 2:
|
||||
time.sleep(wait)
|
||||
continue
|
||||
return r
|
||||
except requests.RequestException:
|
||||
if attempt < 2:
|
||||
time.sleep(wait)
|
||||
else:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
return open(CURSOR_FILE, encoding="utf-8").read().strip()
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def save_cursor(event_id):
|
||||
tmp = CURSOR_FILE + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.write(str(event_id))
|
||||
os.replace(tmp, CURSOR_FILE)
|
||||
|
||||
|
||||
def load_seen():
|
||||
global seen
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
seen = {l.strip() for l in f if l.strip()}
|
||||
except OSError:
|
||||
seen = set()
|
||||
|
||||
|
||||
def save_seen():
|
||||
items = list(seen)[-SEEN_CAP:]
|
||||
tmp = SEEN_FILE + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(items))
|
||||
os.replace(tmp, SEEN_FILE)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
|
||||
plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
if any(g in key for g in GARBAGE):
|
||||
return 0
|
||||
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
||||
with lock:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(h, key, prov, "NEW", "ghrealtime", repo[:300]))
|
||||
conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
def scan_diff(text, repo):
|
||||
"""Scan only added lines of a unified diff."""
|
||||
n = 0
|
||||
added = [l[1:] for l in (text or "").splitlines()
|
||||
if l.startswith("+") and not l.startswith("+++")]
|
||||
for prov, key in scan_text("\n".join(added)):
|
||||
if len(key) < 20 or len(set(key)) < 6:
|
||||
continue
|
||||
n += db_add(key, prov, repo)
|
||||
return n
|
||||
|
||||
|
||||
def fetch_commit(repo, sha):
|
||||
if sha in seen:
|
||||
return 0
|
||||
with lock:
|
||||
seen.add(sha)
|
||||
r = get(f"https://api.github.com/repos/{repo}/commits/{sha}",
|
||||
accept="application/vnd.github.diff")
|
||||
if r is None or r.status_code != 200:
|
||||
with lock:
|
||||
stats["fails"] += 1
|
||||
return 0
|
||||
body = r.text[:DIFF_CAP]
|
||||
with lock:
|
||||
stats["commits"] += 1
|
||||
return scan_diff(body, repo)
|
||||
|
||||
|
||||
def cycle(max_items):
|
||||
r = get("https://api.github.com/events?per_page=100")
|
||||
if r is None or r.status_code != 200:
|
||||
log(f"events fetch failed: {r.status_code if r is not None else 'net'}")
|
||||
return 0
|
||||
events = r.json()
|
||||
with lock:
|
||||
stats["events"] += len(events)
|
||||
cursor = load_cursor()
|
||||
jobs = [] # (repo, sha), interesting-file pushes first
|
||||
push_events = 0
|
||||
newest = cursor
|
||||
for ev in events:
|
||||
eid = str(ev.get("id", ""))
|
||||
if cursor and eid and int(eid) <= int(cursor or 0):
|
||||
continue
|
||||
if eid and (not newest or int(eid) > int(newest or 0)):
|
||||
newest = eid
|
||||
repo = (ev.get("repo") or {}).get("name", "")
|
||||
pl = ev.get("payload") or {}
|
||||
if ev.get("type") != "PushEvent":
|
||||
continue
|
||||
push_events += 1
|
||||
cand = [] # (priority, sha)
|
||||
for c in (pl.get("commits") or [])[:3]:
|
||||
sha = c.get("id", "")
|
||||
if not sha:
|
||||
continue
|
||||
files = " ".join((c.get("modified") or []) + (c.get("added") or []))
|
||||
interesting = (not files) or bool(INTERESTING.search(files))
|
||||
cand.append((0 if interesting else 1, sha))
|
||||
if not cand and pl.get("head"):
|
||||
cand.append((1, pl["head"])) # big push: commits[] truncated
|
||||
cand.sort()
|
||||
jobs += [(repo, sha) for _, sha in cand]
|
||||
with lock:
|
||||
stats["push_events"] = stats.get("push_events", 0) + push_events
|
||||
stats["jobs_queued"] = stats.get("jobs_queued", 0) + len(jobs)
|
||||
jobs = jobs[:max_items]
|
||||
found = 0
|
||||
if jobs:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
for n in ex.map(lambda j: fetch_commit(*j), jobs):
|
||||
found += n
|
||||
if newest:
|
||||
save_cursor(newest)
|
||||
return found
|
||||
|
||||
|
||||
def main():
|
||||
once = "--once" in sys.argv
|
||||
load_seen()
|
||||
log(f"=== gh_realtime START tokens={len(TOKENS)} once={once} ===")
|
||||
cycles = 0
|
||||
while True:
|
||||
try:
|
||||
budget = min(MAX_ITEMS, 5) if once else MAX_ITEMS
|
||||
n = cycle(budget)
|
||||
cycles += 1
|
||||
with lock:
|
||||
stats["keys"] += n
|
||||
snap = dict(stats)
|
||||
log(f"cycle {cycles}: +{n} keys | {snap}")
|
||||
save_seen()
|
||||
except Exception as e:
|
||||
log(f"cycle error: {e!r}")
|
||||
if once:
|
||||
print(f"gh_realtime --once done: {stats}")
|
||||
return
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user