"""ETERNAL gh_realtime: GitHub public Events firehose — fresh pushes scanned
within seconds of landing (keys get revoked fast; realtime beats dork crawls).

Endpoint(s): https://api.github.com/events (probed 2026-10-06: 200 unauth),
             https://api.github.com/repos/{o}/{r}/commits/{sha} (diff, capped)
Auth: none | PAT pool (gh_tokens.txt) for 5000/h instead of 60/h
"""
import os
BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(BASE, "data"), exist_ok=True)

import hashlib
import json
import re
import sqlite3
import sys
import threading
import time

import requests

DB = os.environ.get("KF_DB") or os.path.join(BASE, "data", "keys.db")
LOG = os.path.join(BASE, "data", "gh_realtime_eternal.log")
CURSOR_FILE = os.path.join(BASE, "data", "gh_realtime_cursor.txt")
SEEN_FILE = os.path.join(BASE, "data", "gh_realtime_seen.txt")
TOKENS_FILE = os.path.join(BASE, "gh_tokens.txt")
N_THREADS = 4            # commit fetches in parallel
SLEEP = 75               # events page rotates fast; poll just under a minute
MAX_ITEMS = 40           # commits fetched per cycle
DIFF_CAP = 400_000       # bytes per commit diff
SEEN_CAP = 20_000

from key_patterns import scan_text

INTERESTING = re.compile(
    r'(\.env|\.ya?ml|\.json|\.toml|\.ini|\.cfg|\.conf|\.properties'
    r'|config|secret|credential|setting|key|token|deploy|docker-compose'
    r'|application|settings|\.py|\.js|\.ts|\.sh)\b', re.I)
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "placeholder",
           "CHANGEME", "changeme", "dummy", "DUMMY", "abcdefghijklmnopqrst",
           "1234567890abcdef")

S = requests.Session()
S.trust_env = False      # LANE_CONTRACT §4: local proxy kills long requests
lock = threading.Lock()
seen = set()
stats = {"events": 0, "commits": 0, "keys": 0, "fails": 0}


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    try:
        with open(LOG, "a", encoding="utf-8", errors="replace") as f:
            f.write(line)
    except OSError:
        pass


def load_tokens():
    toks = []
    try:
        with open(TOKENS_FILE, encoding="utf-8", errors="replace") as f:
            for line in f:
                t = line.strip()
                if t.startswith(("ghp_", "github_pat_")) and t not in toks:
                    toks.append(t)
    except OSError:
        pass
    return toks


TOKENS = load_tokens()
_tok_i = [0]


def auth_headers():
    h = {"Accept": "application/vnd.github+json",
         "User-Agent": "keyfarm-realtime",
         "X-GitHub-Api-Version": "2022-11-28"}
    with lock:
        if TOKENS:
            t = TOKENS[_tok_i[0] % len(TOKENS)]
            _tok_i[0] += 1
            h["Authorization"] = "Bearer " + t
    return h


def get(url, accept=None, timeout=25):
    h = auth_headers()
    if accept:
        h["Accept"] = accept
    for attempt, wait in enumerate((2, 4, 8)):
        try:
            r = S.get(url, headers=h, timeout=timeout)
            if r.status_code in (403, 429):
                ra = int(r.headers.get("Retry-After", "0") or 0)
                time.sleep(max(ra, wait))
                continue
            if r.status_code >= 500 and attempt < 2:
                time.sleep(wait)
                continue
            return r
        except requests.RequestException:
            if attempt < 2:
                time.sleep(wait)
            else:
                return None
    return None


def load_cursor():
    try:
        return open(CURSOR_FILE, encoding="utf-8").read().strip()
    except OSError:
        return ""


def save_cursor(event_id):
    tmp = CURSOR_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        f.write(str(event_id))
    os.replace(tmp, CURSOR_FILE)


def load_seen():
    global seen
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            seen = {l.strip() for l in f if l.strip()}
    except OSError:
        seen = set()


def save_seen():
    items = list(seen)[-SEEN_CAP:]
    tmp = SEEN_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        f.write("\n".join(items))
    os.replace(tmp, SEEN_FILE)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False)
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
    plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
conn.commit()


def db_add(key, prov, repo):
    if any(g in key for g in GARBAGE):
        return 0
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    with lock:
        cur = conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (h, key, prov, "NEW", "ghrealtime", repo[:300]))
        conn.commit()
        return cur.rowcount


def scan_diff(text, repo):
    """Scan only added lines of a unified diff."""
    n = 0
    added = [l[1:] for l in (text or "").splitlines()
             if l.startswith("+") and not l.startswith("+++")]
    for prov, key in scan_text("\n".join(added)):
        if len(key) < 20 or len(set(key)) < 6:
            continue
        n += db_add(key, prov, repo)
    return n


def fetch_commit(repo, sha):
    if sha in seen:
        return 0
    with lock:
        seen.add(sha)
    r = get(f"https://api.github.com/repos/{repo}/commits/{sha}",
            accept="application/vnd.github.diff")
    if r is None or r.status_code != 200:
        with lock:
            stats["fails"] += 1
        return 0
    body = r.text[:DIFF_CAP]
    with lock:
        stats["commits"] += 1
    return scan_diff(body, repo)


def cycle(max_items):
    r = get("https://api.github.com/events?per_page=100")
    if r is None or r.status_code != 200:
        log(f"events fetch failed: {r.status_code if r is not None else 'net'}")
        return 0
    events = r.json()
    with lock:
        stats["events"] += len(events)
    cursor = load_cursor()
    jobs = []          # (repo, sha), interesting-file pushes first
    push_events = 0
    newest = cursor
    for ev in events:
        eid = str(ev.get("id", ""))
        if cursor and eid and int(eid) <= int(cursor or 0):
            continue
        if eid and (not newest or int(eid) > int(newest or 0)):
            newest = eid
        repo = (ev.get("repo") or {}).get("name", "")
        pl = ev.get("payload") or {}
        if ev.get("type") != "PushEvent":
            continue
        push_events += 1
        cand = []      # (priority, sha)
        for c in (pl.get("commits") or [])[:3]:
            sha = c.get("id", "")
            if not sha:
                continue
            files = " ".join((c.get("modified") or []) + (c.get("added") or []))
            interesting = (not files) or bool(INTERESTING.search(files))
            cand.append((0 if interesting else 1, sha))
        if not cand and pl.get("head"):
            cand.append((1, pl["head"]))   # big push: commits[] truncated
        cand.sort()
        jobs += [(repo, sha) for _, sha in cand]
    with lock:
        stats["push_events"] = stats.get("push_events", 0) + push_events
        stats["jobs_queued"] = stats.get("jobs_queued", 0) + len(jobs)
    jobs = jobs[:max_items]
    found = 0
    if jobs:
        from concurrent.futures import ThreadPoolExecutor
        with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
            for n in ex.map(lambda j: fetch_commit(*j), jobs):
                found += n
    if newest:
        save_cursor(newest)
    return found


def main():
    once = "--once" in sys.argv
    load_seen()
    log(f"=== gh_realtime START tokens={len(TOKENS)} once={once} ===")
    cycles = 0
    while True:
        try:
            budget = min(MAX_ITEMS, 5) if once else MAX_ITEMS
            n = cycle(budget)
            cycles += 1
            with lock:
                stats["keys"] += n
                snap = dict(stats)
            log(f"cycle {cycles}: +{n} keys | {snap}")
            save_seen()
        except Exception as e:
            log(f"cycle error: {e!r}")
        if once:
            print(f"gh_realtime --once done: {stats}")
            return
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()