keyhunter-parser/goproxy_eternal.py

"""ETERNAL Go module proxy firehose: index.golang.org feed + proxy.golang.org zips.
Anonymous, no rate gate. Each new module version in the index is fetched as a
source zip and scanned with the shared key_patterns scan_text().
Cursor advances by processed index timestamps; seen-modules dedup file.
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import json
import sqlite3
import sys
import threading
import time
import zipfile

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "data/goproxy_eternal.log")
CURSOR_FILE = os.path.join(KF_BASE, "data/goproxy_cursor.txt")
SEEN_FILE = os.path.join(KF_BASE, "data/goproxy_seen.txt")
N_THREADS = 8
BATCH = 400          # index records per cycle
ZIP_CAP = 6 * 1024 * 1024
FILE_CAP = 500_000
SLEEP = 30

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "INSERT", "FAKE", "fake", "Fake", "test", "Test", "dddd",
           "12345", "CHANGE", "deadbeef", "fffffff", "0000000000",
           "abcdefghijklmnopqrst", "1234567890abcdef")

SCAN_EXT = (".go", ".json", ".yaml", ".yml", ".toml", ".mod", ".env",
            ".sh", ".py", ".js", ".ts", ".cfg", ".ini", ".properties")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8", errors="replace") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "goproxy", repo))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            return f.read().strip()
    except Exception:
        return "2019-04-10T19:08:52.997264Z"


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(c)


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                s.add(line.strip())
    except Exception:
        pass
    return s


seen_mods = load_seen()


def mark_seen(p):
    seen_mods.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")
def work(path, version):
    n = 0
    url = f"https://proxy.golang.org/{path}/@v/{version}.zip"
    for attempt in (1, 2):
        try:
            r = requests.get(url, timeout=15)
            if r.status_code != 200:
                return -1
            if len(r.content) > ZIP_CAP:
                return 0
            z = zipfile.ZipFile(io.BytesIO(r.content))
            for fn in z.namelist():
                if not fn.lower().endswith(SCAN_EXT):
                    continue
                try:
                    data = z.read(fn)
                except Exception:
                    continue
                if len(data) > FILE_CAP:
                    continue
                if b"\x00" in data[:8192]:
                    continue  # binary archive member — no real text key has NUL
                text = data.decode("utf-8", errors="ignore")
                for prov, key in scan_text(text):
                    if isinstance(key, tuple):
                        key = key[0]
                    if any(g in key for g in GARBAGE) or len(key) < 30:
                        continue
                    db_add(key, prov, f"go:{path}")
                    n += 1
            return n
        except Exception as e:
            if attempt == 2:
                log(f"  mod-err {path}: {type(e).__name__}")
                return -1
            time.sleep(3)
    return -1


def cycle():
    global total
    total = 0
    cur = load_cursor()
    try:
        r = requests.get("https://index.golang.org/index",
                         params={"limit": BATCH, "since": cur}, timeout=30)
        if r.status_code != 200:
            log(f"index http {r.status_code}")
            return
        lines = [ln for ln in r.text.strip().split("\n") if ln]
        if not lines:
            log("index exhausted, sleep long")
            time.sleep(600)
            return
    except Exception as e:
        log(f"index net-err {type(e).__name__}")
        return

    mods = []
    last_ts = cur
    for ln in lines:
        try:
            rec = json.loads(ln)
        except Exception:
            continue
        p, v, ts = rec.get("Path", ""), rec.get("Version", ""), \
            rec.get("Timestamp", "")
        last_ts = ts or last_ts
        if p and p not in seen_mods and not p.endswith(".git"):
            mods.append((p, v))

    ok = fail = 0
    from concurrent.futures import ThreadPoolExecutor, as_completed
    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futs = {ex.submit(work, p, v): (p, v) for p, v in mods}
        for fut in as_completed(futs):
            p, v = futs[fut]
            try:
                res = fut.result()
            except Exception:
                res = -1
            if res == -1:
                fail += 1          # NOT marked seen: retried next run
            else:
                ok += 1
                mark_seen(p)

    save_cursor(last_ts)
    log(f"cycle: {ok} ok, {fail} fail, +{total} keys, cursor={last_ts[:19]}")

def main():
    log("goproxy_eternal started")
    if "--once" in sys.argv:
        cycle()
        return
    while True:
        try:
            cycle()
        except Exception as e:
            log(f"cycle-err {type(e).__name__}: {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()