KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
203
goproxy_eternal.py
Normal file
203
goproxy_eternal.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""ETERNAL Go module proxy firehose: index.golang.org feed + proxy.golang.org zips.
|
||||
Anonymous, no rate gate. Each new module version in the index is fetched as a
|
||||
source zip and scanned with the shared key_patterns scan_text().
|
||||
Cursor advances by processed index timestamps; seen-modules dedup file.
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "data/goproxy_eternal.log")
|
||||
CURSOR_FILE = os.path.join(KF_BASE, "data/goproxy_cursor.txt")
|
||||
SEEN_FILE = os.path.join(KF_BASE, "data/goproxy_seen.txt")
|
||||
N_THREADS = 8
|
||||
BATCH = 400 # index records per cycle
|
||||
ZIP_CAP = 6 * 1024 * 1024
|
||||
FILE_CAP = 500_000
|
||||
SLEEP = 30
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"INSERT", "FAKE", "fake", "Fake", "test", "Test", "dddd",
|
||||
"12345", "CHANGE", "deadbeef", "fffffff", "0000000000",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
SCAN_EXT = (".go", ".json", ".yaml", ".yml", ".toml", ".mod", ".env",
|
||||
".sh", ".py", ".js", ".ts", ".cfg", ".ini", ".properties")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "goproxy", repo))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||||
return f.read().strip()
|
||||
except Exception:
|
||||
return "2019-04-10T19:08:52.997264Z"
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(c)
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
s.add(line.strip())
|
||||
except Exception:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen_mods = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen_mods.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
def work(path, version):
|
||||
n = 0
|
||||
url = f"https://proxy.golang.org/{path}/@v/{version}.zip"
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
r = requests.get(url, timeout=15)
|
||||
if r.status_code != 200:
|
||||
return -1
|
||||
if len(r.content) > ZIP_CAP:
|
||||
return 0
|
||||
z = zipfile.ZipFile(io.BytesIO(r.content))
|
||||
for fn in z.namelist():
|
||||
if not fn.lower().endswith(SCAN_EXT):
|
||||
continue
|
||||
try:
|
||||
data = z.read(fn)
|
||||
except Exception:
|
||||
continue
|
||||
if len(data) > FILE_CAP:
|
||||
continue
|
||||
if b"\x00" in data[:8192]:
|
||||
continue # binary archive member — no real text key has NUL
|
||||
text = data.decode("utf-8", errors="ignore")
|
||||
for prov, key in scan_text(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
db_add(key, prov, f"go:{path}")
|
||||
n += 1
|
||||
return n
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
log(f" mod-err {path}: {type(e).__name__}")
|
||||
return -1
|
||||
time.sleep(3)
|
||||
return -1
|
||||
|
||||
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
cur = load_cursor()
|
||||
try:
|
||||
r = requests.get("https://index.golang.org/index",
|
||||
params={"limit": BATCH, "since": cur}, timeout=30)
|
||||
if r.status_code != 200:
|
||||
log(f"index http {r.status_code}")
|
||||
return
|
||||
lines = [ln for ln in r.text.strip().split("\n") if ln]
|
||||
if not lines:
|
||||
log("index exhausted, sleep long")
|
||||
time.sleep(600)
|
||||
return
|
||||
except Exception as e:
|
||||
log(f"index net-err {type(e).__name__}")
|
||||
return
|
||||
|
||||
mods = []
|
||||
last_ts = cur
|
||||
for ln in lines:
|
||||
try:
|
||||
rec = json.loads(ln)
|
||||
except Exception:
|
||||
continue
|
||||
p, v, ts = rec.get("Path", ""), rec.get("Version", ""), \
|
||||
rec.get("Timestamp", "")
|
||||
last_ts = ts or last_ts
|
||||
if p and p not in seen_mods and not p.endswith(".git"):
|
||||
mods.append((p, v))
|
||||
|
||||
ok = fail = 0
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, p, v): (p, v) for p, v in mods}
|
||||
for fut in as_completed(futs):
|
||||
p, v = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception:
|
||||
res = -1
|
||||
if res == -1:
|
||||
fail += 1 # NOT marked seen: retried next run
|
||||
else:
|
||||
ok += 1
|
||||
mark_seen(p)
|
||||
|
||||
save_cursor(last_ts)
|
||||
log(f"cycle: {ok} ok, {fail} fail, +{total} keys, cursor={last_ts[:19]}")
|
||||
|
||||
def main():
|
||||
log("goproxy_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle-err {type(e).__name__}: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user