KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
242
clojars_eternal.py
Normal file
242
clojars_eternal.py
Normal file
@@ -0,0 +1,242 @@
|
||||
"""ETERNAL Clojars firehose: repo root group-walk + /api/groups + jar scan.
|
||||
|
||||
Anonymous. Root HTML listing (https://repo.clojars.org/) = ~11k group dirs
|
||||
(fetched once per cycle). Cursor walks GROUPS_PER_CYCLE groups; per group
|
||||
GET /api/groups/{g} (list of {jar_name, latest_version}); per unseen g:a
|
||||
fetch jar from repo.clojars.org (cap 8MB), zipfile scan. seen=g:a.
|
||||
mark_seen ONLY on success.
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "data/clojars_eternal.log")
|
||||
CURSOR_FILE = os.path.join(KF_BASE, "data/clojars_cursor.txt")
|
||||
SEEN_FILE = os.path.join(KF_BASE, "data/clojars_seen.txt")
|
||||
N_THREADS = 6
|
||||
GROUPS_PER_CYCLE = 40
|
||||
JAR_CAP = 8 * 1024 * 1024
|
||||
FILE_CAP = 500_000
|
||||
SLEEP = 90
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
SCAN_EXT = (".clj", ".cljs", ".cljc", ".edn", ".json", ".yaml", ".yml",
|
||||
".properties", ".xml", ".env", ".txt", ".cfg")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-clojars/1.0"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "clojars", repo))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||||
return int(f.read().strip() or 0)
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
s.add(line)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def work(g, a, v):
|
||||
"""Fetch jar, scan. -1 on final fail."""
|
||||
n = 0
|
||||
gp = g.replace(".", "/")
|
||||
url = f"https://repo.clojars.org/{gp}/{a}/{v}/{a}-{v}.jar"
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = requests.get(url, headers=UA, timeout=40)
|
||||
if r.status_code == 404:
|
||||
return 0
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
if len(r.content) > JAR_CAP:
|
||||
return 0
|
||||
zf = zipfile.ZipFile(io.BytesIO(r.content))
|
||||
for m in zf.infolist():
|
||||
if m.is_dir() or m.file_size > FILE_CAP:
|
||||
continue
|
||||
fn = m.filename.lower()
|
||||
if not any(fn.endswith(e) for e in SCAN_EXT):
|
||||
continue
|
||||
data = zf.read(m)
|
||||
if b"\x00" in data[:8192]:
|
||||
continue # binary archive member — no real text key has NUL
|
||||
text = data.decode("utf-8", errors="ignore")
|
||||
for prov, key in scan_text(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
db_add(key, prov, f"clojars:{g}:{a}")
|
||||
n += 1
|
||||
return n
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
log(f" jar-err {g}:{a}")
|
||||
return -1
|
||||
time.sleep(8)
|
||||
return -1
|
||||
|
||||
|
||||
def group_artifacts(g):
|
||||
"""GET /api/groups/{g} -> [(a, v), ...]; -1 on final fail."""
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = requests.get(f"https://clojars.org/api/groups/{g}",
|
||||
headers=UA, timeout=30)
|
||||
if r.status_code == 404:
|
||||
return []
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
out = []
|
||||
for it in r.json():
|
||||
a = it.get("jar_name")
|
||||
v = it.get("latest_version")
|
||||
if a and v:
|
||||
out.append((a, v))
|
||||
return out
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
log(f" group-err {g}")
|
||||
return None
|
||||
time.sleep(8)
|
||||
return None
|
||||
|
||||
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
# root group listing (1.4MB html)
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = requests.get("https://repo.clojars.org/", headers=UA,
|
||||
timeout=60)
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 3:
|
||||
log(f"root-err {type(e).__name__}")
|
||||
return
|
||||
time.sleep(8)
|
||||
groups = re.findall(r'href="([^"/]+)/"', r.text)
|
||||
groups = [g for g in groups if g not in (".meta",)]
|
||||
if not groups:
|
||||
log("root-err empty")
|
||||
return
|
||||
|
||||
idx = load_cursor() % len(groups)
|
||||
items = []
|
||||
for _ in range(GROUPS_PER_CYCLE):
|
||||
g = groups[idx]
|
||||
idx = (idx + 1) % len(groups)
|
||||
arts = group_artifacts(g)
|
||||
if arts is None:
|
||||
continue # group retried next rotation
|
||||
for a, v in arts:
|
||||
sid = f"{g}:{a}"
|
||||
if sid not in seen:
|
||||
items.append((g, a, v))
|
||||
save_cursor(idx)
|
||||
|
||||
ok = fail = 0
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, g, a, v): (g, a) for g, a, v in items}
|
||||
for fut in as_completed(futs):
|
||||
g, a = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception:
|
||||
res = -1
|
||||
if res == -1:
|
||||
fail += 1 # NOT marked seen: retried next cycle
|
||||
else:
|
||||
ok += 1
|
||||
mark_seen(f"{g}:{a}")
|
||||
|
||||
log(f"cycle: groups={len(groups)} idx={idx} items={len(items)} ok={ok} "
|
||||
f"fail={fail} +{total} keys")
|
||||
|
||||
|
||||
def main():
|
||||
log("clojars_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle-err {type(e).__name__}: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user