243 lines
7.1 KiB
Python
243 lines
7.1 KiB
Python
"""ETERNAL Clojars firehose: repo root group-walk + /api/groups + jar scan.
|
|
|
|
Anonymous. Root HTML listing (https://repo.clojars.org/) = ~11k group dirs
|
|
(fetched once per cycle). Cursor walks GROUPS_PER_CYCLE groups; per group
|
|
GET /api/groups/{g} (list of {jar_name, latest_version}); per unseen g:a
|
|
fetch jar from repo.clojars.org (cap 8MB), zipfile scan. seen=g:a.
|
|
mark_seen ONLY on success.
|
|
"""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import io
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
|
|
import requests
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "data/clojars_eternal.log")
|
|
CURSOR_FILE = os.path.join(KF_BASE, "data/clojars_cursor.txt")
|
|
SEEN_FILE = os.path.join(KF_BASE, "data/clojars_seen.txt")
|
|
N_THREADS = 6
|
|
GROUPS_PER_CYCLE = 40
|
|
JAR_CAP = 8 * 1024 * 1024
|
|
FILE_CAP = 500_000
|
|
SLEEP = 90
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef")
|
|
|
|
SCAN_EXT = (".clj", ".cljs", ".cljc", ".edn", ".json", ".yaml", ".yml",
|
|
".properties", ".xml", ".env", ".txt", ".cfg")
|
|
|
|
UA = {"User-Agent": "keyfarm-clojars/1.0"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
with lock:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "clojars", repo))
|
|
total += 1
|
|
|
|
|
|
def load_cursor():
|
|
try:
|
|
with open(CURSOR_FILE, encoding="utf-8") as f:
|
|
return int(f.read().strip() or 0)
|
|
except (FileNotFoundError, ValueError):
|
|
return 0
|
|
|
|
|
|
def save_cursor(c):
|
|
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
|
f.write(str(c))
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
s.add(line)
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def work(g, a, v):
|
|
"""Fetch jar, scan. -1 on final fail."""
|
|
n = 0
|
|
gp = g.replace(".", "/")
|
|
url = f"https://repo.clojars.org/{gp}/{a}/{v}/{a}-{v}.jar"
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = requests.get(url, headers=UA, timeout=40)
|
|
if r.status_code == 404:
|
|
return 0
|
|
if r.status_code != 200:
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
if len(r.content) > JAR_CAP:
|
|
return 0
|
|
zf = zipfile.ZipFile(io.BytesIO(r.content))
|
|
for m in zf.infolist():
|
|
if m.is_dir() or m.file_size > FILE_CAP:
|
|
continue
|
|
fn = m.filename.lower()
|
|
if not any(fn.endswith(e) for e in SCAN_EXT):
|
|
continue
|
|
data = zf.read(m)
|
|
if b"\x00" in data[:8192]:
|
|
continue # binary archive member — no real text key has NUL
|
|
text = data.decode("utf-8", errors="ignore")
|
|
for prov, key in scan_text(text):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
db_add(key, prov, f"clojars:{g}:{a}")
|
|
n += 1
|
|
return n
|
|
except Exception:
|
|
if attempt == 3:
|
|
log(f" jar-err {g}:{a}")
|
|
return -1
|
|
time.sleep(8)
|
|
return -1
|
|
|
|
|
|
def group_artifacts(g):
|
|
"""GET /api/groups/{g} -> [(a, v), ...]; -1 on final fail."""
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = requests.get(f"https://clojars.org/api/groups/{g}",
|
|
headers=UA, timeout=30)
|
|
if r.status_code == 404:
|
|
return []
|
|
if r.status_code != 200:
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
out = []
|
|
for it in r.json():
|
|
a = it.get("jar_name")
|
|
v = it.get("latest_version")
|
|
if a and v:
|
|
out.append((a, v))
|
|
return out
|
|
except Exception:
|
|
if attempt == 3:
|
|
log(f" group-err {g}")
|
|
return None
|
|
time.sleep(8)
|
|
return None
|
|
|
|
|
|
def cycle():
|
|
global total
|
|
total = 0
|
|
# root group listing (1.4MB html)
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = requests.get("https://repo.clojars.org/", headers=UA,
|
|
timeout=60)
|
|
if r.status_code != 200:
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
break
|
|
except Exception as e:
|
|
if attempt == 3:
|
|
log(f"root-err {type(e).__name__}")
|
|
return
|
|
time.sleep(8)
|
|
groups = re.findall(r'href="([^"/]+)/"', r.text)
|
|
groups = [g for g in groups if g not in (".meta",)]
|
|
if not groups:
|
|
log("root-err empty")
|
|
return
|
|
|
|
idx = load_cursor() % len(groups)
|
|
items = []
|
|
for _ in range(GROUPS_PER_CYCLE):
|
|
g = groups[idx]
|
|
idx = (idx + 1) % len(groups)
|
|
arts = group_artifacts(g)
|
|
if arts is None:
|
|
continue # group retried next rotation
|
|
for a, v in arts:
|
|
sid = f"{g}:{a}"
|
|
if sid not in seen:
|
|
items.append((g, a, v))
|
|
save_cursor(idx)
|
|
|
|
ok = fail = 0
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
|
futs = {ex.submit(work, g, a, v): (g, a) for g, a, v in items}
|
|
for fut in as_completed(futs):
|
|
g, a = futs[fut]
|
|
try:
|
|
res = fut.result()
|
|
except Exception:
|
|
res = -1
|
|
if res == -1:
|
|
fail += 1 # NOT marked seen: retried next cycle
|
|
else:
|
|
ok += 1
|
|
mark_seen(f"{g}:{a}")
|
|
|
|
log(f"cycle: groups={len(groups)} idx={idx} items={len(items)} ok={ok} "
|
|
f"fail={fail} +{total} keys")
|
|
|
|
|
|
def main():
|
|
log("clojars_eternal started")
|
|
if "--once" in sys.argv:
|
|
cycle()
|
|
return
|
|
while True:
|
|
try:
|
|
cycle()
|
|
except Exception as e:
|
|
log(f"cycle-err {type(e).__name__}: {e}")
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|