KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
222
maven_eternal.py
Normal file
222
maven_eternal.py
Normal file
@@ -0,0 +1,222 @@
|
||||
"""ETERNAL Maven Central firehose: Central Portal browse (publishedDate desc)
|
||||
+ repo1 POM/JAR scan.
|
||||
|
||||
Portal POST /api/internal/browse/components: 20 comps/page, 500 pages max
|
||||
(10k newest). seen=g:a (one scan per artifact, version churn skipped).
|
||||
POM text + JAR/WAR/AAR/ZIP (zipfile) scan via key_patterns.scan_text.
|
||||
mark_seen ONLY on success (fail=-1 retried next cycle).
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import io
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "data/maven_eternal.log")
|
||||
PORTAL_URL = "https://central.sonatype.com/api/internal/browse/components"
|
||||
SEEN_FILE = os.path.join(KF_BASE, "data/maven_seen.txt")
|
||||
MAX_PAGES = 10 # portal pages per cycle (20 comps each)
|
||||
N_THREADS = 6
|
||||
JAR_CAP = 8 * 1024 * 1024
|
||||
FILE_CAP = 500_000
|
||||
SLEEP = 60
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
SCAN_EXT = (".xml", ".properties", ".json", ".yaml", ".yml", ".toml", ".env",
|
||||
".sh", ".py", ".js", ".ts", ".cfg", ".ini", ".gradle", ".MF", ".txt")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "maven", repo))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
s.add(line.strip())
|
||||
except Exception:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def scan_bytes(data, repo, n):
|
||||
text = data.decode("utf-8", errors="ignore")
|
||||
for prov, key in scan_text(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
db_add(key, prov, repo)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def work(g, a, v, packaging):
|
||||
"""Fetch POM (+packaged zip if jar/war/aar) for g:a:v, scan. -1 on final fail."""
|
||||
n = 0
|
||||
path = f"{g.replace('.', '/')}/{a}/{v}/{a}-{v}"
|
||||
base = f"https://repo1.maven.org/maven2/{path}"
|
||||
targets = [(base + ".pom", False)]
|
||||
if packaging in ("jar", "war", "aar", "zip"):
|
||||
targets.append((base + "." + packaging, True))
|
||||
ok_any = False
|
||||
for url, is_zip in targets:
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
r = requests.get(url, timeout=20)
|
||||
if r.status_code == 404:
|
||||
ok_any = True
|
||||
break
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
if len(r.content) > JAR_CAP:
|
||||
ok_any = True
|
||||
break
|
||||
if is_zip:
|
||||
try:
|
||||
z = zipfile.ZipFile(io.BytesIO(r.content))
|
||||
except Exception:
|
||||
ok_any = True
|
||||
break
|
||||
for fn in z.namelist():
|
||||
if not fn.lower().endswith(SCAN_EXT):
|
||||
continue
|
||||
try:
|
||||
data = z.read(fn)
|
||||
except Exception:
|
||||
continue
|
||||
if len(data) > FILE_CAP:
|
||||
continue
|
||||
n = scan_bytes(data, f"maven:{g}:{a}", n)
|
||||
else:
|
||||
n = scan_bytes(r.content, f"maven:{g}:{a}", n)
|
||||
ok_any = True
|
||||
break
|
||||
except Exception:
|
||||
if attempt == 2:
|
||||
log(f" mvn-err {path} {url.rsplit('.', 1)[-1]}")
|
||||
return -1 if not ok_any else n
|
||||
time.sleep(3)
|
||||
if not ok_any:
|
||||
return -1
|
||||
return n
|
||||
|
||||
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
UA = {"User-Agent": "keyfarm-maven/1.0",
|
||||
"Content-Type": "application/json"}
|
||||
items = []
|
||||
for page in range(MAX_PAGES):
|
||||
payload = {"page": page, "sortField": "publishedDate",
|
||||
"sortDirection": "desc"}
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = requests.post(PORTAL_URL, headers=UA, json=payload,
|
||||
timeout=40)
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
d = r.json()
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 3:
|
||||
log(f"portal-err page={page} {type(e).__name__}: "
|
||||
f"{str(e)[:100]}")
|
||||
return
|
||||
time.sleep(8)
|
||||
for comp in d.get("components", []):
|
||||
g = comp.get("namespace")
|
||||
a = comp.get("name")
|
||||
lvi = comp.get("latestVersionInfo") or {}
|
||||
v = lvi.get("version")
|
||||
p = comp.get("packaging") or "jar"
|
||||
if not (g and a and v):
|
||||
continue
|
||||
if p not in ("jar", "war", "aar", "zip", "pom"):
|
||||
p = "jar"
|
||||
sid = f"{g}:{a}"
|
||||
if sid in seen:
|
||||
continue
|
||||
items.append((g, a, v, p))
|
||||
|
||||
ok = fail = 0
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, g, a, v, p): (g, a, v) for g, a, v, p in items}
|
||||
for fut in as_completed(futs):
|
||||
g, a, v = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception:
|
||||
res = -1
|
||||
if res == -1:
|
||||
fail += 1 # NOT marked seen: retried next cycle
|
||||
else:
|
||||
ok += 1
|
||||
mark_seen(f"{g}:{a}")
|
||||
|
||||
log(f"cycle: items={len(items)} ok={ok} fail={fail} +{total} keys")
|
||||
|
||||
|
||||
def main():
|
||||
log("maven_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle-err {type(e).__name__}: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user