keyhunter-parser/maven_eternal.py

"""ETERNAL Maven Central firehose: Central Portal browse (publishedDate desc)
+ repo1 POM/JAR scan.

Portal POST /api/internal/browse/components: 20 comps/page, 500 pages max
(10k newest). seen=g:a (one scan per artifact, version churn skipped).
POM text + JAR/WAR/AAR/ZIP (zipfile) scan via key_patterns.scan_text.
mark_seen ONLY on success (fail=-1 retried next cycle).
"""
import os
KF_BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
import hashlib
import io
import sqlite3
import sys
import threading
import time
import zipfile

import requests

DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
LOG = os.path.join(KF_BASE, "data/maven_eternal.log")
PORTAL_URL = "https://central.sonatype.com/api/internal/browse/components"
SEEN_FILE = os.path.join(KF_BASE, "data/maven_seen.txt")
MAX_PAGES = 10         # portal pages per cycle (20 comps each)
N_THREADS = 6
JAR_CAP = 8 * 1024 * 1024
FILE_CAP = 500_000
SLEEP = 60

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

SCAN_EXT = (".xml", ".properties", ".json", ".yaml", ".yml", ".toml", ".env",
            ".sh", ".py", ".js", ".ts", ".cfg", ".ini", ".gradle", ".MF", ".txt")


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "maven", repo))
        total += 1


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                s.add(line.strip())
    except Exception:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def scan_bytes(data, repo, n):
    text = data.decode("utf-8", errors="ignore")
    for prov, key in scan_text(text):
        if isinstance(key, tuple):
            key = key[0]
        if any(g in key for g in GARBAGE) or len(key) < 30:
            continue
        db_add(key, prov, repo)
        n += 1
    return n


def work(g, a, v, packaging):
    """Fetch POM (+packaged zip if jar/war/aar) for g:a:v, scan. -1 on final fail."""
    n = 0
    path = f"{g.replace('.', '/')}/{a}/{v}/{a}-{v}"
    base = f"https://repo1.maven.org/maven2/{path}"
    targets = [(base + ".pom", False)]
    if packaging in ("jar", "war", "aar", "zip"):
        targets.append((base + "." + packaging, True))
    ok_any = False
    for url, is_zip in targets:
        for attempt in (1, 2):
            try:
                r = requests.get(url, timeout=20)
                if r.status_code == 404:
                    ok_any = True
                    break
                if r.status_code != 200:
                    raise RuntimeError(f"http {r.status_code}")
                if len(r.content) > JAR_CAP:
                    ok_any = True
                    break
                if is_zip:
                    try:
                        z = zipfile.ZipFile(io.BytesIO(r.content))
                    except Exception:
                        ok_any = True
                        break
                    for fn in z.namelist():
                        if not fn.lower().endswith(SCAN_EXT):
                            continue
                        try:
                            data = z.read(fn)
                        except Exception:
                            continue
                        if len(data) > FILE_CAP:
                            continue
                        n = scan_bytes(data, f"maven:{g}:{a}", n)
                else:
                    n = scan_bytes(r.content, f"maven:{g}:{a}", n)
                ok_any = True
                break
            except Exception:
                if attempt == 2:
                    log(f"  mvn-err {path} {url.rsplit('.', 1)[-1]}")
                    return -1 if not ok_any else n
                time.sleep(3)
    if not ok_any:
        return -1
    return n


def cycle():
    global total
    total = 0
    UA = {"User-Agent": "keyfarm-maven/1.0",
          "Content-Type": "application/json"}
    items = []
    for page in range(MAX_PAGES):
        payload = {"page": page, "sortField": "publishedDate",
                   "sortDirection": "desc"}
        for attempt in (1, 2, 3):
            try:
                r = requests.post(PORTAL_URL, headers=UA, json=payload,
                                  timeout=40)
                if r.status_code != 200:
                    raise RuntimeError(f"http {r.status_code}")
                d = r.json()
                break
            except Exception as e:
                if attempt == 3:
                    log(f"portal-err page={page} {type(e).__name__}: "
                        f"{str(e)[:100]}")
                    return
                time.sleep(8)
        for comp in d.get("components", []):
            g = comp.get("namespace")
            a = comp.get("name")
            lvi = comp.get("latestVersionInfo") or {}
            v = lvi.get("version")
            p = comp.get("packaging") or "jar"
            if not (g and a and v):
                continue
            if p not in ("jar", "war", "aar", "zip", "pom"):
                p = "jar"
            sid = f"{g}:{a}"
            if sid in seen:
                continue
            items.append((g, a, v, p))

    ok = fail = 0
    from concurrent.futures import ThreadPoolExecutor, as_completed
    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futs = {ex.submit(work, g, a, v, p): (g, a, v) for g, a, v, p in items}
        for fut in as_completed(futs):
            g, a, v = futs[fut]
            try:
                res = fut.result()
            except Exception:
                res = -1
            if res == -1:
                fail += 1          # NOT marked seen: retried next cycle
            else:
                ok += 1
                mark_seen(f"{g}:{a}")

    log(f"cycle: items={len(items)} ok={ok} fail={fail} +{total} keys")


def main():
    log("maven_eternal started")
    if "--once" in sys.argv:
        cycle()
        return
    while True:
        try:
            cycle()
        except Exception as e:
            log(f"cycle-err {type(e).__name__}: {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()