KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

test_gitlab_artifacts.py

@@ -0,0 +1,120 @@
+"""Test gitlab_artifacts_eternal: parse trace, zip, junk filter, db_add, dedup."""
+import hashlib, io, os, sqlite3, sys, zipfile
+
+BASE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, BASE)
+
+os.environ["KF_DB"] = os.path.join(BASE, "data", "_smoke_gitlab_artifacts.db")
+
+# clean slate
+try:
+    os.remove(os.environ["KF_DB"])
+except FileNotFoundError:
+    pass
+try:
+    os.remove(os.path.join(BASE, "data", "gitlab_artifacts_seen.txt"))
+except FileNotFoundError:
+    pass
+
+# import after db path set
+from gitlab_artifacts_eternal import (
+    db_add, scan_text_chunk, scan_zip, scan_trace, is_junk,
+    conn, GARBAGE, SCAN_EXT, total, lock,
+)
+
+# ── helpers ───────────────────────────────────────────────
+
+def count_rows():
+    return conn.execute("SELECT COUNT(*) FROM keys").fetchone()[0]
+
+def get_rows():
+    return conn.execute("SELECT val, prov, found, repo FROM keys").fetchall()
+
+# clean db before each test block
+def reset_db():
+    conn.execute("DELETE FROM keys")
+    conn.commit()
+    # reset total counter
+    import gitlab_artifacts_eternal as m
+    m.total = 0
+
+
+def test_junk_filter():
+    reset_db()
+    assert is_junk("xxxxxx"), "garbage string not caught"
+    assert is_junk("your-secret-key"), "template not caught"
+    assert is_junk("abc"), "too-short unique chars not caught"
+    assert not is_junk("sk-proj-A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S9t0"), "real-looking key falsely junked"
+    assert not is_junk("ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S"), "real-looking token falsely junked"
+    print("  PASS junk_filter")
+
+
+def test_trace_scan():
+    reset_db()
+    trace = """\
+Running with gitlab-runner 15.0.0
+$ export CI=true
+$ export DEPLOY_KEY="sk-proj-real1test2key3only4for5test"
+$ export OPENAI_API_KEY="sk-real-key-abc123def456"
+$ echo "Building..."
+$ python -c "import os; print(os.environ.get('GEMINI_API_KEY'))"
+Some random log line
+GEMINI_API_KEY=AIzaSyA1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8
+"""
+    n = scan_trace(trace, "test/repo/job/99")
+    nk = count_rows()
+    print(f"  trace scan: {n} keys from scan_text, {nk} rows in db")
+    assert nk >= 2, f"expected >=2 keys, got {nk}"
+    rows = get_rows()
+    vals = {r[0] for r in rows}
+    assert any("AIzaSy" in v for v in vals), "expected AIzaSy GEMINI key"
+    print("  PASS trace_scan")
+
+
+def test_zip_scan():
+    reset_db()
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr(".env", "OPENAI_API_KEY=sk-proj-TestEnvKey001OnlyForUnitTestPurpose1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopMatch\n")
+        zf.writestr("config.json", 'OPENAI_API_KEY=sk-proj-TestJsonKey002OnlyForUnitTestPurpose1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopMatch')
+        zf.writestr("readme.md", "this is documentation")
+        zf.writestr("secret.bin", b"\x00\x01\x02\x03\xde\xad\xbe\xef")  # binary — skipped
+    buf.seek(0)
+    payload = buf.read()
+
+    n = scan_zip(payload, "test/repo/artifacts")
+    nk = count_rows()
+    print(f"  zip scan: {n} keys, {nk} rows")
+    assert nk >= 2, f"expected >=2 keys from zip, got {nk}"
+    rows = get_rows()
+    repos = {r[3] for r in rows}
+    assert any("config.json" in r for r in repos), "expected repo path with filename"
+    print("  PASS zip_scan")
+
+
+def test_dedup():
+    reset_db()
+    # same key inserted twice
+    db_add("sk-test-dedup-key-999", "OPENAI", "test/repo")
+    n1 = count_rows()
+    db_add("sk-test-dedup-key-999", "OPENAI", "test/repo")
+    n2 = count_rows()
+    assert n1 == n2 == 1, f"dedup should keep 1 row, got {n1}/{n2}"
+    print("  PASS dedup")
+
+
+def test_junk_skip():
+    reset_db()
+    n = scan_text_chunk("MY_API_KEY=placeholder\n", "test/repo")
+    nk = count_rows()
+    assert nk == 0, f"placeholder should be filtered, got {nk} rows"
+    print("  PASS junk_skip")
+
+
+if __name__ == "__main__":
+    test_junk_filter()
+    test_trace_scan()
+    test_zip_scan()
+    test_dedup()
+    test_junk_skip()
+    print("\nALL PASS")