120 lines
4.0 KiB
Python
120 lines
4.0 KiB
Python
"""Test gitlab_artifacts_eternal: parse trace, zip, junk filter, db_add, dedup."""
|
|
import hashlib, io, os, sqlite3, sys, zipfile
|
|
|
|
BASE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, BASE)
|
|
|
|
os.environ["KF_DB"] = os.path.join(BASE, "data", "_smoke_gitlab_artifacts.db")
|
|
|
|
# clean slate
|
|
try:
|
|
os.remove(os.environ["KF_DB"])
|
|
except FileNotFoundError:
|
|
pass
|
|
try:
|
|
os.remove(os.path.join(BASE, "data", "gitlab_artifacts_seen.txt"))
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
# import after db path set
|
|
from gitlab_artifacts_eternal import (
|
|
db_add, scan_text_chunk, scan_zip, scan_trace, is_junk,
|
|
conn, GARBAGE, SCAN_EXT, total, lock,
|
|
)
|
|
|
|
# ── helpers ───────────────────────────────────────────────
|
|
|
|
def count_rows():
|
|
return conn.execute("SELECT COUNT(*) FROM keys").fetchone()[0]
|
|
|
|
def get_rows():
|
|
return conn.execute("SELECT val, prov, found, repo FROM keys").fetchall()
|
|
|
|
# clean db before each test block
|
|
def reset_db():
|
|
conn.execute("DELETE FROM keys")
|
|
conn.commit()
|
|
# reset total counter
|
|
import gitlab_artifacts_eternal as m
|
|
m.total = 0
|
|
|
|
|
|
def test_junk_filter():
|
|
reset_db()
|
|
assert is_junk("xxxxxx"), "garbage string not caught"
|
|
assert is_junk("your-secret-key"), "template not caught"
|
|
assert is_junk("abc"), "too-short unique chars not caught"
|
|
assert not is_junk("sk-proj-A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S9t0"), "real-looking key falsely junked"
|
|
assert not is_junk("ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S"), "real-looking token falsely junked"
|
|
print(" PASS junk_filter")
|
|
|
|
|
|
def test_trace_scan():
|
|
reset_db()
|
|
trace = """\
|
|
Running with gitlab-runner 15.0.0
|
|
$ export CI=true
|
|
$ export DEPLOY_KEY="sk-proj-real1test2key3only4for5test"
|
|
$ export OPENAI_API_KEY="sk-real-key-abc123def456"
|
|
$ echo "Building..."
|
|
$ python -c "import os; print(os.environ.get('GEMINI_API_KEY'))"
|
|
Some random log line
|
|
GEMINI_API_KEY=AIzaSyA1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8
|
|
"""
|
|
n = scan_trace(trace, "test/repo/job/99")
|
|
nk = count_rows()
|
|
print(f" trace scan: {n} keys from scan_text, {nk} rows in db")
|
|
assert nk >= 2, f"expected >=2 keys, got {nk}"
|
|
rows = get_rows()
|
|
vals = {r[0] for r in rows}
|
|
assert any("AIzaSy" in v for v in vals), "expected AIzaSy GEMINI key"
|
|
print(" PASS trace_scan")
|
|
|
|
|
|
def test_zip_scan():
|
|
reset_db()
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
zf.writestr(".env", "OPENAI_API_KEY=sk-proj-TestEnvKey001OnlyForUnitTestPurpose1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopMatch\n")
|
|
zf.writestr("config.json", 'OPENAI_API_KEY=sk-proj-TestJsonKey002OnlyForUnitTestPurpose1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopMatch')
|
|
zf.writestr("readme.md", "this is documentation")
|
|
zf.writestr("secret.bin", b"\x00\x01\x02\x03\xde\xad\xbe\xef") # binary — skipped
|
|
buf.seek(0)
|
|
payload = buf.read()
|
|
|
|
n = scan_zip(payload, "test/repo/artifacts")
|
|
nk = count_rows()
|
|
print(f" zip scan: {n} keys, {nk} rows")
|
|
assert nk >= 2, f"expected >=2 keys from zip, got {nk}"
|
|
rows = get_rows()
|
|
repos = {r[3] for r in rows}
|
|
assert any("config.json" in r for r in repos), "expected repo path with filename"
|
|
print(" PASS zip_scan")
|
|
|
|
|
|
def test_dedup():
|
|
reset_db()
|
|
# same key inserted twice
|
|
db_add("sk-test-dedup-key-999", "OPENAI", "test/repo")
|
|
n1 = count_rows()
|
|
db_add("sk-test-dedup-key-999", "OPENAI", "test/repo")
|
|
n2 = count_rows()
|
|
assert n1 == n2 == 1, f"dedup should keep 1 row, got {n1}/{n2}"
|
|
print(" PASS dedup")
|
|
|
|
|
|
def test_junk_skip():
|
|
reset_db()
|
|
n = scan_text_chunk("MY_API_KEY=placeholder\n", "test/repo")
|
|
nk = count_rows()
|
|
assert nk == 0, f"placeholder should be filtered, got {nk} rows"
|
|
print(" PASS junk_skip")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_junk_filter()
|
|
test_trace_scan()
|
|
test_zip_scan()
|
|
test_dedup()
|
|
test_junk_skip()
|
|
print("\nALL PASS") |