keyhunter-parser/test_gha_logs.py

"""Test gha_logs_eternal: run list parsing, log zip scanning, db_add, junk filter, dedup.

Sets KF_DB to a smoke DB before importing the module. Creates fixture files:
a runs-response JSON and a log zip with a fake key. No network calls.
"""
import hashlib, io, json, os, sqlite3, sys, zipfile

THIS_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(THIS_DIR, "data", "_smoke_gha_logs.db")
os.environ["KF_DB"] = DB_PATH
os.makedirs(os.path.join(THIS_DIR, "data"), exist_ok=True)

# Clean slate
if os.path.exists(DB_PATH):
    os.remove(DB_PATH)
for sf in ["gha_logs_cursor.txt", "gha_logs_seen.txt"]:
    p = os.path.join(THIS_DIR, "data", sf)
    if os.path.exists(p):
        os.remove(p)

boot_conn = sqlite3.connect(DB_PATH)
boot_conn.execute("""CREATE TABLE IF NOT EXISTS keys (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  hash TEXT UNIQUE,
  val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
  plan TEXT, price TEXT, remaining TEXT,
  found TEXT, repo TEXT)""")
boot_conn.commit()
boot_conn.close()

# Import the lane (reads KF_DB)
sys.path.insert(0, THIS_DIR)
import gha_logs_eternal as gha

# ── Fixtures ──────────────────────────────────────────────

# Fake runs response
FIXTURE_RUNS_JSON = json.dumps({
    "total_count": 2,
    "workflow_runs": [
        {
            "id": 100500,
            "name": "CI",
            "status": "completed",
            "conclusion": "success",
            "created_at": "2026-10-01T00:00:00Z",
            "updated_at": "2026-10-01T01:00:00Z"
        },
        {
            "id": 100501,
            "name": "Lint",
            "status": "completed",
            "conclusion": "failure",
            "created_at": "2026-10-01T02:00:00Z",
            "updated_at": "2026-10-01T03:00:00Z"
        }
    ]
})

# Build a log zip fixture with one member containing a real-looking key
# and one member with junk that should be filtered out
FIXTURE_LOG_ZIP_BYTES = io.BytesIO()
with zipfile.ZipFile(FIXTURE_LOG_ZIP_BYTES, "w", zipfile.ZIP_DEFLATED) as z:
    # Member 1: valid keys that pass scan_text + GARBAGE filter
    log_text = (
        "2026-10-01T00:00:01Z Current runner version: '2.320.0'\n"
        "2026-10-01T00:00:02Z ##[group]Build\n"
        "2026-10-01T00:00:03Z export MY_SECRET=ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw\n"
        "2026-10-01T00:00:04Z export DB_PASSWORD=gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw\n"
        "2026-10-01T00:00:05Z ##[endgroup]\n"
    )
    z.writestr("1_build.txt", log_text)

    # Member 2: junk — should be filtered
    junk_log = (
        "2026-10-01T00:01:00Z export TOKEN=your-api-key-here-1234567890abcdefghijklm\n"
    )
    z.writestr("2_init_junk.txt", junk_log)

    # Member 3: no key at all  
    z.writestr("3_setup.txt", "echo hello world\necho done\n")

FIXTURE_LOG_ZIP = FIXTURE_LOG_ZIP_BYTES.getvalue()

# ── Helpers ───────────────────────────────────────────────

def count_rows():
    return gha.conn.execute("SELECT COUNT(*) FROM keys").fetchone()[0]

def get_rows():
    return gha.conn.execute("SELECT val, prov, found, repo FROM keys").fetchall()

def reset_db():
    gha.conn.execute("DELETE FROM keys")
    gha.conn.commit()
    gha.total = 0

# ── Tests ─────────────────────────────────────────────────

def test_runs_parse():
    """Parse runs JSON and extract run IDs."""
    data = json.loads(FIXTURE_RUNS_JSON)
    assert data["total_count"] == 2
    runs = data["workflow_runs"]
    assert len(runs) == 2
    ids = [r["id"] for r in runs]
    assert 100500 in ids
    assert 100501 in ids
    print("  test_runs_parse: PASS")


def test_junk_filter():
    """is_junk rejects placeholder and low-entropy values."""
    assert gha.is_junk("your-api-key-here-1234567890abcdefghijklm")  # "your"
    assert gha.is_junk("abcdefghijklmnopqrst1234567890abcdefghijk")  # both garbage substrings
    assert gha.is_junk("1234567890abcdef1234567890abcdefghijklm")   # garbage
    assert gha.is_junk("xxxxx-example-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx")  # "example"
    assert gha.is_junk("aaaaaaaaaaaaaaaaaaaa")  # entropy < 6 unique chars
    assert gha.is_junk("mock_api_key_1234567890abcdefghijklmno")  # "mock_"

    # valid keys should pass
    assert not gha.is_junk("ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw")
    assert not gha.is_junk("gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw")
    print("  test_junk_filter: PASS")


def test_scan_logs_zip():
    """scan_logs_zip extracts real keys from fixture zip."""
    reset_db()
    n = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
    assert n > 0, f"expected keys, got {n}"
    rows = get_rows()
    vals = [r[0] for r in rows]
    # The real keys should be there
    assert any("ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw" in v for v in vals), \
        f"missing ghp key in {vals}"
    assert any("gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw" in v for v in vals), \
        f"missing gho key in {vals}"
    # Junk should NOT be in the DB
    assert not any("your-api-key-here" in v for v in vals), \
        f"junk not filtered: {vals}"
    print(f"  test_scan_logs_zip: PASS ({n} keys)")


def test_dedup():
    """Same log scanned twice should not duplicate rows."""
    reset_db()
    n1 = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
    r1 = count_rows()
    n2 = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
    r2 = count_rows()
    assert r1 == r2, f"duplicate scan added rows: {r1} -> {r2}"
    print(f"  test_dedup: PASS ({r1} rows stable)")


def test_is_junk_low_entropy():
    """Values where len(set(v)) < 6 are junk."""
    assert gha.is_junk("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
    assert gha.is_junk("1111111111111111111111111111111111111111111111")
    assert gha.is_junk("abcabcabcabcabcabcabcabcabcabcabcabcabcabcabca")
    print("  test_is_junk_low_entropy: PASS")


# ── Run ───────────────────────────────────────────────────

if __name__ == "__main__":
    test_runs_parse()
    test_junk_filter()
    test_scan_logs_zip()
    test_dedup()
    test_is_junk_low_entropy()
    print("\nAll tests passed.")