KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

test_gha_logs.py

@@ -0,0 +1,172 @@
+"""Test gha_logs_eternal: run list parsing, log zip scanning, db_add, junk filter, dedup.
+
+Sets KF_DB to a smoke DB before importing the module. Creates fixture files:
+a runs-response JSON and a log zip with a fake key. No network calls.
+"""
+import hashlib, io, json, os, sqlite3, sys, zipfile
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+DB_PATH = os.path.join(THIS_DIR, "data", "_smoke_gha_logs.db")
+os.environ["KF_DB"] = DB_PATH
+os.makedirs(os.path.join(THIS_DIR, "data"), exist_ok=True)
+
+# Clean slate
+if os.path.exists(DB_PATH):
+    os.remove(DB_PATH)
+for sf in ["gha_logs_cursor.txt", "gha_logs_seen.txt"]:
+    p = os.path.join(THIS_DIR, "data", sf)
+    if os.path.exists(p):
+        os.remove(p)
+
+boot_conn = sqlite3.connect(DB_PATH)
+boot_conn.execute("""CREATE TABLE IF NOT EXISTS keys (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  hash TEXT UNIQUE,
+  val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
+  plan TEXT, price TEXT, remaining TEXT,
+  found TEXT, repo TEXT)""")
+boot_conn.commit()
+boot_conn.close()
+
+# Import the lane (reads KF_DB)
+sys.path.insert(0, THIS_DIR)
+import gha_logs_eternal as gha
+
+# ── Fixtures ──────────────────────────────────────────────
+
+# Fake runs response
+FIXTURE_RUNS_JSON = json.dumps({
+    "total_count": 2,
+    "workflow_runs": [
+        {
+            "id": 100500,
+            "name": "CI",
+            "status": "completed",
+            "conclusion": "success",
+            "created_at": "2026-10-01T00:00:00Z",
+            "updated_at": "2026-10-01T01:00:00Z"
+        },
+        {
+            "id": 100501,
+            "name": "Lint",
+            "status": "completed",
+            "conclusion": "failure",
+            "created_at": "2026-10-01T02:00:00Z",
+            "updated_at": "2026-10-01T03:00:00Z"
+        }
+    ]
+})
+
+# Build a log zip fixture with one member containing a real-looking key
+# and one member with junk that should be filtered out
+FIXTURE_LOG_ZIP_BYTES = io.BytesIO()
+with zipfile.ZipFile(FIXTURE_LOG_ZIP_BYTES, "w", zipfile.ZIP_DEFLATED) as z:
+    # Member 1: valid keys that pass scan_text + GARBAGE filter
+    log_text = (
+        "2026-10-01T00:00:01Z Current runner version: '2.320.0'\n"
+        "2026-10-01T00:00:02Z ##[group]Build\n"
+        "2026-10-01T00:00:03Z export MY_SECRET=ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw\n"
+        "2026-10-01T00:00:04Z export DB_PASSWORD=gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw\n"
+        "2026-10-01T00:00:05Z ##[endgroup]\n"
+    )
+    z.writestr("1_build.txt", log_text)
+
+    # Member 2: junk — should be filtered
+    junk_log = (
+        "2026-10-01T00:01:00Z export TOKEN=your-api-key-here-1234567890abcdefghijklm\n"
+    )
+    z.writestr("2_init_junk.txt", junk_log)
+
+    # Member 3: no key at all  
+    z.writestr("3_setup.txt", "echo hello world\necho done\n")
+
+FIXTURE_LOG_ZIP = FIXTURE_LOG_ZIP_BYTES.getvalue()
+
+# ── Helpers ───────────────────────────────────────────────
+
+def count_rows():
+    return gha.conn.execute("SELECT COUNT(*) FROM keys").fetchone()[0]
+
+def get_rows():
+    return gha.conn.execute("SELECT val, prov, found, repo FROM keys").fetchall()
+
+def reset_db():
+    gha.conn.execute("DELETE FROM keys")
+    gha.conn.commit()
+    gha.total = 0
+
+# ── Tests ─────────────────────────────────────────────────
+
+def test_runs_parse():
+    """Parse runs JSON and extract run IDs."""
+    data = json.loads(FIXTURE_RUNS_JSON)
+    assert data["total_count"] == 2
+    runs = data["workflow_runs"]
+    assert len(runs) == 2
+    ids = [r["id"] for r in runs]
+    assert 100500 in ids
+    assert 100501 in ids
+    print("  test_runs_parse: PASS")
+
+
+def test_junk_filter():
+    """is_junk rejects placeholder and low-entropy values."""
+    assert gha.is_junk("your-api-key-here-1234567890abcdefghijklm")  # "your"
+    assert gha.is_junk("abcdefghijklmnopqrst1234567890abcdefghijk")  # both garbage substrings
+    assert gha.is_junk("1234567890abcdef1234567890abcdefghijklm")   # garbage
+    assert gha.is_junk("xxxxx-example-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx")  # "example"
+    assert gha.is_junk("aaaaaaaaaaaaaaaaaaaa")  # entropy < 6 unique chars
+    assert gha.is_junk("mock_api_key_1234567890abcdefghijklmno")  # "mock_"
+
+    # valid keys should pass
+    assert not gha.is_junk("ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw")
+    assert not gha.is_junk("gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw")
+    print("  test_junk_filter: PASS")
+
+
+def test_scan_logs_zip():
+    """scan_logs_zip extracts real keys from fixture zip."""
+    reset_db()
+    n = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
+    assert n > 0, f"expected keys, got {n}"
+    rows = get_rows()
+    vals = [r[0] for r in rows]
+    # The real keys should be there
+    assert any("ghp_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw" in v for v in vals), \
+        f"missing ghp key in {vals}"
+    assert any("gho_QwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQwQw" in v for v in vals), \
+        f"missing gho key in {vals}"
+    # Junk should NOT be in the DB
+    assert not any("your-api-key-here" in v for v in vals), \
+        f"junk not filtered: {vals}"
+    print(f"  test_scan_logs_zip: PASS ({n} keys)")
+
+
+def test_dedup():
+    """Same log scanned twice should not duplicate rows."""
+    reset_db()
+    n1 = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
+    r1 = count_rows()
+    n2 = gha.scan_logs_zip(FIXTURE_LOG_ZIP, "org/repo", 100500)
+    r2 = count_rows()
+    assert r1 == r2, f"duplicate scan added rows: {r1} -> {r2}"
+    print(f"  test_dedup: PASS ({r1} rows stable)")
+
+
+def test_is_junk_low_entropy():
+    """Values where len(set(v)) < 6 are junk."""
+    assert gha.is_junk("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+    assert gha.is_junk("1111111111111111111111111111111111111111111111")
+    assert gha.is_junk("abcabcabcabcabcabcabcabcabcabcabcabcabcabcabca")
+    print("  test_is_junk_low_entropy: PASS")
+
+
+# ── Run ───────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    test_runs_parse()
+    test_junk_filter()
+    test_scan_logs_zip()
+    test_dedup()
+    test_is_junk_low_entropy()
+    print("\nAll tests passed.")