vlad/keyhunter-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

Browse Source
This commit is contained in:
vlad
2026-10-20 07:50:07 +00:00
commit a5008d837b
178 changed files with 36909 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
test_sourceforge.py Normal file
View File

@@ -0,0 +1,192 @@
"""Offline test for sourceforge_eternal.py — no network."""
import hashlib
import os
import sqlite3
import sys
import tempfile
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import sourceforge_eternal as sf
# ── Fixtures ──────────────────────────────────────────────────────────
FIXTURE_TBODY_FOLDERS = """<tbody>
<tr title="KeePass 2.x" class="folder ">
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/"
title="Click to enter KeePass 2.x"><svg class="svgico"><use xlink:href="#folder-o"></use></svg>
<span class="name">KeePass 2.x</span></a>
</th>
</tr>
<tr title="Plugins" class="folder ">
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/Plugins/"
title="Click to enter Plugins"><svg class="svgico"><use xlink:href="#folder-o"></use></svg>
<span class="name">Plugins</span></a>
</th>
</tr>
</tbody>"""
# Latest fixture with mixed project
# We test each with its correct esc match below
# A separate URL to test absolute URL variant
FIXTURE_TBODY_FILES_MIXED = """<tbody>
<tr title="KeePass-2.61-Setup.exe" class="file ">
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/2.61/KeePass-2.61-Setup.exe/download"
title="Click to download KeePass-2.61-Setup.exe">
<span class="name">KeePass-2.61-Setup.exe</span></a>
</th>
</tr>
<tr title="KeePass-2.61.zip" class="file ">
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/2.61/KeePass-2.61.zip/download"
title="Click to download KeePass-2.61.zip">
<span class="name">KeePass-2.61.zip</span></a>
</th>
</tr>
<tr title="config.sample.php" class="file ">
<th scope="row" headers="files_name_h"><a href="https://sourceforge.net/projects/keepass/files/KeePass%202.x/2.61/config.sample.php/download"
title="Click to download config.sample.php">
<span class="name">config.sample.php</span></a>
</th>
</tr>
</tbody>"""
# A fake text file content that contains an API key pattern
FAKE_KEY_FILE = """<?php
// Auto-generated configuration
define('DB_NAME', 'mydb');
define('DB_USER', 'admin');
define('DB_PASSWORD', 'supersecret123');
// OpenAI key — 60+ chars after sk-proj- as required by key_patterns
define('OPENAI_API_KEY', 'sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc012345678901234567890123456789');
$stripe_secret_key = 'sk_live_abcdefghijklmnopqrstuvwx1234567890123';
$aws_access_key = 'AKIAIOSFODNN7EXAMPLE';
// end
"""
def test_list_dir_parse():
"""Verify regex on fixture HTML simulates list_dir."""
import re
esc = re.escape("keepass")
# Folders
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
results = [(True, m.group(1)) for m in re.finditer(pattern, FIXTURE_TBODY_FOLDERS, re.DOTALL)]
assert len(results) == 2, f"expected 2 folders, got {len(results)}"
names = [r[1] for r in results]
assert "KeePass%202.x" in names
assert "Plugins" in names
# Files
pattern2 = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
file_results = [(m.group(1), m.group(2)) for m in re.finditer(pattern2, FIXTURE_TBODY_FILES_MIXED, re.DOTALL)]
assert len(file_results) == 3, f"expected 3 files, got {len(file_results)}"
file_paths = [r[0] for r in file_results]
has_config = any("config.sample.php" in p for p in file_paths)
assert has_config, f"config.sample.php not found among {file_paths}"
# Test absolute URL variant
assert any("config.sample.php" in p for p in file_paths), "absolute URL pattern should match"
print("OK test_list_dir_parse")
def test_is_junk():
assert sf.is_junk("sk-xxxx-example")
assert sf.is_junk("aaa") # too short / low entropy
assert not sf.is_junk("sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc")
print("OK test_is_junk")
def test_scan_text():
"""scan_text should find patterns in fake file."""
hits = sf.scan_text(FAKE_KEY_FILE)
assert len(hits) > 0, f"expected hits, got {hits}"
provs = [p for p, _ in hits]
# Should find OPENAI and STRIPE patterns
assert "OPENAI" in provs or "OPENAI_API_KEY" in provs, f"OPENAI not in {provs}"
print(f"OK test_scan_text — found {len(hits)} patterns: {provs}")
def test_db_add():
"""db_add should write a row and dedupe."""
tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
tmp.close()
try:
conn = sqlite3.connect(tmp.name, timeout=120)
try:
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT UNIQUE,
val TEXT, prov TEXT, status DEFAULT "NEW",
plan TEXT, price TEXT, remaining TEXT,
found TEXT, repo TEXT)""")
key = "sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc"
h = hashlib.sha1(key.encode()).hexdigest()[:16]
conn.execute(
"INSERT INTO keys (hash, val, prov, status, found, repo) "
"VALUES (?,?,?,?,?,?) "
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
"found=excluded.found, repo=excluded.repo "
"WHERE keys.status IN ('DEAD','ERR')",
(h, key, "TEST", "NEW", "sourceforge", "test"))
conn.commit()
cur = conn.execute("SELECT val, prov, found FROM keys WHERE hash=?", (h,))
row = cur.fetchone()
assert row is not None, "row should exist"
assert row[0] == key
assert row[2] == "sourceforge"
print("OK test_db_add - row written")
conn.execute(
"INSERT INTO keys (hash, val, prov, status, found, repo) "
"VALUES (?,?,?,?,?,?) "
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
"found=excluded.found, repo=excluded.repo "
"WHERE keys.status IN ('DEAD','ERR')",
(h, key, "TEST", "NEW", "sourceforge", "test"))
conn.commit()
cur = conn.execute("SELECT COUNT(*) FROM keys WHERE hash=?", (h,))
assert cur.fetchone()[0] == 1, "dedupe should keep 1 row"
print("OK test_db_add - dedupe works")
finally:
conn.close()
finally:
os.unlink(tmp.name)
def test_garbage_filter():
"""Garbage strings should be filtered out."""
for junk in ("xxxx", "EXAMPLE_KEY", "your-api-key-here", "abcdefghijklmnopqrst"):
assert sf.is_junk(junk), f"{junk} should be junk"
print("OK test_garbage_filter")
def test_generator_pattern():
"""Verify the regex on the fixtures directly."""
import re
# Folder regex
esc = re.escape("keepass")
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
matches = list(re.finditer(pattern, FIXTURE_TBODY_FOLDERS, re.DOTALL))
assert len(matches) == 2, f"folder regex: expected 2, got {len(matches)}"
assert "KeePass%202.x" in matches[0].group(1)
# File regex - test both relative and absolute href
pattern2 = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
matches2 = list(re.finditer(pattern2, FIXTURE_TBODY_FILES_MIXED, re.DOTALL))
assert len(matches2) == 3, f"file regex: expected 3, got {len(matches2)}"
# Absolute URL variant (https://sourceforge.net/...)
abs_matches = [m for m in matches2 if m.group(1).startswith("KeePass%202.x/2.61/config")]
assert len(abs_matches) == 1, "should match absolute URL file"
print("OK test_generator_pattern")
if __name__ == "__main__":
test_generator_pattern()
test_list_dir_parse()
test_is_junk()
test_garbage_filter()
test_scan_text()
test_db_add()
print("\nAll tests PASSED")