192 lines
8.5 KiB
Python
192 lines
8.5 KiB
Python
"""Offline test for sourceforge_eternal.py — no network."""
|
|
import hashlib
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import sourceforge_eternal as sf
|
|
|
|
# ── Fixtures ──────────────────────────────────────────────────────────
|
|
|
|
FIXTURE_TBODY_FOLDERS = """<tbody>
|
|
<tr title="KeePass 2.x" class="folder ">
|
|
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/"
|
|
title="Click to enter KeePass 2.x"><svg class="svgico"><use xlink:href="#folder-o"></use></svg>
|
|
<span class="name">KeePass 2.x</span></a>
|
|
</th>
|
|
</tr>
|
|
<tr title="Plugins" class="folder ">
|
|
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/Plugins/"
|
|
title="Click to enter Plugins"><svg class="svgico"><use xlink:href="#folder-o"></use></svg>
|
|
<span class="name">Plugins</span></a>
|
|
</th>
|
|
</tr>
|
|
</tbody>"""
|
|
|
|
# Latest fixture with mixed project
|
|
# We test each with its correct esc match below
|
|
# A separate URL to test absolute URL variant
|
|
FIXTURE_TBODY_FILES_MIXED = """<tbody>
|
|
<tr title="KeePass-2.61-Setup.exe" class="file ">
|
|
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/2.61/KeePass-2.61-Setup.exe/download"
|
|
title="Click to download KeePass-2.61-Setup.exe">
|
|
<span class="name">KeePass-2.61-Setup.exe</span></a>
|
|
</th>
|
|
</tr>
|
|
<tr title="KeePass-2.61.zip" class="file ">
|
|
<th scope="row" headers="files_name_h"><a href="/projects/keepass/files/KeePass%202.x/2.61/KeePass-2.61.zip/download"
|
|
title="Click to download KeePass-2.61.zip">
|
|
<span class="name">KeePass-2.61.zip</span></a>
|
|
</th>
|
|
</tr>
|
|
<tr title="config.sample.php" class="file ">
|
|
<th scope="row" headers="files_name_h"><a href="https://sourceforge.net/projects/keepass/files/KeePass%202.x/2.61/config.sample.php/download"
|
|
title="Click to download config.sample.php">
|
|
<span class="name">config.sample.php</span></a>
|
|
</th>
|
|
</tr>
|
|
</tbody>"""
|
|
|
|
# A fake text file content that contains an API key pattern
|
|
FAKE_KEY_FILE = """<?php
|
|
// Auto-generated configuration
|
|
define('DB_NAME', 'mydb');
|
|
define('DB_USER', 'admin');
|
|
define('DB_PASSWORD', 'supersecret123');
|
|
// OpenAI key — 60+ chars after sk-proj- as required by key_patterns
|
|
define('OPENAI_API_KEY', 'sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc012345678901234567890123456789');
|
|
$stripe_secret_key = 'sk_live_abcdefghijklmnopqrstuvwx1234567890123';
|
|
$aws_access_key = 'AKIAIOSFODNN7EXAMPLE';
|
|
// end
|
|
"""
|
|
|
|
|
|
def test_list_dir_parse():
|
|
"""Verify regex on fixture HTML simulates list_dir."""
|
|
import re
|
|
esc = re.escape("keepass")
|
|
|
|
# Folders
|
|
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
|
|
results = [(True, m.group(1)) for m in re.finditer(pattern, FIXTURE_TBODY_FOLDERS, re.DOTALL)]
|
|
assert len(results) == 2, f"expected 2 folders, got {len(results)}"
|
|
names = [r[1] for r in results]
|
|
assert "KeePass%202.x" in names
|
|
assert "Plugins" in names
|
|
|
|
# Files
|
|
pattern2 = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
|
|
file_results = [(m.group(1), m.group(2)) for m in re.finditer(pattern2, FIXTURE_TBODY_FILES_MIXED, re.DOTALL)]
|
|
assert len(file_results) == 3, f"expected 3 files, got {len(file_results)}"
|
|
file_paths = [r[0] for r in file_results]
|
|
has_config = any("config.sample.php" in p for p in file_paths)
|
|
assert has_config, f"config.sample.php not found among {file_paths}"
|
|
# Test absolute URL variant
|
|
assert any("config.sample.php" in p for p in file_paths), "absolute URL pattern should match"
|
|
|
|
print("OK test_list_dir_parse")
|
|
|
|
|
|
def test_is_junk():
|
|
assert sf.is_junk("sk-xxxx-example")
|
|
assert sf.is_junk("aaa") # too short / low entropy
|
|
assert not sf.is_junk("sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc")
|
|
print("OK test_is_junk")
|
|
|
|
|
|
def test_scan_text():
|
|
"""scan_text should find patterns in fake file."""
|
|
hits = sf.scan_text(FAKE_KEY_FILE)
|
|
assert len(hits) > 0, f"expected hits, got {hits}"
|
|
provs = [p for p, _ in hits]
|
|
# Should find OPENAI and STRIPE patterns
|
|
assert "OPENAI" in provs or "OPENAI_API_KEY" in provs, f"OPENAI not in {provs}"
|
|
print(f"OK test_scan_text — found {len(hits)} patterns: {provs}")
|
|
|
|
|
|
def test_db_add():
|
|
"""db_add should write a row and dedupe."""
|
|
tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
|
|
tmp.close()
|
|
try:
|
|
conn = sqlite3.connect(tmp.name, timeout=120)
|
|
try:
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE,
|
|
val TEXT, prov TEXT, status DEFAULT "NEW",
|
|
plan TEXT, price TEXT, remaining TEXT,
|
|
found TEXT, repo TEXT)""")
|
|
|
|
key = "sk-proj-abc123def456ghi789jkl012mno345pqr678stu901vwx234yz012abc"
|
|
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(h, key, "TEST", "NEW", "sourceforge", "test"))
|
|
conn.commit()
|
|
|
|
cur = conn.execute("SELECT val, prov, found FROM keys WHERE hash=?", (h,))
|
|
row = cur.fetchone()
|
|
assert row is not None, "row should exist"
|
|
assert row[0] == key
|
|
assert row[2] == "sourceforge"
|
|
print("OK test_db_add - row written")
|
|
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(h, key, "TEST", "NEW", "sourceforge", "test"))
|
|
conn.commit()
|
|
cur = conn.execute("SELECT COUNT(*) FROM keys WHERE hash=?", (h,))
|
|
assert cur.fetchone()[0] == 1, "dedupe should keep 1 row"
|
|
print("OK test_db_add - dedupe works")
|
|
finally:
|
|
conn.close()
|
|
finally:
|
|
os.unlink(tmp.name)
|
|
|
|
|
|
def test_garbage_filter():
|
|
"""Garbage strings should be filtered out."""
|
|
for junk in ("xxxx", "EXAMPLE_KEY", "your-api-key-here", "abcdefghijklmnopqrst"):
|
|
assert sf.is_junk(junk), f"{junk} should be junk"
|
|
print("OK test_garbage_filter")
|
|
|
|
|
|
def test_generator_pattern():
|
|
"""Verify the regex on the fixtures directly."""
|
|
import re
|
|
# Folder regex
|
|
esc = re.escape("keepass")
|
|
pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
|
|
matches = list(re.finditer(pattern, FIXTURE_TBODY_FOLDERS, re.DOTALL))
|
|
assert len(matches) == 2, f"folder regex: expected 2, got {len(matches)}"
|
|
assert "KeePass%202.x" in matches[0].group(1)
|
|
|
|
# File regex - test both relative and absolute href
|
|
pattern2 = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
|
|
matches2 = list(re.finditer(pattern2, FIXTURE_TBODY_FILES_MIXED, re.DOTALL))
|
|
assert len(matches2) == 3, f"file regex: expected 3, got {len(matches2)}"
|
|
# Absolute URL variant (https://sourceforge.net/...)
|
|
abs_matches = [m for m in matches2 if m.group(1).startswith("KeePass%202.x/2.61/config")]
|
|
assert len(abs_matches) == 1, "should match absolute URL file"
|
|
print("OK test_generator_pattern")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_generator_pattern()
|
|
test_list_dir_parse()
|
|
test_is_junk()
|
|
test_garbage_filter()
|
|
test_scan_text()
|
|
test_db_add()
|
|
print("\nAll tests PASSED") |