KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
352
vscode_eternal.py
Normal file
352
vscode_eternal.py
Normal file
@@ -0,0 +1,352 @@
|
||||
"""ETERNAL VS Code Marketplace: walk all VS Code extensions via gallery API,
|
||||
download VSIX, unzip in memory, scan bundled files for API keys/config secrets.
|
||||
|
||||
Endpoint: POST https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery
|
||||
(anonymous, per-IP rate limit, 150K+ extensions)
|
||||
Auth: none
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
|
||||
import hashlib, io, re, sqlite3, sys, threading, time, zipfile
|
||||
import requests
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "vscode_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "vscode_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "vscode_seen.txt")
|
||||
N_THREADS = 3
|
||||
SLEEP = 90
|
||||
MAX_ITEMS = 30
|
||||
VSIX_CAP = 20 * 1024 * 1024 # 20 MB max VSIX download
|
||||
FILE_CAP = 500_000
|
||||
PAGE_SIZE = 50
|
||||
|
||||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
# File extensions to scan inside VSIX
|
||||
SCAN_EXT = (".js", ".json", ".ts", ".env", ".yaml", ".yml", ".xml",
|
||||
".html", ".txt", ".cfg", ".conf", ".properties", ".ini",
|
||||
".sh", ".bat", ".ps1", ".py", ".rb", ".php")
|
||||
|
||||
# Path segments inside VSIX to skip
|
||||
SKIP_DIRS = ("/node_modules/", "/.git/", "/test/", "/tests/")
|
||||
|
||||
# Very long single-line JS (minified bundle) — likely noise but may still
|
||||
# contain keys; we scan them but skip scanning rows > 100 chars content
|
||||
MINIFIED_LINE_THRESHOLD = 5000
|
||||
|
||||
UA = {"User-Agent": "keyfarm-vscode/1.0"}
|
||||
GALLERY_URL = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery"
|
||||
GALLERY_HEADERS = {"Accept": "application/json;api-version=7.2-preview.1", **UA}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE,
|
||||
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||||
plan TEXT, price TEXT, remaining TEXT,
|
||||
found TEXT, repo TEXT)
|
||||
""")
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "vscode", repo))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||||
return int(f.read().strip() or 0)
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
s.add(line)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetch_page(page):
|
||||
"""Fetch one page of extensions from the gallery API."""
|
||||
body = {
|
||||
"filters": [{
|
||||
"criteria": [
|
||||
{"filterType": 8, "value": "Microsoft.VisualStudio.Code"}
|
||||
],
|
||||
"pageNumber": page,
|
||||
"pageSize": PAGE_SIZE
|
||||
}],
|
||||
"flags": 0x206 # IncludeVersions | IncludeFiles | IncludeVersionProperties
|
||||
}
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = S.post(GALLERY_URL, json=body, headers=GALLERY_HEADERS,
|
||||
timeout=30)
|
||||
if r.status_code == 429:
|
||||
retry = int(r.headers.get("Retry-After", 10))
|
||||
log(f" 429 on page {page}, sleeping {retry}s")
|
||||
time.sleep(retry)
|
||||
continue
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
exts = (data.get("results") or [{}])[0].get("extensions") or []
|
||||
return exts
|
||||
except requests.exceptions.RequestException as e:
|
||||
if attempt < 2:
|
||||
time.sleep(2 ** (attempt + 1))
|
||||
else:
|
||||
log(f" page {page} fail after 3 retries: {e}")
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def download_vsix(ext_id, vsix_url):
|
||||
"""Download VSIX with cap-before-download via HEAD then stream.
|
||||
Returns bytes or None on fail/oversize.
|
||||
"""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
head = S.head(vsix_url, headers=UA, timeout=15)
|
||||
cl = int(head.headers.get("Content-Length", 0) or 0)
|
||||
if cl > VSIX_CAP:
|
||||
log(f" {ext_id}: Content-Length {cl} > {VSIX_CAP}, skip")
|
||||
return None
|
||||
|
||||
if cl > 0:
|
||||
r = S.get(vsix_url, headers=UA, timeout=120, stream=True)
|
||||
r.raise_for_status()
|
||||
chunks = []
|
||||
downloaded = 0
|
||||
for chunk in r.iter_content(chunk_size=65536):
|
||||
downloaded += len(chunk)
|
||||
if downloaded > VSIX_CAP:
|
||||
log(f" {ext_id}: stream exceeded cap")
|
||||
return None
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
else:
|
||||
# Content-Length 0 or missing — download directly with cap
|
||||
r = S.get(vsix_url, headers=UA, timeout=120)
|
||||
r.raise_for_status()
|
||||
if len(r.content) > VSIX_CAP:
|
||||
log(f" {ext_id}: body {len(r.content)} > cap")
|
||||
return None
|
||||
return r.content
|
||||
except requests.exceptions.RequestException as e:
|
||||
if attempt < 2:
|
||||
time.sleep(2 ** (attempt + 1))
|
||||
else:
|
||||
log(f" {ext_id}: VSIX download fail: {e}")
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def scan_vsix(raw, ext_key):
|
||||
"""Unzip VSIX in memory, scan text files for keys.
|
||||
Returns number of keys found.
|
||||
"""
|
||||
try:
|
||||
z = zipfile.ZipFile(io.BytesIO(raw))
|
||||
except zipfile.BadZipFile:
|
||||
log(f" {ext_key}: bad zip")
|
||||
return 0
|
||||
|
||||
n = 0
|
||||
for info in z.infolist():
|
||||
fn = info.filename
|
||||
# Skip node_modules, .git, test dirs
|
||||
if any(sd in fn for sd in SKIP_DIRS):
|
||||
continue
|
||||
# Only scan relevant extensions
|
||||
if not fn.endswith(SCAN_EXT):
|
||||
continue
|
||||
# Skip huge files (but check member size first — don't read then slice)
|
||||
if info.file_size > FILE_CAP:
|
||||
continue
|
||||
# Skip minified single-line bundles (high noise)
|
||||
if info.file_size > MINIFIED_LINE_THRESHOLD:
|
||||
# Could still be worth checking the first few bytes for key patterns
|
||||
# But full scan of minified JS is very noisy
|
||||
pass
|
||||
|
||||
try:
|
||||
content = z.read(info.filename)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Try decode
|
||||
try:
|
||||
text = content.decode("utf-8", "replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# For long single-line content, take first 200KB to avoid noise
|
||||
# but keep scanning for patterns
|
||||
if len(text) > FILE_CAP:
|
||||
text = text[:FILE_CAP]
|
||||
|
||||
hits = scan_text(text)
|
||||
for prov, key in hits:
|
||||
if not is_junk(key):
|
||||
# Use extension key as repo identifier
|
||||
db_add(key, prov, ext_key)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def work(ext):
|
||||
"""Process one extension: download VSIX, unzip, scan. Returns key count."""
|
||||
pub = ext.get("publisher", {}).get("publisherName", "?")
|
||||
name = ext.get("extensionName", "?")
|
||||
ver_obj = (ext.get("versions") or [{}])[0]
|
||||
ver = ver_obj.get("version", "?")
|
||||
ext_key = f"{pub}/{name}/{ver}"
|
||||
# extract a shorter ext_ident for repo column (max 300 chars)
|
||||
repo_slug = f"{pub}/{name}"
|
||||
|
||||
if ext_key in seen:
|
||||
return 0
|
||||
|
||||
fb = ver_obj.get("fallbackAssetUri") or ver_obj.get("assetUri")
|
||||
if not fb:
|
||||
log(f" {ext_key}: no assetUri, skip")
|
||||
mark_seen(ext_key)
|
||||
return 0
|
||||
|
||||
vsix_url = f"{fb}/Microsoft.VisualStudio.Services.VSIXPackage"
|
||||
|
||||
raw = download_vsix(ext_key, vsix_url)
|
||||
if raw is None:
|
||||
mark_seen(ext_key)
|
||||
return 0
|
||||
|
||||
n = scan_vsix(raw, repo_slug)
|
||||
if n > 0:
|
||||
log(f" {ext_key}: +{n} keys")
|
||||
mark_seen(ext_key)
|
||||
return n
|
||||
|
||||
|
||||
def cycle(max_items=MAX_ITEMS):
|
||||
"""Run one bounded cycle: walk pages from cursor."""
|
||||
global total, seen
|
||||
|
||||
page = load_cursor()
|
||||
if page < 1:
|
||||
page = 1
|
||||
processed = 0
|
||||
fail = 0
|
||||
|
||||
log(f"cycle start: page={page}, max_items={max_items}")
|
||||
|
||||
while processed < max_items:
|
||||
exts = fetch_page(page)
|
||||
if exts is None:
|
||||
log(f" page {page} returned no extensions, wrap to page 1")
|
||||
page = 1
|
||||
save_cursor(page)
|
||||
continue
|
||||
if not exts:
|
||||
log(f" page {page} empty, wrap to page 1")
|
||||
page = 1
|
||||
save_cursor(page)
|
||||
continue
|
||||
|
||||
per_page = min(len(exts), max_items - processed)
|
||||
for i in range(per_page):
|
||||
try:
|
||||
n = work(exts[i])
|
||||
if n > 0:
|
||||
processed += 1
|
||||
else:
|
||||
processed += 1 # still counts as processed
|
||||
except Exception as e:
|
||||
log(f" error processing {exts[i].get('extensionName','?')}: {e}")
|
||||
fail += 1
|
||||
processed += 1
|
||||
|
||||
if processed >= max_items:
|
||||
break
|
||||
|
||||
page += 1
|
||||
save_cursor(page)
|
||||
# Brief pause between pages
|
||||
time.sleep(1)
|
||||
|
||||
log(f"cycle done: page={page} processed={processed} fail={fail} +{total} keys")
|
||||
|
||||
|
||||
def main():
|
||||
log("vscode_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle(max_items=min(MAX_ITEMS, 5))
|
||||
log("--once complete")
|
||||
return
|
||||
while True:
|
||||
cycle()
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user