352 lines
10 KiB
Python
352 lines
10 KiB
Python
"""ETERNAL VS Code Marketplace: walk all VS Code extensions via gallery API,
|
|
download VSIX, unzip in memory, scan bundled files for API keys/config secrets.
|
|
|
|
Endpoint: POST https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery
|
|
(anonymous, per-IP rate limit, 150K+ extensions)
|
|
Auth: none
|
|
"""
|
|
import os as _os
|
|
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
|
|
import hashlib, io, re, sqlite3, sys, threading, time, zipfile
|
|
import requests
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
|
LOG = _os.path.join(BASE, "data", "vscode_eternal.log")
|
|
CURSOR_FILE = _os.path.join(BASE, "data", "vscode_cursor.txt")
|
|
SEEN_FILE = _os.path.join(BASE, "data", "vscode_seen.txt")
|
|
N_THREADS = 3
|
|
SLEEP = 90
|
|
MAX_ITEMS = 30
|
|
VSIX_CAP = 20 * 1024 * 1024 # 20 MB max VSIX download
|
|
FILE_CAP = 500_000
|
|
PAGE_SIZE = 50
|
|
|
|
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef")
|
|
|
|
# File extensions to scan inside VSIX
|
|
SCAN_EXT = (".js", ".json", ".ts", ".env", ".yaml", ".yml", ".xml",
|
|
".html", ".txt", ".cfg", ".conf", ".properties", ".ini",
|
|
".sh", ".bat", ".ps1", ".py", ".rb", ".php")
|
|
|
|
# Path segments inside VSIX to skip
|
|
SKIP_DIRS = ("/node_modules/", "/.git/", "/test/", "/tests/")
|
|
|
|
# Very long single-line JS (minified bundle) — likely noise but may still
|
|
# contain keys; we scan them but skip scanning rows > 100 chars content
|
|
MINIFIED_LINE_THRESHOLD = 5000
|
|
|
|
UA = {"User-Agent": "keyfarm-vscode/1.0"}
|
|
GALLERY_URL = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery"
|
|
GALLERY_HEADERS = {"Accept": "application/json;api-version=7.2-preview.1", **UA}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE,
|
|
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
|
plan TEXT, price TEXT, remaining TEXT,
|
|
found TEXT, repo TEXT)
|
|
""")
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
with lock:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "vscode", repo))
|
|
total += 1
|
|
|
|
|
|
def load_cursor():
|
|
try:
|
|
with open(CURSOR_FILE, encoding="utf-8") as f:
|
|
return int(f.read().strip() or 0)
|
|
except (FileNotFoundError, ValueError):
|
|
return 0
|
|
|
|
|
|
def save_cursor(c):
|
|
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
|
f.write(str(c))
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
s.add(line)
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def is_junk(v):
|
|
if any(g in v for g in GARBAGE):
|
|
return True
|
|
if len(set(v)) < 6:
|
|
return True
|
|
return False
|
|
|
|
|
|
def fetch_page(page):
|
|
"""Fetch one page of extensions from the gallery API."""
|
|
body = {
|
|
"filters": [{
|
|
"criteria": [
|
|
{"filterType": 8, "value": "Microsoft.VisualStudio.Code"}
|
|
],
|
|
"pageNumber": page,
|
|
"pageSize": PAGE_SIZE
|
|
}],
|
|
"flags": 0x206 # IncludeVersions | IncludeFiles | IncludeVersionProperties
|
|
}
|
|
for attempt in range(3):
|
|
try:
|
|
r = S.post(GALLERY_URL, json=body, headers=GALLERY_HEADERS,
|
|
timeout=30)
|
|
if r.status_code == 429:
|
|
retry = int(r.headers.get("Retry-After", 10))
|
|
log(f" 429 on page {page}, sleeping {retry}s")
|
|
time.sleep(retry)
|
|
continue
|
|
r.raise_for_status()
|
|
data = r.json()
|
|
exts = (data.get("results") or [{}])[0].get("extensions") or []
|
|
return exts
|
|
except requests.exceptions.RequestException as e:
|
|
if attempt < 2:
|
|
time.sleep(2 ** (attempt + 1))
|
|
else:
|
|
log(f" page {page} fail after 3 retries: {e}")
|
|
return None
|
|
return None
|
|
|
|
|
|
def download_vsix(ext_id, vsix_url):
|
|
"""Download VSIX with cap-before-download via HEAD then stream.
|
|
Returns bytes or None on fail/oversize.
|
|
"""
|
|
for attempt in range(3):
|
|
try:
|
|
head = S.head(vsix_url, headers=UA, timeout=15)
|
|
cl = int(head.headers.get("Content-Length", 0) or 0)
|
|
if cl > VSIX_CAP:
|
|
log(f" {ext_id}: Content-Length {cl} > {VSIX_CAP}, skip")
|
|
return None
|
|
|
|
if cl > 0:
|
|
r = S.get(vsix_url, headers=UA, timeout=120, stream=True)
|
|
r.raise_for_status()
|
|
chunks = []
|
|
downloaded = 0
|
|
for chunk in r.iter_content(chunk_size=65536):
|
|
downloaded += len(chunk)
|
|
if downloaded > VSIX_CAP:
|
|
log(f" {ext_id}: stream exceeded cap")
|
|
return None
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
else:
|
|
# Content-Length 0 or missing — download directly with cap
|
|
r = S.get(vsix_url, headers=UA, timeout=120)
|
|
r.raise_for_status()
|
|
if len(r.content) > VSIX_CAP:
|
|
log(f" {ext_id}: body {len(r.content)} > cap")
|
|
return None
|
|
return r.content
|
|
except requests.exceptions.RequestException as e:
|
|
if attempt < 2:
|
|
time.sleep(2 ** (attempt + 1))
|
|
else:
|
|
log(f" {ext_id}: VSIX download fail: {e}")
|
|
return None
|
|
return None
|
|
|
|
|
|
def scan_vsix(raw, ext_key):
|
|
"""Unzip VSIX in memory, scan text files for keys.
|
|
Returns number of keys found.
|
|
"""
|
|
try:
|
|
z = zipfile.ZipFile(io.BytesIO(raw))
|
|
except zipfile.BadZipFile:
|
|
log(f" {ext_key}: bad zip")
|
|
return 0
|
|
|
|
n = 0
|
|
for info in z.infolist():
|
|
fn = info.filename
|
|
# Skip node_modules, .git, test dirs
|
|
if any(sd in fn for sd in SKIP_DIRS):
|
|
continue
|
|
# Only scan relevant extensions
|
|
if not fn.endswith(SCAN_EXT):
|
|
continue
|
|
# Skip huge files (but check member size first — don't read then slice)
|
|
if info.file_size > FILE_CAP:
|
|
continue
|
|
# Skip minified single-line bundles (high noise)
|
|
if info.file_size > MINIFIED_LINE_THRESHOLD:
|
|
# Could still be worth checking the first few bytes for key patterns
|
|
# But full scan of minified JS is very noisy
|
|
pass
|
|
|
|
try:
|
|
content = z.read(info.filename)
|
|
except Exception:
|
|
continue
|
|
|
|
# Try decode
|
|
try:
|
|
text = content.decode("utf-8", "replace")
|
|
except Exception:
|
|
continue
|
|
|
|
# For long single-line content, take first 200KB to avoid noise
|
|
# but keep scanning for patterns
|
|
if len(text) > FILE_CAP:
|
|
text = text[:FILE_CAP]
|
|
|
|
hits = scan_text(text)
|
|
for prov, key in hits:
|
|
if not is_junk(key):
|
|
# Use extension key as repo identifier
|
|
db_add(key, prov, ext_key)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def work(ext):
|
|
"""Process one extension: download VSIX, unzip, scan. Returns key count."""
|
|
pub = ext.get("publisher", {}).get("publisherName", "?")
|
|
name = ext.get("extensionName", "?")
|
|
ver_obj = (ext.get("versions") or [{}])[0]
|
|
ver = ver_obj.get("version", "?")
|
|
ext_key = f"{pub}/{name}/{ver}"
|
|
# extract a shorter ext_ident for repo column (max 300 chars)
|
|
repo_slug = f"{pub}/{name}"
|
|
|
|
if ext_key in seen:
|
|
return 0
|
|
|
|
fb = ver_obj.get("fallbackAssetUri") or ver_obj.get("assetUri")
|
|
if not fb:
|
|
log(f" {ext_key}: no assetUri, skip")
|
|
mark_seen(ext_key)
|
|
return 0
|
|
|
|
vsix_url = f"{fb}/Microsoft.VisualStudio.Services.VSIXPackage"
|
|
|
|
raw = download_vsix(ext_key, vsix_url)
|
|
if raw is None:
|
|
mark_seen(ext_key)
|
|
return 0
|
|
|
|
n = scan_vsix(raw, repo_slug)
|
|
if n > 0:
|
|
log(f" {ext_key}: +{n} keys")
|
|
mark_seen(ext_key)
|
|
return n
|
|
|
|
|
|
def cycle(max_items=MAX_ITEMS):
|
|
"""Run one bounded cycle: walk pages from cursor."""
|
|
global total, seen
|
|
|
|
page = load_cursor()
|
|
if page < 1:
|
|
page = 1
|
|
processed = 0
|
|
fail = 0
|
|
|
|
log(f"cycle start: page={page}, max_items={max_items}")
|
|
|
|
while processed < max_items:
|
|
exts = fetch_page(page)
|
|
if exts is None:
|
|
log(f" page {page} returned no extensions, wrap to page 1")
|
|
page = 1
|
|
save_cursor(page)
|
|
continue
|
|
if not exts:
|
|
log(f" page {page} empty, wrap to page 1")
|
|
page = 1
|
|
save_cursor(page)
|
|
continue
|
|
|
|
per_page = min(len(exts), max_items - processed)
|
|
for i in range(per_page):
|
|
try:
|
|
n = work(exts[i])
|
|
if n > 0:
|
|
processed += 1
|
|
else:
|
|
processed += 1 # still counts as processed
|
|
except Exception as e:
|
|
log(f" error processing {exts[i].get('extensionName','?')}: {e}")
|
|
fail += 1
|
|
processed += 1
|
|
|
|
if processed >= max_items:
|
|
break
|
|
|
|
page += 1
|
|
save_cursor(page)
|
|
# Brief pause between pages
|
|
time.sleep(1)
|
|
|
|
log(f"cycle done: page={page} processed={processed} fail={fail} +{total} keys")
|
|
|
|
|
|
def main():
|
|
log("vscode_eternal started")
|
|
if "--once" in sys.argv:
|
|
cycle(max_items=min(MAX_ITEMS, 5))
|
|
log("--once complete")
|
|
return
|
|
while True:
|
|
cycle()
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |