345 lines
10 KiB
Python
345 lines
10 KiB
Python
"""ETERNAL WP SVN: WordPress plugin SVN trunk walk.
|
|
|
|
Walks plugins.svn.wordpress.org/<slug>/trunk/ directory listings → fetches
|
|
small text files (php, json, config, js, env, etc.) and scans for secrets.
|
|
|
|
Endpoint(s):
|
|
https://api.wordpress.org/plugins/info/1.2/?action=query_plugins (seed slugs)
|
|
https://plugins.svn.wordpress.org/<slug>/trunk/ (listing + files)
|
|
Auth: none
|
|
"""
|
|
import os as _os
|
|
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
|
|
|
import hashlib, html, re, sqlite3, sys, threading, time
|
|
import requests
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False # contract 4: bypass local dashscope proxy
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
|
LOG = _os.path.join(BASE, "data", "wp_svn_eternal.log")
|
|
CURSOR_FILE = _os.path.join(BASE, "data", "wp_svn_cursor.txt")
|
|
SEEN_FILE = _os.path.join(BASE, "data", "wp_svn_seen.txt")
|
|
N_THREADS = 4
|
|
SLEEP = 90
|
|
MAX_ITEMS = 200
|
|
FILE_CAP = 500_000
|
|
MAX_DEPTH = 2 # 0 = trunk only, 1 = trunk + one subdir level, etc.
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef")
|
|
|
|
SCAN_EXT = (".php", ".js", ".json", ".txt", ".env", ".xml", ".yaml", ".yml",
|
|
".ini", ".cfg", ".md", ".log", ".conf", ".config", ".htaccess")
|
|
|
|
SKIP_DIRS = {"vendor", "node_modules", ".git", ".svn", "tests", "test",
|
|
"assets", "images", "img", "css", "scss", "fonts", "i18n",
|
|
"languages", "lang", "po", "svn", "bin", "build", "dist"}
|
|
|
|
UA = {"User-Agent": "keyfarm-wp-svn/1.0"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
return
|
|
if len(set(key)) < 6:
|
|
return
|
|
try:
|
|
lock.acquire()
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) ON CONFLICT(hash) DO UPDATE SET "
|
|
"status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "wp-svn", repo[:300]))
|
|
conn.commit()
|
|
total += 1
|
|
finally:
|
|
lock.release()
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for ln in f:
|
|
s.add(ln.strip())
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def load_cursor():
|
|
try:
|
|
with open(CURSOR_FILE) as f:
|
|
return int(f.read().strip())
|
|
except (FileNotFoundError, ValueError):
|
|
return 0
|
|
|
|
|
|
def save_cursor(c):
|
|
with open(CURSOR_FILE, "w") as f:
|
|
f.write(str(c))
|
|
|
|
|
|
def slug_cache(limit=0):
|
|
"""Return a dict slug->download_link from the plugins API.
|
|
limit=0 means unlimited (up to 10000). --once uses limit=5."""
|
|
out = {}
|
|
want = limit or 10000
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
params = {"action": "query_plugins",
|
|
"request[per_page]": str(min(want, 100)),
|
|
"request[page]": "1",
|
|
"request[browse]": "popular"}
|
|
r = S.get(
|
|
"https://api.wordpress.org/plugins/info/1.2/",
|
|
params=params, headers=UA, timeout=40)
|
|
if r.status_code == 200:
|
|
break
|
|
except Exception:
|
|
if attempt == 3:
|
|
return out
|
|
time.sleep(8 * attempt)
|
|
data = r.json()
|
|
for p in data.get("plugins") or []:
|
|
slug = p.get("slug")
|
|
if slug and slug not in out:
|
|
out[slug] = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
|
if limit and len(out) >= limit:
|
|
return out
|
|
page = 1
|
|
total_plugins = data.get("info", {}).get("results", 0)
|
|
while len(out) < min(total_plugins, want) and page <= 100:
|
|
page += 1
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
params["request[page]"] = str(page)
|
|
r = S.get(
|
|
"https://api.wordpress.org/plugins/info/1.2/",
|
|
params=params, headers=UA, timeout=40)
|
|
if r.status_code == 200:
|
|
break
|
|
except Exception:
|
|
if attempt == 3:
|
|
return out
|
|
time.sleep(8 * attempt)
|
|
data = r.json()
|
|
for p in data.get("plugins") or []:
|
|
slug = p.get("slug")
|
|
if slug and slug not in out:
|
|
out[slug] = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
|
if limit and len(out) >= limit:
|
|
return out
|
|
return out
|
|
|
|
|
|
def fetch_svn_list(url):
|
|
"""Fetch SVN directory listing HTML, return (files, dirs)."""
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = S.get(url, headers=UA, timeout=30)
|
|
if r.status_code == 200:
|
|
break
|
|
if r.status_code == 404:
|
|
return [], []
|
|
if r.status_code == 429:
|
|
ra = r.headers.get("Retry-After", "10")
|
|
try:
|
|
time.sleep(int(ra))
|
|
except ValueError:
|
|
time.sleep(10)
|
|
continue
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
except Exception:
|
|
if attempt == 3:
|
|
return None, None # transient error
|
|
time.sleep(4 * attempt)
|
|
text = r.text
|
|
files = []
|
|
dirs = []
|
|
for li in re.finditer(r'<li><a\s+href="([^"]+)">([^<]*)</a></li>', text):
|
|
href = li.group(1)
|
|
name = html.unescape(li.group(2))
|
|
if name == ".." or not name:
|
|
continue
|
|
if href.endswith("/"):
|
|
dirs.append(name.rstrip("/"))
|
|
elif not href.startswith("?"):
|
|
files.append(name)
|
|
return files, dirs
|
|
|
|
|
|
def should_scan(fn):
|
|
lower = fn.lower()
|
|
return any(lower.endswith(e) for e in SCAN_EXT)
|
|
|
|
|
|
def fetch_and_scan(url, slug):
|
|
"""Fetch a single file from SVN trunk and scan."""
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = S.get(url, headers=UA, timeout=30)
|
|
if r.status_code == 200:
|
|
break
|
|
if r.status_code in (404, 403, 410):
|
|
return 0
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
except Exception:
|
|
if attempt == 3:
|
|
return -1
|
|
time.sleep(4 * attempt)
|
|
cl = int(r.headers.get("Content-Length") or 0)
|
|
if cl > FILE_CAP:
|
|
return 0
|
|
text = r.text
|
|
if not text or len(text) > FILE_CAP:
|
|
return 0
|
|
n = 0
|
|
for prov, key in scan_text(text):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
if len(set(key)) < 6:
|
|
continue
|
|
db_add(key, prov, f"wp:{slug}")
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def work(slug):
|
|
"""Walk SVN trunk for one plugin, scan files. Returns key count or -1.
|
|
--once: trunk root only, max 3 files."""
|
|
base_url = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
|
files, dirs = fetch_svn_list(base_url)
|
|
if files is None:
|
|
return -1
|
|
is_once = "--once" in sys.argv
|
|
n = 0
|
|
scanned = 0
|
|
for f in files:
|
|
if not should_scan(f):
|
|
continue
|
|
r = fetch_and_scan(base_url + f, slug)
|
|
if r == -1:
|
|
continue
|
|
n += r
|
|
scanned += 1
|
|
if is_once and scanned >= 3:
|
|
break
|
|
if is_once:
|
|
return n
|
|
# Recurse into subdirectories (up to MAX_DEPTH)
|
|
if MAX_DEPTH >= 1:
|
|
for d in dirs:
|
|
if d.lower() in SKIP_DIRS:
|
|
continue
|
|
sub_url = base_url + d + "/"
|
|
sub_files, sub_dirs = fetch_svn_list(sub_url)
|
|
if sub_files is None:
|
|
continue
|
|
for f in sub_files:
|
|
if not should_scan(f):
|
|
continue
|
|
r = fetch_and_scan(sub_url + f, slug)
|
|
if r == -1:
|
|
continue
|
|
n += r
|
|
if MAX_DEPTH >= 2:
|
|
for sd in sub_dirs:
|
|
if sd.lower() in SKIP_DIRS:
|
|
continue
|
|
sd_url = sub_url + sd + "/"
|
|
sd_files, _ = fetch_svn_list(sd_url)
|
|
if sd_files is None:
|
|
continue
|
|
for f in sd_files:
|
|
if not should_scan(f):
|
|
continue
|
|
r = fetch_and_scan(sd_url + f, slug)
|
|
if r == -1:
|
|
continue
|
|
n += r
|
|
return n
|
|
|
|
|
|
def cycle():
|
|
global total
|
|
total = 0
|
|
is_once = "--once" in sys.argv
|
|
slugs = slug_cache(limit=2 if is_once else 0)
|
|
items = [s for s in slugs if s not in seen]
|
|
cursor = load_cursor()
|
|
if cursor < len(items):
|
|
batch = items[cursor:cursor + MAX_ITEMS]
|
|
else:
|
|
cursor = 0
|
|
save_cursor(0)
|
|
batch = items[:MAX_ITEMS]
|
|
if not batch:
|
|
return
|
|
ok = fail = 0
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
|
futs = {ex.submit(work, s): s for s in batch}
|
|
for fut in as_completed(futs):
|
|
s = futs[fut]
|
|
try:
|
|
res = fut.result()
|
|
except Exception:
|
|
res = -1
|
|
if res == -1:
|
|
fail += 1
|
|
else:
|
|
ok += 1
|
|
mark_seen(s)
|
|
save_cursor(cursor + len(batch))
|
|
log(f"cycle: slugs={len(slugs)} scanned={len(batch)} ok={ok} "
|
|
f"fail={fail} +{total} keys")
|
|
|
|
|
|
def main():
|
|
log("wp_svn_eternal started")
|
|
if "--once" in sys.argv:
|
|
cycle()
|
|
return
|
|
while True:
|
|
try:
|
|
cycle()
|
|
except Exception as e:
|
|
log(f"cycle-err {type(e).__name__}: {e}")
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |