KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
345
wp_svn_eternal.py
Normal file
345
wp_svn_eternal.py
Normal file
@@ -0,0 +1,345 @@
|
||||
"""ETERNAL WP SVN: WordPress plugin SVN trunk walk.
|
||||
|
||||
Walks plugins.svn.wordpress.org/<slug>/trunk/ directory listings → fetches
|
||||
small text files (php, json, config, js, env, etc.) and scans for secrets.
|
||||
|
||||
Endpoint(s):
|
||||
https://api.wordpress.org/plugins/info/1.2/?action=query_plugins (seed slugs)
|
||||
https://plugins.svn.wordpress.org/<slug>/trunk/ (listing + files)
|
||||
Auth: none
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib, html, re, sqlite3, sys, threading, time
|
||||
import requests
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False # contract 4: bypass local dashscope proxy
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "wp_svn_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "wp_svn_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "wp_svn_seen.txt")
|
||||
N_THREADS = 4
|
||||
SLEEP = 90
|
||||
MAX_ITEMS = 200
|
||||
FILE_CAP = 500_000
|
||||
MAX_DEPTH = 2 # 0 = trunk only, 1 = trunk + one subdir level, etc.
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
SCAN_EXT = (".php", ".js", ".json", ".txt", ".env", ".xml", ".yaml", ".yml",
|
||||
".ini", ".cfg", ".md", ".log", ".conf", ".config", ".htaccess")
|
||||
|
||||
SKIP_DIRS = {"vendor", "node_modules", ".git", ".svn", "tests", "test",
|
||||
"assets", "images", "img", "css", "scss", "fonts", "i18n",
|
||||
"languages", "lang", "po", "svn", "bin", "build", "dist"}
|
||||
|
||||
UA = {"User-Agent": "keyfarm-wp-svn/1.0"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
return
|
||||
if len(set(key)) < 6:
|
||||
return
|
||||
try:
|
||||
lock.acquire()
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) ON CONFLICT(hash) DO UPDATE SET "
|
||||
"status='NEW', found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "wp-svn", repo[:300]))
|
||||
conn.commit()
|
||||
total += 1
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for ln in f:
|
||||
s.add(ln.strip())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE) as f:
|
||||
return int(f.read().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def slug_cache(limit=0):
|
||||
"""Return a dict slug->download_link from the plugins API.
|
||||
limit=0 means unlimited (up to 10000). --once uses limit=5."""
|
||||
out = {}
|
||||
want = limit or 10000
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
params = {"action": "query_plugins",
|
||||
"request[per_page]": str(min(want, 100)),
|
||||
"request[page]": "1",
|
||||
"request[browse]": "popular"}
|
||||
r = S.get(
|
||||
"https://api.wordpress.org/plugins/info/1.2/",
|
||||
params=params, headers=UA, timeout=40)
|
||||
if r.status_code == 200:
|
||||
break
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
return out
|
||||
time.sleep(8 * attempt)
|
||||
data = r.json()
|
||||
for p in data.get("plugins") or []:
|
||||
slug = p.get("slug")
|
||||
if slug and slug not in out:
|
||||
out[slug] = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
||||
if limit and len(out) >= limit:
|
||||
return out
|
||||
page = 1
|
||||
total_plugins = data.get("info", {}).get("results", 0)
|
||||
while len(out) < min(total_plugins, want) and page <= 100:
|
||||
page += 1
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
params["request[page]"] = str(page)
|
||||
r = S.get(
|
||||
"https://api.wordpress.org/plugins/info/1.2/",
|
||||
params=params, headers=UA, timeout=40)
|
||||
if r.status_code == 200:
|
||||
break
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
return out
|
||||
time.sleep(8 * attempt)
|
||||
data = r.json()
|
||||
for p in data.get("plugins") or []:
|
||||
slug = p.get("slug")
|
||||
if slug and slug not in out:
|
||||
out[slug] = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
||||
if limit and len(out) >= limit:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
def fetch_svn_list(url):
|
||||
"""Fetch SVN directory listing HTML, return (files, dirs)."""
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = S.get(url, headers=UA, timeout=30)
|
||||
if r.status_code == 200:
|
||||
break
|
||||
if r.status_code == 404:
|
||||
return [], []
|
||||
if r.status_code == 429:
|
||||
ra = r.headers.get("Retry-After", "10")
|
||||
try:
|
||||
time.sleep(int(ra))
|
||||
except ValueError:
|
||||
time.sleep(10)
|
||||
continue
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
return None, None # transient error
|
||||
time.sleep(4 * attempt)
|
||||
text = r.text
|
||||
files = []
|
||||
dirs = []
|
||||
for li in re.finditer(r'<li><a\s+href="([^"]+)">([^<]*)</a></li>', text):
|
||||
href = li.group(1)
|
||||
name = html.unescape(li.group(2))
|
||||
if name == ".." or not name:
|
||||
continue
|
||||
if href.endswith("/"):
|
||||
dirs.append(name.rstrip("/"))
|
||||
elif not href.startswith("?"):
|
||||
files.append(name)
|
||||
return files, dirs
|
||||
|
||||
|
||||
def should_scan(fn):
|
||||
lower = fn.lower()
|
||||
return any(lower.endswith(e) for e in SCAN_EXT)
|
||||
|
||||
|
||||
def fetch_and_scan(url, slug):
|
||||
"""Fetch a single file from SVN trunk and scan."""
|
||||
for attempt in (1, 2, 3):
|
||||
try:
|
||||
r = S.get(url, headers=UA, timeout=30)
|
||||
if r.status_code == 200:
|
||||
break
|
||||
if r.status_code in (404, 403, 410):
|
||||
return 0
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
except Exception:
|
||||
if attempt == 3:
|
||||
return -1
|
||||
time.sleep(4 * attempt)
|
||||
cl = int(r.headers.get("Content-Length") or 0)
|
||||
if cl > FILE_CAP:
|
||||
return 0
|
||||
text = r.text
|
||||
if not text or len(text) > FILE_CAP:
|
||||
return 0
|
||||
n = 0
|
||||
for prov, key in scan_text(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
if len(set(key)) < 6:
|
||||
continue
|
||||
db_add(key, prov, f"wp:{slug}")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def work(slug):
|
||||
"""Walk SVN trunk for one plugin, scan files. Returns key count or -1.
|
||||
--once: trunk root only, max 3 files."""
|
||||
base_url = f"https://plugins.svn.wordpress.org/{slug}/trunk/"
|
||||
files, dirs = fetch_svn_list(base_url)
|
||||
if files is None:
|
||||
return -1
|
||||
is_once = "--once" in sys.argv
|
||||
n = 0
|
||||
scanned = 0
|
||||
for f in files:
|
||||
if not should_scan(f):
|
||||
continue
|
||||
r = fetch_and_scan(base_url + f, slug)
|
||||
if r == -1:
|
||||
continue
|
||||
n += r
|
||||
scanned += 1
|
||||
if is_once and scanned >= 3:
|
||||
break
|
||||
if is_once:
|
||||
return n
|
||||
# Recurse into subdirectories (up to MAX_DEPTH)
|
||||
if MAX_DEPTH >= 1:
|
||||
for d in dirs:
|
||||
if d.lower() in SKIP_DIRS:
|
||||
continue
|
||||
sub_url = base_url + d + "/"
|
||||
sub_files, sub_dirs = fetch_svn_list(sub_url)
|
||||
if sub_files is None:
|
||||
continue
|
||||
for f in sub_files:
|
||||
if not should_scan(f):
|
||||
continue
|
||||
r = fetch_and_scan(sub_url + f, slug)
|
||||
if r == -1:
|
||||
continue
|
||||
n += r
|
||||
if MAX_DEPTH >= 2:
|
||||
for sd in sub_dirs:
|
||||
if sd.lower() in SKIP_DIRS:
|
||||
continue
|
||||
sd_url = sub_url + sd + "/"
|
||||
sd_files, _ = fetch_svn_list(sd_url)
|
||||
if sd_files is None:
|
||||
continue
|
||||
for f in sd_files:
|
||||
if not should_scan(f):
|
||||
continue
|
||||
r = fetch_and_scan(sd_url + f, slug)
|
||||
if r == -1:
|
||||
continue
|
||||
n += r
|
||||
return n
|
||||
|
||||
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
is_once = "--once" in sys.argv
|
||||
slugs = slug_cache(limit=2 if is_once else 0)
|
||||
items = [s for s in slugs if s not in seen]
|
||||
cursor = load_cursor()
|
||||
if cursor < len(items):
|
||||
batch = items[cursor:cursor + MAX_ITEMS]
|
||||
else:
|
||||
cursor = 0
|
||||
save_cursor(0)
|
||||
batch = items[:MAX_ITEMS]
|
||||
if not batch:
|
||||
return
|
||||
ok = fail = 0
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, s): s for s in batch}
|
||||
for fut in as_completed(futs):
|
||||
s = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception:
|
||||
res = -1
|
||||
if res == -1:
|
||||
fail += 1
|
||||
else:
|
||||
ok += 1
|
||||
mark_seen(s)
|
||||
save_cursor(cursor + len(batch))
|
||||
log(f"cycle: slugs={len(slugs)} scanned={len(batch)} ok={ok} "
|
||||
f"fail={fail} +{total} keys")
|
||||
|
||||
|
||||
def main():
|
||||
log("wp_svn_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle-err {type(e).__name__}: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user