412 lines
13 KiB
Python
412 lines
13 KiB
Python
"""ETERNAL Wayback Machine: CDX prefix crawl over GitHub orgs with leak history.
|
|
|
|
Walks Internet Archive CDX with matchType=prefix over raw.githubusercontent.com/<ORG>/
|
|
for orgs mined from keys.db (top by historical key count, cached in
|
|
data/wayback_orgs.txt). Server-side filter original:.*/\.env + statuscode:200
|
|
(broad regex filters → CDX 503, proven live 2026-10-19; matchType=domain —
|
|
dead, timeouts). Fetches raw archived content, scans for keys.
|
|
pastebin removed: CDX does not index pastebin content. gist: TODO if needed.
|
|
|
|
Endpoint(s): http://web.archive.org/cdx/search/cdx (CDX index API)
|
|
https://web.archive.org/web/<ts>id_/<url> (raw archived content)
|
|
Auth: none
|
|
Flags: --once (single small cycle), --remine (rebuild data/wayback_orgs.txt)
|
|
"""
|
|
import os as _os
|
|
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
|
|
|
import hashlib, re, sqlite3, sys, threading, time
|
|
import requests
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False # contract 4: bypass local dashscope proxy
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
|
LOG = _os.path.join(BASE, "data", "wayback_eternal.log")
|
|
CURSOR_FILE = _os.path.join(BASE, "data", "wayback_cursor.txt")
|
|
SEEN_FILE = _os.path.join(BASE, "data", "wayback_seen.txt")
|
|
N_THREADS = 1
|
|
SLEEP = 90
|
|
MAX_ITEMS = 200
|
|
CDX_LIMIT = 5000
|
|
FILE_CAP = 500_000
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef")
|
|
|
|
UA = {"User-Agent": "keyfarm-wayback/1.0"}
|
|
|
|
ORGS_FILE = _os.path.join(BASE, "data", "wayback_orgs.txt")
|
|
# Server-side CDX filters: narrow single-pattern each (wide alternation → 503,
|
|
# proven 2026-10-19). Rotated per org cycle; all pass the is_config_url gate.
|
|
CDX_FILTERS = (
|
|
"original:.*/\\.env",
|
|
"original:.*/\\.npmrc",
|
|
"original:.*/\\.netrc",
|
|
"original:.*/\\.git-credentials",
|
|
"original:.*credentials\\.json",
|
|
"original:.*/config\\.json",
|
|
)
|
|
# Fallback if keys.db mining fails
|
|
SEED_ORGS = ["dpangestuw", "moleway", "sse-casb-lab"]
|
|
|
|
# Config-like filenames that may harbour secrets / tokens
|
|
CONFIG_NAMES = frozenset([
|
|
".env", ".env.example", ".env.production", ".env.local",
|
|
".env.dev", ".env.staging", ".envrc", ".env.template", ".env.sample",
|
|
"credentials", "credentials.json", "credentials.yml", "credentials.yaml",
|
|
"config.js", "config.json",
|
|
"application.properties",
|
|
".npmrc",
|
|
"id_rsa", "id_rsa.pub", "id_ed25519", "id_ecdsa",
|
|
"secret", "secrets", "secrets.json", "secrets.yml", "secrets.yaml",
|
|
".netrc", ".git-credentials", ".dockercfg",
|
|
"composer.json",
|
|
])
|
|
|
|
# Match a URL path's final segment against CONFIG_NAMES and also catch
|
|
# paths like /foo/.env or /foo/credentials.json via substring check.
|
|
_CONFIG_RE = re.compile(
|
|
r'/(?:'
|
|
r'\.env(?:\.\w+)?|'
|
|
r'credentials(?:\.\w+)?|'
|
|
r'config\.(?:js|json)|'
|
|
r'application\.properties|'
|
|
r'\.npmrc|'
|
|
r'id_(?:rsa|ed25519|ecdsa)(?:\.pub)?|'
|
|
r'secret(?:s)?(?:\.\w+)?|'
|
|
r'\.netrc|'
|
|
r'\.git-credentials|'
|
|
r'\.dockercfg|'
|
|
r'composer\.json'
|
|
r')(?:\?.*)?$',
|
|
re.IGNORECASE
|
|
)
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE,
|
|
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
|
plan TEXT, price TEXT, remaining TEXT,
|
|
found TEXT, repo TEXT)
|
|
""")
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
with lock:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "wayback", repo[:300]))
|
|
total += 1
|
|
|
|
|
|
def load_cursor():
|
|
"""Returns (domain_idx, offset)."""
|
|
try:
|
|
with open(CURSOR_FILE, encoding="utf-8") as f:
|
|
raw = f.read().strip()
|
|
if raw:
|
|
parts = raw.split()
|
|
return int(parts[0]), int(parts[1])
|
|
except (FileNotFoundError, ValueError, IndexError):
|
|
pass
|
|
return 0, 0
|
|
|
|
|
|
def save_cursor(domain_idx, offset):
|
|
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
|
f.write(f"{domain_idx} {offset}")
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
s.add(line)
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(url):
|
|
seen.add(url)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(url + "\n")
|
|
|
|
|
|
def is_junk(v):
|
|
if any(g in v for g in GARBAGE):
|
|
return True
|
|
if len(set(v)) < 6:
|
|
return True
|
|
return False
|
|
|
|
|
|
def is_config_url(original_url):
|
|
"""Check if URL's path ends with a known config filename."""
|
|
path = original_url.split("?")[0].rstrip("/")
|
|
return bool(_CONFIG_RE.search(path))
|
|
|
|
|
|
_SKIP_PREFIXES = ("gist:", "hf:", "codeberg:", "gitea:", "gl:", "bb:", "sf:")
|
|
_ORG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,38}$")
|
|
|
|
|
|
def _clean_org(repo):
|
|
"""keys.repo column value -> GitHub org/user (lowercase) or None."""
|
|
if not repo:
|
|
return None
|
|
r = repo.strip()
|
|
if r.startswith(_SKIP_PREFIXES):
|
|
return None
|
|
if r.startswith("issue:"):
|
|
r = r[6:]
|
|
if r.startswith("go:github.com/"):
|
|
r = r[14:]
|
|
elif r.startswith("github.com/"):
|
|
r = r[11:]
|
|
org = r.split("/")[0].split("#")[0].lower()
|
|
return org if _ORG_RE.match(org) else None
|
|
|
|
|
|
def mine_orgs(limit=3000):
|
|
"""Top GitHub orgs by historical key count from keys.db (read-only)."""
|
|
counts = {}
|
|
try:
|
|
ro = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=30)
|
|
cur = ro.execute(
|
|
"SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != ''")
|
|
for (repo,) in cur:
|
|
o = _clean_org(repo)
|
|
if o:
|
|
counts[o] = counts.get(o, 0) + 1
|
|
ro.close()
|
|
except Exception as e:
|
|
log(f"mine_orgs DB error: {type(e).__name__} {e}")
|
|
ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:limit]
|
|
return [o for o, _ in ranked] or list(SEED_ORGS)
|
|
|
|
|
|
def load_orgs():
|
|
"""Cached org list; mines keys.db and writes ORGS_FILE on first run."""
|
|
try:
|
|
with open(ORGS_FILE, encoding="utf-8") as f:
|
|
orgs = [l.strip().lower() for l in f if l.strip()]
|
|
if orgs:
|
|
return orgs
|
|
except FileNotFoundError:
|
|
pass
|
|
orgs = mine_orgs()
|
|
try:
|
|
with open(ORGS_FILE, "w", encoding="utf-8") as f:
|
|
f.write("\n".join(orgs) + "\n")
|
|
except OSError as e:
|
|
log(f"orgs file write error: {e}")
|
|
return orgs
|
|
|
|
|
|
def fetch_cdx(prefix, offset, limit, flt):
|
|
"""Fetch one CDX page. Returns list of (timestamp, original_url) or None."""
|
|
url = (
|
|
"http://web.archive.org/cdx/search/cdx"
|
|
f"?url={prefix}&matchType=prefix"
|
|
f"&filter={flt}&filter=statuscode:200"
|
|
f"&collapse=urlkey"
|
|
f"&limit={limit}&offset={offset}"
|
|
)
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = S.get(url, headers=UA, timeout=60)
|
|
if r.status_code == 429:
|
|
retry = int(r.headers.get("Retry-After", 60))
|
|
log(f" cdx 429, retry-after {retry}s")
|
|
time.sleep(retry)
|
|
continue
|
|
if r.status_code == 503:
|
|
log(f" cdx 503 (offline), backing off")
|
|
time.sleep(30)
|
|
continue
|
|
if r.status_code != 200:
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
lines = r.text.strip().split("\n")
|
|
out = []
|
|
for line in lines:
|
|
line = line.strip()
|
|
if not line or line.startswith("<"):
|
|
continue
|
|
parts = line.split()
|
|
if len(parts) < 6:
|
|
continue
|
|
ts = parts[1]
|
|
orig = parts[2]
|
|
if orig and ts:
|
|
out.append((ts, orig))
|
|
return out
|
|
except requests.RequestException as e:
|
|
if attempt == 3:
|
|
log(f" cdx-err {prefix} offset={offset}: {e}")
|
|
return None
|
|
time.sleep(8 * attempt)
|
|
except Exception as e:
|
|
if attempt == 3:
|
|
log(f" cdx-err {prefix}: {e}")
|
|
return None
|
|
time.sleep(8)
|
|
return None
|
|
|
|
|
|
def fetch_archive(timestamp, original_url):
|
|
"""Fetch raw archived content from Wayback Machine."""
|
|
url = f"https://web.archive.org/web/{timestamp}id_/{original_url}"
|
|
for attempt in (1, 2, 3):
|
|
try:
|
|
r = S.get(url, headers=UA, timeout=30, stream=True)
|
|
if r.status_code == 429:
|
|
r.close()
|
|
retry = int(r.headers.get("Retry-After", 30))
|
|
log(f" archive 429, retry-after {retry}s")
|
|
time.sleep(retry)
|
|
continue
|
|
if r.status_code == 404:
|
|
r.close()
|
|
return None
|
|
if r.status_code != 200:
|
|
r.close()
|
|
raise RuntimeError(f"http {r.status_code}")
|
|
chunks = []
|
|
size = 0
|
|
for chunk in r.iter_content(65536):
|
|
size += len(chunk)
|
|
if size > FILE_CAP:
|
|
r.close()
|
|
return None
|
|
chunks.append(chunk)
|
|
r.close()
|
|
return b"".join(chunks)
|
|
except requests.RequestException as e:
|
|
if attempt == 3:
|
|
return None
|
|
time.sleep(8 * attempt)
|
|
except Exception:
|
|
if attempt == 3:
|
|
return None
|
|
time.sleep(8)
|
|
return None
|
|
|
|
|
|
def process_content(raw_bytes, original_url):
|
|
"""Decode, scan_text, filter junk, db_add. Returns key count."""
|
|
text = raw_bytes.decode("utf-8", "replace")
|
|
n = 0
|
|
for prov, key in scan_text(text):
|
|
if is_junk(key) or len(key) < 30:
|
|
continue
|
|
db_add(key, prov, original_url)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def cycle(orgs, max_items=MAX_ITEMS):
|
|
global total, seen
|
|
total = 0
|
|
org_idx, offset = load_cursor()
|
|
|
|
org = orgs[org_idx % len(orgs)]
|
|
flt = CDX_FILTERS[org_idx % len(CDX_FILTERS)]
|
|
prefix = f"raw.githubusercontent.com/{org}/"
|
|
log(f"cycle {prefix} offset={offset} filter={flt.split(':')[-1]}")
|
|
|
|
rows = fetch_cdx(prefix, offset, CDX_LIMIT, flt)
|
|
if rows is None:
|
|
log(f" cdx-fail {prefix}, rotating")
|
|
save_cursor((org_idx + 1) % len(orgs), 0)
|
|
return
|
|
|
|
if not rows:
|
|
log(f" cdx-empty {prefix}, rotating")
|
|
save_cursor((org_idx + 1) % len(orgs), 0)
|
|
return
|
|
|
|
processed = 0
|
|
config_items = [(ts, orig) for ts, orig in rows if is_config_url(orig)]
|
|
|
|
config_items.sort(key=lambda x: x[0]) # oldest first for stability
|
|
log(f" cdx rows={len(rows)} config={len(config_items)}")
|
|
|
|
for ts, orig in config_items:
|
|
if processed >= max_items:
|
|
break
|
|
if orig in seen:
|
|
continue
|
|
|
|
mark_seen(orig)
|
|
raw = fetch_archive(ts, orig)
|
|
if raw is None:
|
|
continue
|
|
n = process_content(raw, orig)
|
|
processed += 1
|
|
|
|
# Advance cursor: even if we hit max_items early, advance offset so
|
|
# next cycle doesn't re-scan the same rows.
|
|
new_offset = offset + CDX_LIMIT
|
|
if new_offset > len(rows) + offset or len(rows) < CDX_LIMIT:
|
|
# Exhausted this org's batch or hit end; rotate
|
|
save_cursor((org_idx + 1) % len(orgs), 0)
|
|
else:
|
|
save_cursor(org_idx, new_offset)
|
|
|
|
log(f" +{total} keys processed={processed} / {len(config_items)} configs")
|
|
|
|
|
|
def main():
|
|
log("wayback_eternal started (prefix mode)")
|
|
if "--remine" in sys.argv:
|
|
try:
|
|
_os.remove(ORGS_FILE)
|
|
except OSError:
|
|
pass
|
|
orgs = load_orgs()
|
|
log(f"orgs loaded: {len(orgs)}")
|
|
if "--once" in sys.argv:
|
|
cycle(orgs, max_items=min(MAX_ITEMS, 5))
|
|
log("wayback_eternal --once done")
|
|
return
|
|
|
|
while True:
|
|
try:
|
|
cycle(orgs)
|
|
except Exception as e:
|
|
log(f"cycle-exception {type(e).__name__} {e}")
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |