KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

wayback_eternal.py

@@ -0,0 +1,412 @@
+"""ETERNAL Wayback Machine: CDX prefix crawl over GitHub orgs with leak history.
+
+Walks Internet Archive CDX with matchType=prefix over raw.githubusercontent.com/<ORG>/
+for orgs mined from keys.db (top by historical key count, cached in
+data/wayback_orgs.txt). Server-side filter original:.*/\.env + statuscode:200
+(broad regex filters → CDX 503, proven live 2026-10-19; matchType=domain —
+dead, timeouts). Fetches raw archived content, scans for keys.
+pastebin removed: CDX does not index pastebin content. gist: TODO if needed.
+
+Endpoint(s): http://web.archive.org/cdx/search/cdx (CDX index API)
+             https://web.archive.org/web/<ts>id_/<url> (raw archived content)
+Auth: none
+Flags: --once (single small cycle), --remine (rebuild data/wayback_orgs.txt)
+"""
+import os as _os
+BASE = _os.path.dirname(_os.path.abspath(__file__))
+_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
+
+import hashlib, re, sqlite3, sys, threading, time
+import requests
+
+S = requests.Session()
+S.trust_env = False  # contract 4: bypass local dashscope proxy
+
+DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
+LOG = _os.path.join(BASE, "data", "wayback_eternal.log")
+CURSOR_FILE = _os.path.join(BASE, "data", "wayback_cursor.txt")
+SEEN_FILE = _os.path.join(BASE, "data", "wayback_seen.txt")
+N_THREADS = 1
+SLEEP = 90
+MAX_ITEMS = 200
+CDX_LIMIT = 5000
+FILE_CAP = 500_000
+
+from key_patterns import scan_text
+
+GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
+           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
+           "abcdefghijklmnopqrst", "1234567890abcdef")
+
+UA = {"User-Agent": "keyfarm-wayback/1.0"}
+
+ORGS_FILE = _os.path.join(BASE, "data", "wayback_orgs.txt")
+# Server-side CDX filters: narrow single-pattern each (wide alternation → 503,
+# proven 2026-10-19). Rotated per org cycle; all pass the is_config_url gate.
+CDX_FILTERS = (
+    "original:.*/\\.env",
+    "original:.*/\\.npmrc",
+    "original:.*/\\.netrc",
+    "original:.*/\\.git-credentials",
+    "original:.*credentials\\.json",
+    "original:.*/config\\.json",
+)
+# Fallback if keys.db mining fails
+SEED_ORGS = ["dpangestuw", "moleway", "sse-casb-lab"]
+
+# Config-like filenames that may harbour secrets / tokens
+CONFIG_NAMES = frozenset([
+    ".env", ".env.example", ".env.production", ".env.local",
+    ".env.dev", ".env.staging", ".envrc", ".env.template", ".env.sample",
+    "credentials", "credentials.json", "credentials.yml", "credentials.yaml",
+    "config.js", "config.json",
+    "application.properties",
+    ".npmrc",
+    "id_rsa", "id_rsa.pub", "id_ed25519", "id_ecdsa",
+    "secret", "secrets", "secrets.json", "secrets.yml", "secrets.yaml",
+    ".netrc", ".git-credentials", ".dockercfg",
+    "composer.json",
+])
+
+# Match a URL path's final segment against CONFIG_NAMES and also catch
+# paths like /foo/.env or /foo/credentials.json via substring check.
+_CONFIG_RE = re.compile(
+    r'/(?:'
+    r'\.env(?:\.\w+)?|'
+    r'credentials(?:\.\w+)?|'
+    r'config\.(?:js|json)|'
+    r'application\.properties|'
+    r'\.npmrc|'
+    r'id_(?:rsa|ed25519|ecdsa)(?:\.pub)?|'
+    r'secret(?:s)?(?:\.\w+)?|'
+    r'\.netrc|'
+    r'\.git-credentials|'
+    r'\.dockercfg|'
+    r'composer\.json'
+    r')(?:\?.*)?$',
+    re.IGNORECASE
+)
+
+
+def log(msg):
+    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
+    with open(LOG, "a", encoding="utf-8") as f:
+        f.write(line)
+
+
+conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
+                       isolation_level=None)
+conn.execute("""
+    CREATE TABLE IF NOT EXISTS keys (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      hash TEXT UNIQUE,
+      val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
+      plan TEXT, price TEXT, remaining TEXT,
+      found TEXT, repo TEXT)
+""")
+lock = threading.Lock()
+total = 0
+
+
+def db_add(key, prov, repo):
+    global total
+    with lock:
+        conn.execute(
+            "INSERT INTO keys (hash, val, prov, status, found, repo) "
+            "VALUES (?,?,?,?,?,?) "
+            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
+            "found=excluded.found, repo=excluded.repo "
+            "WHERE keys.status IN ('DEAD','ERR')",
+            (hashlib.sha1(key.encode()).hexdigest()[:16],
+             key, prov, "NEW", "wayback", repo[:300]))
+        total += 1
+
+
+def load_cursor():
+    """Returns (domain_idx, offset)."""
+    try:
+        with open(CURSOR_FILE, encoding="utf-8") as f:
+            raw = f.read().strip()
+            if raw:
+                parts = raw.split()
+                return int(parts[0]), int(parts[1])
+    except (FileNotFoundError, ValueError, IndexError):
+        pass
+    return 0, 0
+
+
+def save_cursor(domain_idx, offset):
+    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
+        f.write(f"{domain_idx} {offset}")
+
+
+def load_seen():
+    s = set()
+    try:
+        with open(SEEN_FILE, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    s.add(line)
+    except FileNotFoundError:
+        pass
+    return s
+
+
+seen = load_seen()
+
+
+def mark_seen(url):
+    seen.add(url)
+    with open(SEEN_FILE, "a", encoding="utf-8") as f:
+        f.write(url + "\n")
+
+
+def is_junk(v):
+    if any(g in v for g in GARBAGE):
+        return True
+    if len(set(v)) < 6:
+        return True
+    return False
+
+
+def is_config_url(original_url):
+    """Check if URL's path ends with a known config filename."""
+    path = original_url.split("?")[0].rstrip("/")
+    return bool(_CONFIG_RE.search(path))
+
+
+_SKIP_PREFIXES = ("gist:", "hf:", "codeberg:", "gitea:", "gl:", "bb:", "sf:")
+_ORG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,38}$")
+
+
+def _clean_org(repo):
+    """keys.repo column value -> GitHub org/user (lowercase) or None."""
+    if not repo:
+        return None
+    r = repo.strip()
+    if r.startswith(_SKIP_PREFIXES):
+        return None
+    if r.startswith("issue:"):
+        r = r[6:]
+    if r.startswith("go:github.com/"):
+        r = r[14:]
+    elif r.startswith("github.com/"):
+        r = r[11:]
+    org = r.split("/")[0].split("#")[0].lower()
+    return org if _ORG_RE.match(org) else None
+
+
+def mine_orgs(limit=3000):
+    """Top GitHub orgs by historical key count from keys.db (read-only)."""
+    counts = {}
+    try:
+        ro = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=30)
+        cur = ro.execute(
+            "SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != ''")
+        for (repo,) in cur:
+            o = _clean_org(repo)
+            if o:
+                counts[o] = counts.get(o, 0) + 1
+        ro.close()
+    except Exception as e:
+        log(f"mine_orgs DB error: {type(e).__name__} {e}")
+    ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:limit]
+    return [o for o, _ in ranked] or list(SEED_ORGS)
+
+
+def load_orgs():
+    """Cached org list; mines keys.db and writes ORGS_FILE on first run."""
+    try:
+        with open(ORGS_FILE, encoding="utf-8") as f:
+            orgs = [l.strip().lower() for l in f if l.strip()]
+        if orgs:
+            return orgs
+    except FileNotFoundError:
+        pass
+    orgs = mine_orgs()
+    try:
+        with open(ORGS_FILE, "w", encoding="utf-8") as f:
+            f.write("\n".join(orgs) + "\n")
+    except OSError as e:
+        log(f"orgs file write error: {e}")
+    return orgs
+
+
+def fetch_cdx(prefix, offset, limit, flt):
+    """Fetch one CDX page. Returns list of (timestamp, original_url) or None."""
+    url = (
+        "http://web.archive.org/cdx/search/cdx"
+        f"?url={prefix}&matchType=prefix"
+        f"&filter={flt}&filter=statuscode:200"
+        f"&collapse=urlkey"
+        f"&limit={limit}&offset={offset}"
+    )
+    for attempt in (1, 2, 3):
+        try:
+            r = S.get(url, headers=UA, timeout=60)
+            if r.status_code == 429:
+                retry = int(r.headers.get("Retry-After", 60))
+                log(f"  cdx 429, retry-after {retry}s")
+                time.sleep(retry)
+                continue
+            if r.status_code == 503:
+                log(f"  cdx 503 (offline), backing off")
+                time.sleep(30)
+                continue
+            if r.status_code != 200:
+                raise RuntimeError(f"http {r.status_code}")
+            lines = r.text.strip().split("\n")
+            out = []
+            for line in lines:
+                line = line.strip()
+                if not line or line.startswith("<"):
+                    continue
+                parts = line.split()
+                if len(parts) < 6:
+                    continue
+                ts = parts[1]
+                orig = parts[2]
+                if orig and ts:
+                    out.append((ts, orig))
+            return out
+        except requests.RequestException as e:
+            if attempt == 3:
+                log(f"  cdx-err {prefix} offset={offset}: {e}")
+                return None
+            time.sleep(8 * attempt)
+        except Exception as e:
+            if attempt == 3:
+                log(f"  cdx-err {prefix}: {e}")
+                return None
+            time.sleep(8)
+    return None
+
+
+def fetch_archive(timestamp, original_url):
+    """Fetch raw archived content from Wayback Machine."""
+    url = f"https://web.archive.org/web/{timestamp}id_/{original_url}"
+    for attempt in (1, 2, 3):
+        try:
+            r = S.get(url, headers=UA, timeout=30, stream=True)
+            if r.status_code == 429:
+                r.close()
+                retry = int(r.headers.get("Retry-After", 30))
+                log(f"  archive 429, retry-after {retry}s")
+                time.sleep(retry)
+                continue
+            if r.status_code == 404:
+                r.close()
+                return None
+            if r.status_code != 200:
+                r.close()
+                raise RuntimeError(f"http {r.status_code}")
+            chunks = []
+            size = 0
+            for chunk in r.iter_content(65536):
+                size += len(chunk)
+                if size > FILE_CAP:
+                    r.close()
+                    return None
+                chunks.append(chunk)
+            r.close()
+            return b"".join(chunks)
+        except requests.RequestException as e:
+            if attempt == 3:
+                return None
+            time.sleep(8 * attempt)
+        except Exception:
+            if attempt == 3:
+                return None
+            time.sleep(8)
+    return None
+
+
+def process_content(raw_bytes, original_url):
+    """Decode, scan_text, filter junk, db_add. Returns key count."""
+    text = raw_bytes.decode("utf-8", "replace")
+    n = 0
+    for prov, key in scan_text(text):
+        if is_junk(key) or len(key) < 30:
+            continue
+        db_add(key, prov, original_url)
+        n += 1
+    return n
+
+
+def cycle(orgs, max_items=MAX_ITEMS):
+    global total, seen
+    total = 0
+    org_idx, offset = load_cursor()
+
+    org = orgs[org_idx % len(orgs)]
+    flt = CDX_FILTERS[org_idx % len(CDX_FILTERS)]
+    prefix = f"raw.githubusercontent.com/{org}/"
+    log(f"cycle {prefix} offset={offset} filter={flt.split(':')[-1]}")
+
+    rows = fetch_cdx(prefix, offset, CDX_LIMIT, flt)
+    if rows is None:
+        log(f"  cdx-fail {prefix}, rotating")
+        save_cursor((org_idx + 1) % len(orgs), 0)
+        return
+
+    if not rows:
+        log(f"  cdx-empty {prefix}, rotating")
+        save_cursor((org_idx + 1) % len(orgs), 0)
+        return
+
+    processed = 0
+    config_items = [(ts, orig) for ts, orig in rows if is_config_url(orig)]
+
+    config_items.sort(key=lambda x: x[0])  # oldest first for stability
+    log(f"  cdx rows={len(rows)} config={len(config_items)}")
+
+    for ts, orig in config_items:
+        if processed >= max_items:
+            break
+        if orig in seen:
+            continue
+
+        mark_seen(orig)
+        raw = fetch_archive(ts, orig)
+        if raw is None:
+            continue
+        n = process_content(raw, orig)
+        processed += 1
+
+    # Advance cursor: even if we hit max_items early, advance offset so
+    # next cycle doesn't re-scan the same rows.
+    new_offset = offset + CDX_LIMIT
+    if new_offset > len(rows) + offset or len(rows) < CDX_LIMIT:
+        # Exhausted this org's batch or hit end; rotate
+        save_cursor((org_idx + 1) % len(orgs), 0)
+    else:
+        save_cursor(org_idx, new_offset)
+
+    log(f"  +{total} keys processed={processed} / {len(config_items)} configs")
+
+
+def main():
+    log("wayback_eternal started (prefix mode)")
+    if "--remine" in sys.argv:
+        try:
+            _os.remove(ORGS_FILE)
+        except OSError:
+            pass
+    orgs = load_orgs()
+    log(f"orgs loaded: {len(orgs)}")
+    if "--once" in sys.argv:
+        cycle(orgs, max_items=min(MAX_ITEMS, 5))
+        log("wayback_eternal --once done")
+        return
+
+    while True:
+        try:
+            cycle(orgs)
+        except Exception as e:
+            log(f"cycle-exception {type(e).__name__} {e}")
+        time.sleep(SLEEP)
+
+
+if __name__ == "__main__":
+    main()