KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
375
gitlab_artifacts_eternal.py
Normal file
375
gitlab_artifacts_eternal.py
Normal file
@@ -0,0 +1,375 @@
|
||||
"""ETERNAL GitLab public CI artifacts: project list -> pipelines -> jobs -> traces + zips.
|
||||
|
||||
Endpoint(s):
|
||||
- api/v4/projects?visibility=public (200, no auth)
|
||||
- api/v4/projects/{pid}/pipelines (200, no auth)
|
||||
- api/v4/projects/{pid}/pipelines/{plid}/jobs (200, no auth)
|
||||
- /path/-/jobs/{jid}/raw (200, no auth — raw trace)
|
||||
- /path/-/jobs/{jid}/artifacts/download (302->CDN, some projects — zip)
|
||||
Auth: none (public GitLab.com data)
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib, io, json, re, sqlite3, sys, threading, time, zipfile
|
||||
import requests
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "gitlab_artifacts_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_seen.txt")
|
||||
N_THREADS = 4
|
||||
SLEEP = 120
|
||||
MAX_ITEMS = 200
|
||||
FILE_CAP = 500_000 # max bytes to scan per text file
|
||||
ARCHIVE_CAP = 20 * 1024 * 1024 # max zip to download (20MB)
|
||||
TRACE_CAP = 2 * 1024 * 1024 # max trace to scan (2MB)
|
||||
ARTIFACT_CAP = 20 * 1024 * 1024
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = (
|
||||
"xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef", "0000000", "fffffff",
|
||||
)
|
||||
|
||||
SCAN_EXT = (
|
||||
".env", ".json", ".yml", ".yaml", ".txt", ".cfg", ".conf", ".config",
|
||||
".ini", ".py", ".sh", ".xml", ".properties", ".toml", ".yaml", ".md",
|
||||
".js", ".ts", ".html",
|
||||
)
|
||||
|
||||
FOUND_TAG = "gitlab-artifacts"
|
||||
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||||
plan TEXT, price TEXT, remaining TEXT,
|
||||
found TEXT, repo TEXT)""")
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
if any(g in key for g in GARBAGE) or len(set(key)) < 6:
|
||||
return
|
||||
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
||||
repo_trunc = (repo or "")[:300]
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(h, key, prov, "NEW", FOUND_TAG, repo_trunc))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE) as f:
|
||||
return int(f.read().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for ln in f:
|
||||
s.add(ln.strip())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def scan_text_chunk(text, repo):
|
||||
"""Scan text for keys, feed to db_add."""
|
||||
n = 0
|
||||
for prov, key in scan_text(text):
|
||||
if is_junk(key):
|
||||
continue
|
||||
db_add(key, prov, repo)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
# ── HTTP helpers ──────────────────────────────────────────
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False # bypass local dashscope proxy
|
||||
S.headers.update(UA)
|
||||
|
||||
|
||||
def _get(url, timeout=30, stream=False):
|
||||
"""GET with retry and backoff."""
|
||||
for wait in (2, 4, 8):
|
||||
try:
|
||||
r = S.get(url, timeout=timeout, stream=stream)
|
||||
if r.status_code == 429 or r.status_code == 403:
|
||||
ra = r.headers.get("Retry-After")
|
||||
if ra and ra.isdigit():
|
||||
time.sleep(int(ra))
|
||||
else:
|
||||
time.sleep(30)
|
||||
continue
|
||||
if r.status_code == 200 or r.status_code == 302:
|
||||
return r
|
||||
if r.status_code >= 500:
|
||||
time.sleep(wait)
|
||||
continue
|
||||
return None # 4xx that isn't 429=non-retryable
|
||||
except (requests.ConnectionError, requests.Timeout):
|
||||
time.sleep(wait)
|
||||
return None
|
||||
|
||||
|
||||
# ── Scanning ──────────────────────────────────────────────
|
||||
|
||||
def scan_trace(text, repo):
|
||||
"""Scan CI trace text for keys."""
|
||||
return scan_text_chunk(text, repo)
|
||||
|
||||
|
||||
def scan_zip(content, repo):
|
||||
"""Download and scan zip members (text files only)."""
|
||||
n = 0
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
for name in zf.namelist():
|
||||
if not any(name.endswith(ex) for ex in SCAN_EXT):
|
||||
continue
|
||||
try:
|
||||
info = zf.getinfo(name)
|
||||
# zip-bomb guard: check UNCOMPRESSED size from metadata BEFORE
|
||||
# reading; zf.read() would fully decompress into RAM (Linux
|
||||
# OOM-killer SIGKILLs, no MemoryError raised).
|
||||
if info.file_size == 0 or info.file_size > FILE_CAP:
|
||||
continue
|
||||
data = zf.open(name).read(FILE_CAP + 1)
|
||||
text = data.decode("utf-8", "replace")[:FILE_CAP]
|
||||
for prov, key in scan_text(text):
|
||||
if is_junk(key):
|
||||
continue
|
||||
db_add(key, prov, f"{repo}#{name}")
|
||||
n += 1
|
||||
except (zipfile.BadZipFile, MemoryError):
|
||||
continue
|
||||
except (zipfile.BadZipFile, MemoryError):
|
||||
pass
|
||||
return n
|
||||
|
||||
|
||||
def _read_capped(r, cap):
|
||||
"""Stream-read r up to cap bytes; None if oversized. Always closes r."""
|
||||
if r is None:
|
||||
return None
|
||||
chunks = []
|
||||
size = 0
|
||||
try:
|
||||
for chunk in r.iter_content(65536):
|
||||
size += len(chunk)
|
||||
if size > cap:
|
||||
return None
|
||||
chunks.append(chunk)
|
||||
finally:
|
||||
r.close()
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def process_job(pid, path, jid, jname):
|
||||
"""Fetch trace + artifacts for one job, return key counts."""
|
||||
nk = 0
|
||||
repo = f"{path}/job/{jid}"
|
||||
|
||||
# raw trace (streamed, capped)
|
||||
trace_url = f"https://gitlab.com/{path}/-/jobs/{jid}/raw"
|
||||
r = _get(trace_url, timeout=30, stream=True)
|
||||
if r and r.status_code == 200:
|
||||
raw = _read_capped(r, TRACE_CAP)
|
||||
if raw:
|
||||
nk += scan_trace(raw.decode("utf-8", "replace"), repo)
|
||||
elif r is not None:
|
||||
r.close()
|
||||
|
||||
# artifacts zip (streamed, capped)
|
||||
art_url = f"https://gitlab.com/{path}/-/jobs/{jid}/artifacts/download"
|
||||
r2 = _get(art_url, timeout=60, stream=True)
|
||||
if r2 and r2.status_code in (200, 302):
|
||||
if r2.status_code == 302:
|
||||
loc = r2.headers.get("location", "")
|
||||
r2.close()
|
||||
if loc:
|
||||
r3 = _get(loc, timeout=60, stream=True)
|
||||
if r3 and r3.status_code == 200:
|
||||
payload = _read_capped(r3, ARCHIVE_CAP)
|
||||
if payload:
|
||||
nk += scan_zip(payload, repo)
|
||||
elif r3 is not None:
|
||||
r3.close()
|
||||
else:
|
||||
payload = _read_capped(r2, ARCHIVE_CAP)
|
||||
if payload:
|
||||
nk += scan_zip(payload, repo)
|
||||
elif r2 is not None:
|
||||
r2.close()
|
||||
|
||||
return nk
|
||||
|
||||
|
||||
def process_project(pid, path):
|
||||
"""Fetch pipelines → jobs for a single project, return key count."""
|
||||
nk = 0
|
||||
# recent pipelines
|
||||
pl_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines?per_page=5"
|
||||
r = _get(pl_url)
|
||||
if not r or r.status_code != 200:
|
||||
return 0
|
||||
try:
|
||||
pipelines = r.json()
|
||||
except json.JSONDecodeError:
|
||||
return 0
|
||||
if not isinstance(pipelines, list):
|
||||
return 0
|
||||
|
||||
for pl in pipelines[:3]: # at most 3 pipelines per cycle
|
||||
plid = pl.get("id")
|
||||
if not plid:
|
||||
continue
|
||||
# jobs for this pipeline
|
||||
jobs_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines/{plid}/jobs?per_page=15"
|
||||
r2 = _get(jobs_url)
|
||||
if not r2 or r2.status_code != 200:
|
||||
continue
|
||||
try:
|
||||
jobs = r2.json()
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(jobs, list):
|
||||
continue
|
||||
|
||||
for j in jobs:
|
||||
jid = j.get("id")
|
||||
if not jid:
|
||||
continue
|
||||
jname = j.get("name", "")
|
||||
skey = f"{pid}/{jid}"
|
||||
if skey in seen:
|
||||
continue
|
||||
nk += process_job(pid, path, jid, jname)
|
||||
mark_seen(skey)
|
||||
|
||||
return nk
|
||||
|
||||
|
||||
def cycle(once=False):
|
||||
"""One full pass: discover projects, scan their CI artifacts."""
|
||||
global total
|
||||
n_max = MAX_ITEMS if not once else min(MAX_ITEMS, 5)
|
||||
cursor = load_cursor()
|
||||
page = 1
|
||||
projects_seen = 0
|
||||
fail = 0
|
||||
keys_found = 0
|
||||
|
||||
while projects_seen < n_max:
|
||||
url = (f"https://gitlab.com/api/v4/projects"
|
||||
f"?visibility=public&order_by=last_activity_at"
|
||||
f"&per_page=20&page={page}")
|
||||
r = _get(url)
|
||||
if not r or r.status_code != 200:
|
||||
fail += 1
|
||||
break
|
||||
try:
|
||||
projects = r.json()
|
||||
except json.JSONDecodeError:
|
||||
fail += 1
|
||||
break
|
||||
if not isinstance(projects, list) or not projects:
|
||||
break
|
||||
|
||||
for p in projects:
|
||||
pid = p.get("id")
|
||||
path = p.get("path_with_namespace")
|
||||
if not pid or not path:
|
||||
continue
|
||||
if pid <= cursor:
|
||||
continue
|
||||
if projects_seen >= n_max:
|
||||
break
|
||||
|
||||
nk = process_project(pid, path)
|
||||
if nk is None:
|
||||
fail += 1
|
||||
continue
|
||||
keys_found += nk
|
||||
projects_seen += 1
|
||||
save_cursor(pid)
|
||||
|
||||
if once and projects_seen >= 5:
|
||||
break
|
||||
|
||||
page += 1
|
||||
if once and page > 2:
|
||||
break
|
||||
|
||||
log(f"cycle projects={projects_seen} keys={keys_found} fail={fail} +{total}")
|
||||
return projects_seen, keys_found, fail
|
||||
|
||||
|
||||
def main():
|
||||
log("gitlab_artifacts_eternal started")
|
||||
once = "--once" in sys.argv
|
||||
if once:
|
||||
init_total = total
|
||||
n, nk, f = cycle(once=True)
|
||||
print(f"OK scanned={n} keys={nk} fail={f} db_keys={total - init_total}")
|
||||
sys.exit(0 if n > 0 else 1)
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as exc:
|
||||
log(f"cycle crash: {exc}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user