375 lines
11 KiB
Python
375 lines
11 KiB
Python
"""ETERNAL GitLab public CI artifacts: project list -> pipelines -> jobs -> traces + zips.
|
|
|
|
Endpoint(s):
|
|
- api/v4/projects?visibility=public (200, no auth)
|
|
- api/v4/projects/{pid}/pipelines (200, no auth)
|
|
- api/v4/projects/{pid}/pipelines/{plid}/jobs (200, no auth)
|
|
- /path/-/jobs/{jid}/raw (200, no auth — raw trace)
|
|
- /path/-/jobs/{jid}/artifacts/download (302->CDN, some projects — zip)
|
|
Auth: none (public GitLab.com data)
|
|
"""
|
|
import os as _os
|
|
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
|
|
|
import hashlib, io, json, re, sqlite3, sys, threading, time, zipfile
|
|
import requests
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
|
LOG = _os.path.join(BASE, "data", "gitlab_artifacts_eternal.log")
|
|
CURSOR_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_cursor.txt")
|
|
SEEN_FILE = _os.path.join(BASE, "data", "gitlab_artifacts_seen.txt")
|
|
N_THREADS = 4
|
|
SLEEP = 120
|
|
MAX_ITEMS = 200
|
|
FILE_CAP = 500_000 # max bytes to scan per text file
|
|
ARCHIVE_CAP = 20 * 1024 * 1024 # max zip to download (20MB)
|
|
TRACE_CAP = 2 * 1024 * 1024 # max trace to scan (2MB)
|
|
ARTIFACT_CAP = 20 * 1024 * 1024
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = (
|
|
"xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef", "0000000", "fffffff",
|
|
)
|
|
|
|
SCAN_EXT = (
|
|
".env", ".json", ".yml", ".yaml", ".txt", ".cfg", ".conf", ".config",
|
|
".ini", ".py", ".sh", ".xml", ".properties", ".toml", ".yaml", ".md",
|
|
".js", ".ts", ".html",
|
|
)
|
|
|
|
FOUND_TAG = "gitlab-artifacts"
|
|
|
|
UA = {"User-Agent": "Mozilla/5.0"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
|
plan TEXT, price TEXT, remaining TEXT,
|
|
found TEXT, repo TEXT)""")
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
if any(g in key for g in GARBAGE) or len(set(key)) < 6:
|
|
return
|
|
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
|
repo_trunc = (repo or "")[:300]
|
|
with lock:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(h, key, prov, "NEW", FOUND_TAG, repo_trunc))
|
|
total += 1
|
|
|
|
|
|
def load_cursor():
|
|
try:
|
|
with open(CURSOR_FILE) as f:
|
|
return int(f.read().strip())
|
|
except (FileNotFoundError, ValueError):
|
|
return 0
|
|
|
|
|
|
def save_cursor(c):
|
|
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
|
f.write(str(c))
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for ln in f:
|
|
s.add(ln.strip())
|
|
except FileNotFoundError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def is_junk(v):
|
|
if any(g in v for g in GARBAGE):
|
|
return True
|
|
if len(set(v)) < 6:
|
|
return True
|
|
return False
|
|
|
|
|
|
def scan_text_chunk(text, repo):
|
|
"""Scan text for keys, feed to db_add."""
|
|
n = 0
|
|
for prov, key in scan_text(text):
|
|
if is_junk(key):
|
|
continue
|
|
db_add(key, prov, repo)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
# ── HTTP helpers ──────────────────────────────────────────
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False # bypass local dashscope proxy
|
|
S.headers.update(UA)
|
|
|
|
|
|
def _get(url, timeout=30, stream=False):
|
|
"""GET with retry and backoff."""
|
|
for wait in (2, 4, 8):
|
|
try:
|
|
r = S.get(url, timeout=timeout, stream=stream)
|
|
if r.status_code == 429 or r.status_code == 403:
|
|
ra = r.headers.get("Retry-After")
|
|
if ra and ra.isdigit():
|
|
time.sleep(int(ra))
|
|
else:
|
|
time.sleep(30)
|
|
continue
|
|
if r.status_code == 200 or r.status_code == 302:
|
|
return r
|
|
if r.status_code >= 500:
|
|
time.sleep(wait)
|
|
continue
|
|
return None # 4xx that isn't 429=non-retryable
|
|
except (requests.ConnectionError, requests.Timeout):
|
|
time.sleep(wait)
|
|
return None
|
|
|
|
|
|
# ── Scanning ──────────────────────────────────────────────
|
|
|
|
def scan_trace(text, repo):
|
|
"""Scan CI trace text for keys."""
|
|
return scan_text_chunk(text, repo)
|
|
|
|
|
|
def scan_zip(content, repo):
|
|
"""Download and scan zip members (text files only)."""
|
|
n = 0
|
|
try:
|
|
zf = zipfile.ZipFile(io.BytesIO(content))
|
|
for name in zf.namelist():
|
|
if not any(name.endswith(ex) for ex in SCAN_EXT):
|
|
continue
|
|
try:
|
|
info = zf.getinfo(name)
|
|
# zip-bomb guard: check UNCOMPRESSED size from metadata BEFORE
|
|
# reading; zf.read() would fully decompress into RAM (Linux
|
|
# OOM-killer SIGKILLs, no MemoryError raised).
|
|
if info.file_size == 0 or info.file_size > FILE_CAP:
|
|
continue
|
|
data = zf.open(name).read(FILE_CAP + 1)
|
|
text = data.decode("utf-8", "replace")[:FILE_CAP]
|
|
for prov, key in scan_text(text):
|
|
if is_junk(key):
|
|
continue
|
|
db_add(key, prov, f"{repo}#{name}")
|
|
n += 1
|
|
except (zipfile.BadZipFile, MemoryError):
|
|
continue
|
|
except (zipfile.BadZipFile, MemoryError):
|
|
pass
|
|
return n
|
|
|
|
|
|
def _read_capped(r, cap):
|
|
"""Stream-read r up to cap bytes; None if oversized. Always closes r."""
|
|
if r is None:
|
|
return None
|
|
chunks = []
|
|
size = 0
|
|
try:
|
|
for chunk in r.iter_content(65536):
|
|
size += len(chunk)
|
|
if size > cap:
|
|
return None
|
|
chunks.append(chunk)
|
|
finally:
|
|
r.close()
|
|
return b"".join(chunks)
|
|
|
|
|
|
def process_job(pid, path, jid, jname):
|
|
"""Fetch trace + artifacts for one job, return key counts."""
|
|
nk = 0
|
|
repo = f"{path}/job/{jid}"
|
|
|
|
# raw trace (streamed, capped)
|
|
trace_url = f"https://gitlab.com/{path}/-/jobs/{jid}/raw"
|
|
r = _get(trace_url, timeout=30, stream=True)
|
|
if r and r.status_code == 200:
|
|
raw = _read_capped(r, TRACE_CAP)
|
|
if raw:
|
|
nk += scan_trace(raw.decode("utf-8", "replace"), repo)
|
|
elif r is not None:
|
|
r.close()
|
|
|
|
# artifacts zip (streamed, capped)
|
|
art_url = f"https://gitlab.com/{path}/-/jobs/{jid}/artifacts/download"
|
|
r2 = _get(art_url, timeout=60, stream=True)
|
|
if r2 and r2.status_code in (200, 302):
|
|
if r2.status_code == 302:
|
|
loc = r2.headers.get("location", "")
|
|
r2.close()
|
|
if loc:
|
|
r3 = _get(loc, timeout=60, stream=True)
|
|
if r3 and r3.status_code == 200:
|
|
payload = _read_capped(r3, ARCHIVE_CAP)
|
|
if payload:
|
|
nk += scan_zip(payload, repo)
|
|
elif r3 is not None:
|
|
r3.close()
|
|
else:
|
|
payload = _read_capped(r2, ARCHIVE_CAP)
|
|
if payload:
|
|
nk += scan_zip(payload, repo)
|
|
elif r2 is not None:
|
|
r2.close()
|
|
|
|
return nk
|
|
|
|
|
|
def process_project(pid, path):
|
|
"""Fetch pipelines → jobs for a single project, return key count."""
|
|
nk = 0
|
|
# recent pipelines
|
|
pl_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines?per_page=5"
|
|
r = _get(pl_url)
|
|
if not r or r.status_code != 200:
|
|
return 0
|
|
try:
|
|
pipelines = r.json()
|
|
except json.JSONDecodeError:
|
|
return 0
|
|
if not isinstance(pipelines, list):
|
|
return 0
|
|
|
|
for pl in pipelines[:3]: # at most 3 pipelines per cycle
|
|
plid = pl.get("id")
|
|
if not plid:
|
|
continue
|
|
# jobs for this pipeline
|
|
jobs_url = f"https://gitlab.com/api/v4/projects/{pid}/pipelines/{plid}/jobs?per_page=15"
|
|
r2 = _get(jobs_url)
|
|
if not r2 or r2.status_code != 200:
|
|
continue
|
|
try:
|
|
jobs = r2.json()
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if not isinstance(jobs, list):
|
|
continue
|
|
|
|
for j in jobs:
|
|
jid = j.get("id")
|
|
if not jid:
|
|
continue
|
|
jname = j.get("name", "")
|
|
skey = f"{pid}/{jid}"
|
|
if skey in seen:
|
|
continue
|
|
nk += process_job(pid, path, jid, jname)
|
|
mark_seen(skey)
|
|
|
|
return nk
|
|
|
|
|
|
def cycle(once=False):
|
|
"""One full pass: discover projects, scan their CI artifacts."""
|
|
global total
|
|
n_max = MAX_ITEMS if not once else min(MAX_ITEMS, 5)
|
|
cursor = load_cursor()
|
|
page = 1
|
|
projects_seen = 0
|
|
fail = 0
|
|
keys_found = 0
|
|
|
|
while projects_seen < n_max:
|
|
url = (f"https://gitlab.com/api/v4/projects"
|
|
f"?visibility=public&order_by=last_activity_at"
|
|
f"&per_page=20&page={page}")
|
|
r = _get(url)
|
|
if not r or r.status_code != 200:
|
|
fail += 1
|
|
break
|
|
try:
|
|
projects = r.json()
|
|
except json.JSONDecodeError:
|
|
fail += 1
|
|
break
|
|
if not isinstance(projects, list) or not projects:
|
|
break
|
|
|
|
for p in projects:
|
|
pid = p.get("id")
|
|
path = p.get("path_with_namespace")
|
|
if not pid or not path:
|
|
continue
|
|
if pid <= cursor:
|
|
continue
|
|
if projects_seen >= n_max:
|
|
break
|
|
|
|
nk = process_project(pid, path)
|
|
if nk is None:
|
|
fail += 1
|
|
continue
|
|
keys_found += nk
|
|
projects_seen += 1
|
|
save_cursor(pid)
|
|
|
|
if once and projects_seen >= 5:
|
|
break
|
|
|
|
page += 1
|
|
if once and page > 2:
|
|
break
|
|
|
|
log(f"cycle projects={projects_seen} keys={keys_found} fail={fail} +{total}")
|
|
return projects_seen, keys_found, fail
|
|
|
|
|
|
def main():
|
|
log("gitlab_artifacts_eternal started")
|
|
once = "--once" in sys.argv
|
|
if once:
|
|
init_total = total
|
|
n, nk, f = cycle(once=True)
|
|
print(f"OK scanned={n} keys={nk} fail={f} db_keys={total - init_total}")
|
|
sys.exit(0 if n > 0 else 1)
|
|
while True:
|
|
try:
|
|
cycle()
|
|
except Exception as exc:
|
|
log(f"cycle crash: {exc}")
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |