KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
363
gha_logs_eternal.py
Normal file
363
gha_logs_eternal.py
Normal file
@@ -0,0 +1,363 @@
|
||||
"""ETERNAL GHA logs: walk public repos -> completed workflow runs -> scan log zips.
|
||||
|
||||
Endpoint: api.github.com/repos/{owner}/{repo}/actions/runs (REST, per-token quota)
|
||||
Auth: PAT pool (gh_tokens.txt, round-robin, ~2600 tokens × 5000 req/hr = no IP collision)
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib, io, json, re, sqlite3, sys, threading, time, zipfile
|
||||
import requests
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False
|
||||
|
||||
TOKENS_FILE = _os.path.join(BASE, "gh_tokens.txt")
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "gha_logs_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "gha_logs_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "gha_logs_seen.txt")
|
||||
N_THREADS = 4
|
||||
SLEEP = 120
|
||||
MAX_ITEMS = 200 # runs to process per cycle
|
||||
FILE_CAP = 500_000 # max bytes to scan per log file member
|
||||
ARCHIVE_CAP = 20 * 1024 * 1024 # max zip download
|
||||
BATCH = 20 # repos to check per cycle
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "xxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef", "test-key",
|
||||
"sample", "SAMPLE", "insert", "replace", "aaaaaa", "redacted",
|
||||
"masked", "censored", "your_api_key", "replace_with",
|
||||
"fill_in", "undefined", "mock_", "stub_", "todo", "fixme",
|
||||
"XXXXXXXX", "0000000000000000", "skip_")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-gha-logs/1.0"}
|
||||
S.headers.update(UA)
|
||||
|
||||
API = "https://api.github.com"
|
||||
|
||||
# Rate limit safety margin
|
||||
RL_RESERVE = 100
|
||||
|
||||
# ── Logging ──────────────────────────────────────────────
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
# ── PAT pool (round-robin) ──────────────────────────────
|
||||
|
||||
def _load_tokens():
|
||||
if not _os.path.exists(TOKENS_FILE):
|
||||
return []
|
||||
with open(TOKENS_FILE) as f:
|
||||
return [l.strip() for l in f
|
||||
if l.strip() and not l.startswith("ghp_0000")]
|
||||
|
||||
_tokens = _load_tokens()
|
||||
_tok_idx = 0
|
||||
_tok_lock = threading.Lock()
|
||||
|
||||
def next_token():
|
||||
global _tok_idx
|
||||
with _tok_lock:
|
||||
t = _tokens[_tok_idx]
|
||||
_tok_idx = (_tok_idx + 1) % len(_tokens)
|
||||
return t
|
||||
|
||||
def mask_url(url):
|
||||
"""Mask embedded token in URL for logging."""
|
||||
return re.sub(r'https://[^@]*@', 'https://***@', url)
|
||||
|
||||
# ── DB ──────────────────────────────────────────────────
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE,
|
||||
val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||||
plan TEXT, price TEXT, remaining TEXT,
|
||||
found TEXT, repo TEXT)""")
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
h = hashlib.sha1(key.encode()).hexdigest()[:16]
|
||||
try:
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(h, key, prov, "NEW", "gha-logs", repo[:300]))
|
||||
total += 1
|
||||
except sqlite3.Error as e:
|
||||
log(f" db_add error: {e}")
|
||||
|
||||
# ── Cursor & seen ──────────────────────────────────────
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE) as f:
|
||||
return int(f.read().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 1 # repo ID 1+ are public
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w") as f:
|
||||
f.write(str(c))
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
s.add(line)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
def mark_seen(k):
|
||||
seen.add(k)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(k + "\n")
|
||||
|
||||
# ── Junk filter ─────────────────────────────────────────
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
# ── HTTP helper ─────────────────────────────────────────
|
||||
|
||||
def _get(url, headers_extra=None, stream=False, timeout=30):
|
||||
"""GET with retry and backoff, auto-token. Returns response or None."""
|
||||
tok = next_token()
|
||||
hdrs = {"Authorization": f"Bearer {tok}",
|
||||
"Accept": "application/vnd.github+json"}
|
||||
if headers_extra:
|
||||
hdrs.update(headers_extra)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = S.get(url, headers=hdrs, timeout=timeout, stream=stream)
|
||||
if r.status_code == 200:
|
||||
return r
|
||||
if r.status_code in (403, 429):
|
||||
retry_after = int(r.headers.get("Retry-After", 10))
|
||||
log(f" 429/403 on {mask_url(url)} — sleep {retry_after}s")
|
||||
time.sleep(retry_after)
|
||||
continue
|
||||
if r.status_code in (404, 410, 451):
|
||||
return None # gone, not coming back
|
||||
if r.status_code >= 500:
|
||||
log(f" {r.status_code} on {mask_url(url)}, retry {attempt+1}")
|
||||
time.sleep(2 * (attempt + 1))
|
||||
continue
|
||||
return r # unexpected status, caller handles
|
||||
except requests.RequestException as e:
|
||||
log(f" req error: {e}, retry {attempt+1}")
|
||||
time.sleep(2 * (attempt + 1))
|
||||
return None
|
||||
|
||||
# ── Repo discovery ──────────────────────────────────────
|
||||
|
||||
def fetch_repos(since, per_page=100):
|
||||
"""Fetch one page of /repositories. Returns (repos_list, next_since)."""
|
||||
url = f"{API}/repositories?since={since}&per_page={per_page}"
|
||||
r = _get(url)
|
||||
if r is None:
|
||||
return [], since
|
||||
try:
|
||||
repos = r.json()
|
||||
except json.JSONDecodeError:
|
||||
return [], since
|
||||
if not repos:
|
||||
return [], since
|
||||
# next since = last repo ID
|
||||
next_since = repos[-1]["id"]
|
||||
return repos, next_since
|
||||
|
||||
# ── Runs discovery ─────────────────────────────────────
|
||||
|
||||
def fetch_runs(repo_full, per_page=5):
|
||||
"""Fetch completed runs for a repo. Returns list of run dicts or None."""
|
||||
url = f"{API}/repos/{repo_full}/actions/runs?per_page={per_page}&status=completed"
|
||||
r = _get(url)
|
||||
if r is None:
|
||||
return None
|
||||
try:
|
||||
data = r.json()
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
runs = data.get("workflow_runs", [])
|
||||
if not runs:
|
||||
return []
|
||||
return runs
|
||||
|
||||
# ── Log download & scan ─────────────────────────────────
|
||||
|
||||
def scan_logs_zip(zip_bytes, repo_full, run_id):
|
||||
"""Unzip log archive in-memory and scan each member for keys."""
|
||||
n_keys = 0
|
||||
try:
|
||||
z = zipfile.ZipFile(io.BytesIO(zip_bytes))
|
||||
except (zipfile.BadZipFile, Exception) as e:
|
||||
log(f" bad zip for {repo_full} run {run_id}: {e}")
|
||||
return 0
|
||||
for name in z.namelist():
|
||||
info = z.getinfo(name)
|
||||
if info.file_size > FILE_CAP:
|
||||
log(f" skip large log {name} ({info.file_size}B)")
|
||||
continue
|
||||
try:
|
||||
text = z.read(name).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
continue
|
||||
if not text:
|
||||
continue
|
||||
for prov_key in scan_text(text):
|
||||
prov, key = prov_key
|
||||
if is_junk(key):
|
||||
continue
|
||||
run_url = f"{repo_full}/actions/runs/{run_id}"
|
||||
db_add(key, prov, run_url)
|
||||
n_keys += 1
|
||||
return n_keys
|
||||
|
||||
def download_and_scan_logs(repo_full, run_id):
|
||||
"""Download log zip for a run, scan it. Returns key count or -1."""
|
||||
url = f"{API}/repos/{repo_full}/actions/runs/{run_id}/logs"
|
||||
r = _get(url, stream=True)
|
||||
if r is None:
|
||||
return -1
|
||||
cl = r.headers.get("Content-Length")
|
||||
size_hint = int(cl) if cl and cl.isdigit() else None
|
||||
if size_hint and size_hint > ARCHIVE_CAP:
|
||||
log(f" skip oversized log {repo_full} run {run_id}: {size_hint}B")
|
||||
r.close()
|
||||
return 0
|
||||
chunks = []
|
||||
try:
|
||||
for chunk in r.iter_content(65536):
|
||||
chunks.append(chunk)
|
||||
if sum(len(c) for c in chunks) > ARCHIVE_CAP:
|
||||
log(f" abort oversized log {repo_full} run {run_id}")
|
||||
r.close()
|
||||
return 0
|
||||
finally:
|
||||
r.close()
|
||||
data = b"".join(chunks)
|
||||
return scan_logs_zip(data, repo_full, run_id)
|
||||
|
||||
# ── Process one repo ────────────────────────────────────
|
||||
|
||||
def process_repo(repo_full):
|
||||
"""Check for completed runs, scan logs. Returns (keys_found, fails)."""
|
||||
if len(seen) > 10000:
|
||||
log(" seen set full, pausing this repo")
|
||||
return 0, 0
|
||||
runs = fetch_runs(repo_full)
|
||||
if runs is None:
|
||||
return 0, 1 # network fail
|
||||
if not runs:
|
||||
# No runs? Ignored — just cost 1 request
|
||||
return 0, 0
|
||||
kf, fl = 0, 0
|
||||
capped = 0
|
||||
for run in runs:
|
||||
run_id = run["id"]
|
||||
sk = f"{repo_full}/{run_id}"
|
||||
if sk in seen:
|
||||
continue
|
||||
mk = download_and_scan_logs(repo_full, run_id)
|
||||
if mk >= 0:
|
||||
kf += mk
|
||||
mark_seen(sk)
|
||||
else:
|
||||
fl += 1
|
||||
capped += 1
|
||||
if capped >= 5: # max 5 runs per repo per cycle
|
||||
break
|
||||
return kf, fl
|
||||
|
||||
# ── Cycle ───────────────────────────────────────────────
|
||||
|
||||
def cycle(once=False):
|
||||
"""One full pass: discover repos, scan their action log zips."""
|
||||
global total
|
||||
cursor = load_cursor()
|
||||
repos_checked = 0
|
||||
keys_found = 0
|
||||
fail = 0
|
||||
max_repos = 5 if once else BATCH
|
||||
since = cursor
|
||||
|
||||
log(f"cycle starting from since={since}")
|
||||
|
||||
while repos_checked < max_repos:
|
||||
repos_list, next_since = fetch_repos(since)
|
||||
if not repos_list:
|
||||
log(" no more repos, wrap around")
|
||||
since = 1
|
||||
continue
|
||||
for repo in repos_list:
|
||||
if repos_checked >= max_repos:
|
||||
break
|
||||
full_name = repo.get("full_name", "")
|
||||
if not full_name:
|
||||
continue
|
||||
if repo.get("fork") is True:
|
||||
continue # forks are noise, rarely run actions
|
||||
if repo.get("archived") is True:
|
||||
continue
|
||||
if repo.get("private") is True:
|
||||
continue
|
||||
repos_checked += 1
|
||||
kf, fl = process_repo(full_name)
|
||||
keys_found += kf
|
||||
fail += fl
|
||||
if once:
|
||||
break # --once: just one repo
|
||||
since = next_since
|
||||
# Save cursor periodically
|
||||
if repos_checked % 5 == 0:
|
||||
save_cursor(since)
|
||||
|
||||
save_cursor(since)
|
||||
log(f"cycle done: {repos_checked} repos, {keys_found} keys, {fail} fail +{total} total")
|
||||
return repos_checked, keys_found, fail
|
||||
|
||||
# ── Main ────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
log("gha_logs_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle(once=True)
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle error: {e}")
|
||||
import traceback
|
||||
log(traceback.format_exc())
|
||||
time.sleep(SLEEP)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user