KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
283
jenkins_eternal.py.disabled
Normal file
283
jenkins_eternal.py.disabled
Normal file
@@ -0,0 +1,283 @@
|
||||
"""ETERNAL Jenkins: probe public Jenkins instances, fetch lastBuild consoleText.
|
||||
|
||||
Endpoint(s): /api/json?tree=jobs[name,url] + /job/<name>/lastBuild/consoleText
|
||||
Auth: none (instances that require auth are skipped)
|
||||
"""
|
||||
import os as _os
|
||||
BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
|
||||
import hashlib, json, re, sqlite3, sys, threading, time
|
||||
import requests
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(BASE, "data", "jenkins_eternal.log")
|
||||
CURSOR_FILE = _os.path.join(BASE, "data", "jenkins_cursor.txt")
|
||||
SEEN_FILE = _os.path.join(BASE, "data", "jenkins_seen.txt")
|
||||
DEAD_FILE = _os.path.join(BASE, "data", "jenkins_dead.txt")
|
||||
N_THREADS = 4
|
||||
SLEEP = 120
|
||||
MAX_ITEMS = 30
|
||||
FILE_CAP = 500_000
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
# Public Jenkins instances known to exist (static seed, updated periodically).
|
||||
# Only instances responding 200 with a job list on /api/json get walked.
|
||||
SEED_HOSTS = sorted([
|
||||
"https://jenkins.debian.net",
|
||||
"https://ci.freebsd.org/jenkins",
|
||||
"https://ci.jenkins.io",
|
||||
"https://ci.jenkins-ci.org",
|
||||
"https://build.opensuse.org",
|
||||
"https://ci.eclipse.org",
|
||||
"https://ci.inria.fr",
|
||||
"https://jenkins.centos.org",
|
||||
"https://jenkins.istio.io",
|
||||
"https://ci.mariadb.org",
|
||||
"https://ci.xwiki.org",
|
||||
"https://ci.xwiki.org/jenkins",
|
||||
"https://jenkins.openwrt.org",
|
||||
"https://jenkins.rdoproject.org",
|
||||
"https://jenkins.linuxfoundation.org",
|
||||
"https://jenkins.apache.org",
|
||||
"https://jenkins.qt.io",
|
||||
"https://jenkins.ros.org",
|
||||
"https://jenkins.ros2.org",
|
||||
"https://ci.libvirt.org",
|
||||
"https://jenkins.fedoraproject.org",
|
||||
"https://ci.fedoraproject.org",
|
||||
"https://ci.suse.de",
|
||||
"https://ci.suse.com",
|
||||
"https://ci.ubuntu.com",
|
||||
"https://jenkins.kernelci.org",
|
||||
"https://ci.centos.org:8080",
|
||||
])
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-jenkins/1.0"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
hashed = hashlib.sha1(key.encode()).hexdigest()[:16]
|
||||
with lock:
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashed, key, prov, "NEW", "jenkins", repo[:300]))
|
||||
conn.commit()
|
||||
total += 1
|
||||
except Exception as e:
|
||||
log(f"db_add error: {e}")
|
||||
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, "r") as f:
|
||||
for line in f:
|
||||
s.add(line.strip())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(build_id):
|
||||
seen.add(build_id)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(build_id + "\n")
|
||||
|
||||
|
||||
def load_dead():
|
||||
s = set()
|
||||
try:
|
||||
with open(DEAD_FILE, "r") as f:
|
||||
for line in f:
|
||||
s.add(line.strip())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
def mark_dead(host):
|
||||
with open(DEAD_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(host + "\n")
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE, "r") as f:
|
||||
return int(f.read().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def probe_host(host, S):
|
||||
"""Probe a Jenkins host; return (alive, job_urls) or (False, reason)."""
|
||||
api = f"{host}/api/json?tree=jobs%5bname,url%5d"
|
||||
try:
|
||||
r = S.get(api, timeout=20,
|
||||
headers={"User-Agent": "keyfarm-jenkins/1.0"})
|
||||
if r.status_code != 200:
|
||||
return False, f"HTTP {r.status_code}"
|
||||
data = r.json()
|
||||
jobs = data.get("jobs", [])
|
||||
if not jobs:
|
||||
return False, "no jobs in response"
|
||||
urls = [(j["name"], j["url"]) for j in jobs
|
||||
if "name" in j and "url" in j]
|
||||
return True, urls
|
||||
except Exception as e:
|
||||
return False, str(e)[:100]
|
||||
|
||||
|
||||
def fetch_console(job_url, S):
|
||||
"""Fetch last build console text; return text or None."""
|
||||
url = f"{job_url.rstrip('/')}/lastBuild/consoleText"
|
||||
try:
|
||||
r = S.get(url, timeout=30, stream=True,
|
||||
headers={"User-Agent": "keyfarm-jenkins/1.0"})
|
||||
if r.status_code != 200:
|
||||
r.close()
|
||||
return None
|
||||
buf = b""
|
||||
for chunk in r.iter_content(65536):
|
||||
buf += chunk
|
||||
if len(buf) > FILE_CAP:
|
||||
break
|
||||
r.close()
|
||||
return buf.decode("utf-8", "replace")[:FILE_CAP]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def work(host, job_name, job_url, S):
|
||||
"""Fetch console text and scan for keys. Return key count or -1."""
|
||||
text = fetch_console(job_url, S)
|
||||
if text is None:
|
||||
return -1
|
||||
build_id = f"{host}|{job_name}|lastBuild"
|
||||
if build_id in seen:
|
||||
return 0
|
||||
hits = scan_text(text)
|
||||
found = 0
|
||||
for prov, key in hits:
|
||||
if is_junk(key):
|
||||
continue
|
||||
repo = f"{host}/job/{job_name}"
|
||||
db_add(key, prov, repo)
|
||||
found += 1
|
||||
mark_seen(build_id)
|
||||
if found:
|
||||
log(f" {host}/{job_name}: {found} keys")
|
||||
return found
|
||||
|
||||
|
||||
def cycle():
|
||||
"""One cycle: probe the next MAX_ITEMS hosts, scan their jobs."""
|
||||
global total
|
||||
dead = load_dead()
|
||||
cursor = load_cursor()
|
||||
hosts = [h for h in SEED_HOSTS if h not in dead]
|
||||
if not hosts:
|
||||
log("All hosts dead; resetting dead list")
|
||||
with open(DEAD_FILE, "w") as f:
|
||||
f.write("")
|
||||
dead = set()
|
||||
hosts = SEED_HOSTS[:]
|
||||
|
||||
# Determine this cycle's host slice
|
||||
cycle_hosts = hosts[cursor: cursor + MAX_ITEMS]
|
||||
if not cycle_hosts:
|
||||
# Wrap around
|
||||
cycle_hosts = hosts[:MAX_ITEMS]
|
||||
cursor = 0
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False
|
||||
fail = 0
|
||||
|
||||
log(f"Cycle: probing {len(cycle_hosts)} hosts (cursor={cursor})")
|
||||
|
||||
for host in cycle_hosts:
|
||||
alive, result = probe_host(host, S)
|
||||
if not alive:
|
||||
log(f" {host}: DEAD ({result})")
|
||||
mark_dead(host)
|
||||
fail += 1
|
||||
continue
|
||||
jobs = result # list of (name, url)
|
||||
|
||||
# Jobs exceed budget?
|
||||
job_slice = jobs[:5] # conservative per host
|
||||
log(f" {host}: ALIVE ({len(jobs)} jobs, scanning {len(job_slice)})")
|
||||
|
||||
for job_name, job_url in job_slice:
|
||||
keys = work(host, job_name, job_url, S)
|
||||
if keys < 0:
|
||||
fail += 1
|
||||
time.sleep(2) # polite delay between builds
|
||||
|
||||
time.sleep(3) # polite delay between hosts
|
||||
|
||||
new_cursor = (cursor + len(cycle_hosts)) % len(hosts) if hosts else 0
|
||||
save_cursor(new_cursor)
|
||||
|
||||
log(f"Cycle done: +{total} keys, {fail} fails")
|
||||
log("---")
|
||||
|
||||
|
||||
def main():
|
||||
log("jenkins_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
global MAX_ITEMS
|
||||
MAX_ITEMS = min(MAX_ITEMS, 5)
|
||||
log("--once mode")
|
||||
cycle()
|
||||
log("--once done")
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle error: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user