KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
223
pubdev_eternal.py
Normal file
223
pubdev_eternal.py
Normal file
@@ -0,0 +1,223 @@
|
||||
"""ETERNAL pub.dev firehose: /api/packages page-walk + latest archive scan.
|
||||
|
||||
Anonymous. Cursor = page number (next_url), walks a slice of pages per cycle,
|
||||
resets to 1 when list exhausted (new packages appear at head). tar.gz scan
|
||||
via key_patterns.scan_text. mark_seen ONLY on success.
|
||||
"""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
|
||||
import io
|
||||
import sqlite3
|
||||
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "data/pubdev_eternal.log")
|
||||
CURSOR_FILE = os.path.join(KF_BASE, "data/pubdev_cursor.txt")
|
||||
SEEN_FILE = os.path.join(KF_BASE, "data/pubdev_seen.txt")
|
||||
N_THREADS = 4
|
||||
PAGES_PER_CYCLE = 5
|
||||
TAR_CAP = 8 * 1024 * 1024
|
||||
FILE_CAP = 500_000
|
||||
SLEEP = 90
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
SCAN_EXT = (".dart", ".json", ".yaml", ".yml", ".env", ".sh", ".py", ".js",
|
||||
".ts", ".cfg", ".ini", ".md", ".txt", ".gradle", ".lock")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-research/1.0 (package security scan)"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
with lock:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "pubdev", repo))
|
||||
total += 1
|
||||
|
||||
|
||||
def load_cursor():
|
||||
try:
|
||||
with open(CURSOR_FILE, encoding="utf-8") as f:
|
||||
return int(f.read().strip() or 1)
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
|
||||
def save_cursor(c):
|
||||
with open(CURSOR_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(c))
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
s.add(line.strip())
|
||||
except Exception:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def work(name):
|
||||
"""Fetch package meta -> latest tar.gz, scan. -1 on final fail."""
|
||||
n = 0
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
r = requests.get(f"https://pub.dev/api/packages/{name}",
|
||||
headers=UA, timeout=20)
|
||||
if r.status_code == 404:
|
||||
return 0
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
au = r.json().get("latest", {}).get("archive_url")
|
||||
if not au:
|
||||
return 0
|
||||
if not au.startswith("https"):
|
||||
au = "https://pub.dev" + au
|
||||
r2 = requests.get(au, headers=UA, timeout=30)
|
||||
if r2.status_code != 200:
|
||||
raise RuntimeError(f"arch http {r2.status_code}")
|
||||
if len(r2.content) > TAR_CAP:
|
||||
return 0
|
||||
tf = tarfile.open(fileobj=io.BytesIO(r2.content), mode="r:gz")
|
||||
for m in tf.getmembers():
|
||||
if not m.isfile() or m.size > FILE_CAP:
|
||||
continue
|
||||
fn = m.name.lower()
|
||||
if not any(fn.endswith(e) for e in SCAN_EXT):
|
||||
continue
|
||||
ef = tf.extractfile(m)
|
||||
if ef is None:
|
||||
continue
|
||||
data = ef.read()
|
||||
text = data.decode("utf-8", errors="ignore")
|
||||
for prov, key in scan_text(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
db_add(key, prov, f"pub:{name}")
|
||||
n += 1
|
||||
return n
|
||||
except Exception:
|
||||
if attempt == 2:
|
||||
log(f" pkg-err {name}")
|
||||
return -1
|
||||
time.sleep(3)
|
||||
return -1
|
||||
|
||||
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
page = load_cursor()
|
||||
names = []
|
||||
exhausted = False
|
||||
for _ in range(PAGES_PER_CYCLE):
|
||||
url = f"https://pub.dev/api/packages?page={page}"
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
r = requests.get(url, headers=UA, timeout=25)
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"http {r.status_code}")
|
||||
d = r.json()
|
||||
pkgs = d.get("packages") or []
|
||||
for pk in pkgs:
|
||||
nm = pk.get("name")
|
||||
if nm and nm not in seen:
|
||||
names.append(nm)
|
||||
nxt = d.get("next_url")
|
||||
if not nxt:
|
||||
exhausted = True
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
log(f"list-err page={page} {type(e).__name__}")
|
||||
return
|
||||
time.sleep(5)
|
||||
if exhausted:
|
||||
break
|
||||
page += 1
|
||||
|
||||
if exhausted:
|
||||
save_cursor(1)
|
||||
else:
|
||||
save_cursor(page)
|
||||
|
||||
ok = fail = 0
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
||||
futs = {ex.submit(work, nm): nm for nm in names}
|
||||
for fut in as_completed(futs):
|
||||
nm = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception:
|
||||
res = -1
|
||||
if res == -1:
|
||||
fail += 1
|
||||
else:
|
||||
ok += 1
|
||||
mark_seen(nm)
|
||||
|
||||
log(f"cycle: page_next={load_cursor()} pkgs={len(names)} ok={ok} "
|
||||
f"fail={fail} +{total} keys")
|
||||
|
||||
|
||||
def main():
|
||||
log("pubdev_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
cycle()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
cycle()
|
||||
except Exception as e:
|
||||
log(f"cycle-err {type(e).__name__}: {e}")
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user