110 lines
3.5 KiB
Python
110 lines
3.5 KiB
Python
"""Retro full-file scan: all (repo, path) hit-pairs already in DB.
|
|
Cycle parser fetches full files only for current-cycle hits (runtime dedup);
|
|
this one-shot re-harvests every historical hit with the complete file body."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
sys.path.insert(0, KF_BASE)
|
|
from gh_eternal import PATTERNS, GARBAGE, POOL # noqa: E402
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
N_THREADS = 16
|
|
|
|
|
|
def log(msg):
|
|
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
|
|
|
|
|
def load_targets():
|
|
db = sqlite3.connect(DB, timeout=60)
|
|
c = db.cursor()
|
|
c.execute("SELECT repo FROM keys WHERE repo LIKE '%/%' "
|
|
"AND found IN ('gh-eternal','gh-dork2','legacy-waves','wiki')")
|
|
pairs = set()
|
|
for (r,) in c.fetchall():
|
|
if r.count("/") >= 1 and len(r) < 200 and "…" not in r:
|
|
parts = r.split("/")
|
|
repo = "/".join(parts[:2])
|
|
path = "/".join(parts[2:])
|
|
if repo and path and not path.startswith("commit"):
|
|
pairs.add((repo, path))
|
|
db.close()
|
|
return list(pairs)
|
|
|
|
|
|
def fetch_and_scan(pair, tok):
|
|
repo, path = pair
|
|
try:
|
|
import requests
|
|
r = requests.get(
|
|
f"https://raw.githubusercontent.com/{repo}/HEAD/{path}",
|
|
headers={"Authorization": f"Bearer {tok}",
|
|
"User-Agent": "keyfarm"},
|
|
timeout=20)
|
|
if r.status_code != 200 or len(r.text) > 2_000_000:
|
|
return pair, None
|
|
found = []
|
|
for prov, rx in PATTERNS:
|
|
for key in rx.findall(r.text):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
found.append((prov, key))
|
|
return pair, found
|
|
except Exception:
|
|
return pair, None
|
|
|
|
|
|
def main():
|
|
pairs = load_targets()
|
|
log(f"targets: {len(pairs)} files")
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
|
|
cur = db.cursor()
|
|
seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
|
|
new_total = 0
|
|
lock = __import__("threading").Lock()
|
|
|
|
def work(pair):
|
|
nonlocal new_total
|
|
tok = POOL.tokens[hash(pair[0]) % len(POOL.tokens)]
|
|
pair, found = fetch_and_scan(pair, tok)
|
|
n = 0
|
|
if found:
|
|
for prov, key in found:
|
|
if key in seen:
|
|
continue
|
|
with lock:
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO keys "
|
|
"(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
|
"NEW", "retro-scan", f"{pair[0]}/{pair[1]}"))
|
|
new_total += 1
|
|
n += 1
|
|
return n
|
|
|
|
with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
|
|
futs = {ex.submit(work, p): p for p in pairs}
|
|
done = 0
|
|
for fut in as_completed(futs):
|
|
done += 1
|
|
if done % 200 == 0:
|
|
db.commit()
|
|
log(f"{done}/{len(pairs)} scanned, +{new_total} new")
|
|
db.commit()
|
|
log(f"done: +{new_total} new keys")
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |