146 lines
5.0 KiB
Python
146 lines
5.0 KiB
Python
"""ETERNAL HuggingFace hunt: Spaces/Models/Datasets search → README/config scan.
|
|
Independent API lane, no token needed for public search.
|
|
"""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
|
|
import requests
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "hf_eternal.log")
|
|
REST = 60
|
|
|
|
# search terms likely to hit leaked secrets in HF spaces READMEs/configs
|
|
QUERIES = [
|
|
"OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DASHSCOPE_API_KEY", "DEEPSEEK_API_KEY",
|
|
"ZHIPU_API_KEY", "GLM_API_KEY", "MOONSHOT_API_KEY", "KIMI_API_KEY",
|
|
"OPENROUTER_API_KEY", "GROQ_API_KEY", "TOGETHER_API_KEY", "FIREWORKS_API_KEY",
|
|
"PERPLEXITY_API_KEY", "REPLICATE_API_TOKEN", "ELEVENLABS_API_KEY",
|
|
"GEMINI_API_KEY", "GOOGLE_API_KEY", "SILICONFLOW_API_KEY", "COHERE_API_KEY",
|
|
"HF_TOKEN", "TOGETHER_API_KEY config", "llm api key", "apikey", "api_key",
|
|
]
|
|
|
|
from key_patterns import PATTERNS
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "FAKE",
|
|
"fake", "test", "Test", "TEST", "demo", "Demo", "aaaaaaaa", "abcdef",
|
|
"1234567890", "abcdefghij", "placehol", "Placehol", "changeme",
|
|
"deadbeef", "fffffff", "0000000000", "INSERT", "REEMPLAZA", "AQUI")
|
|
|
|
HEADERS = {"User-Agent": "Mozilla/5.0"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
print(line, flush=True)
|
|
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
|
f.write(line)
|
|
|
|
|
|
def hf_search(kind, q):
|
|
"""kind: spaces|models|datasets. Returns list of item ids."""
|
|
try:
|
|
r = requests.get(
|
|
f"https://huggingface.co/api/{kind}",
|
|
params={"search": q, "limit": 100, "full": "false"},
|
|
headers=HEADERS, timeout=30)
|
|
if r.status_code != 200:
|
|
return None
|
|
items = r.json()
|
|
return [i.get("id", "?") for i in items[:100]]
|
|
except Exception as e:
|
|
log(f" search {kind}:{q!r} error: {e}")
|
|
return None
|
|
|
|
|
|
def fetch_files(kind, repo_id):
|
|
"""Pull README/config blobs from an HF space/model/dataset."""
|
|
blobs = []
|
|
files = ("README.md", "app.py", "config.json", "settings.json", ".env")
|
|
if kind == "models":
|
|
files = ("README.md", "config.json", "settings.json", ".env")
|
|
for fn in files:
|
|
try:
|
|
u = f"https://huggingface.co/{kind}/{repo_id}/raw/main/{fn}"
|
|
r = requests.get(u, headers=HEADERS, timeout=20)
|
|
if r.status_code == 200 and len(r.text) < 500_000:
|
|
blobs.append((fn, r.text))
|
|
except Exception:
|
|
pass
|
|
return blobs
|
|
|
|
|
|
def extract(cur, seen, blobs, repo):
|
|
new_n, provs = 0, {}
|
|
for fn, text in blobs:
|
|
for prov, rx in PATTERNS:
|
|
for key in rx.findall(text):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if key in seen or any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
|
"NEW", "hf-eternal", f"{repo}/{fn}"))
|
|
new_n += 1
|
|
provs[prov] = provs.get(prov, 0) + 1
|
|
return new_n, provs
|
|
|
|
|
|
def cycle(db):
|
|
cur = db.cursor()
|
|
cur.execute("SELECT val FROM keys")
|
|
seen = {r[0] for r in cur.fetchall()}
|
|
total, seen_repos = 0, set()
|
|
for kind in ("spaces", "models", "datasets"):
|
|
for q in QUERIES:
|
|
ids = hf_search(kind, q)
|
|
if ids is None:
|
|
time.sleep(10)
|
|
continue
|
|
log(f"{kind} {q!r}: {len(ids)} results")
|
|
for rid in ids:
|
|
if rid in seen_repos:
|
|
continue
|
|
seen_repos.add(rid)
|
|
blobs = fetch_files(kind, rid)
|
|
if not blobs:
|
|
continue
|
|
n, provs = extract(cur, seen, blobs, rid)
|
|
if n:
|
|
log(f" {rid}: +{n} {provs}")
|
|
total += n
|
|
time.sleep(1.5)
|
|
db.commit()
|
|
return total, len(seen_repos)
|
|
|
|
|
|
def main():
|
|
log(f"=== HF eternal hunt START pid={__import__('os').getpid()} ===")
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
db.execute("PRAGMA busy_timeout=120000")
|
|
while True:
|
|
try:
|
|
n, repos = cycle(db)
|
|
log(f"CYCLE done: scanned {repos} repos, +{n} new; resting {REST}s")
|
|
except Exception as e:
|
|
log(f"CYCLE ERROR: {e!r}")
|
|
time.sleep(REST)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
sys.exit(0) |