131 lines
4.0 KiB
Python
131 lines
4.0 KiB
Python
"""ETERNAL Hacker News harvest: hn.algolia.com search API.
|
|
Scans story_text + comment_text + title for leaked API keys.
|
|
No quota; polite sleep. Lane: hn."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import urllib.parse
|
|
import urllib.error
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "hn_eternal.log")
|
|
|
|
QUERIES = [
|
|
"sk-ant-api03", "sk-proj", "sk-or-v1", "gsk_", "tgp_",
|
|
"pplx-", "nvapi-", "r8_", "xai-",
|
|
"OPENAI_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY",
|
|
"leaked api key", "leaked keys github", "api key in .env",
|
|
"accidentally pushed keys", "openai key leaked",
|
|
"anthropic key leaked", "hardcoded api key",
|
|
]
|
|
|
|
from key_patterns import PATTERNS
|
|
|
|
GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
|
|
"fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
|
|
"aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
|
|
"changeme", "deadbeef", "fffffff", "0000000")
|
|
|
|
seen = set()
|
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
print(line, flush=True)
|
|
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
|
f.write(line)
|
|
|
|
|
|
def db_add(cur, key, prov, repo):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
|
|
return 0
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
|
|
"hn", repo))
|
|
return 1
|
|
|
|
|
|
def fetch_json(url):
|
|
req = urllib.request.Request(url, headers=UA)
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return json.loads(r.read().decode("utf-8", "replace"))
|
|
|
|
|
|
def scan_text(cur, text, src):
|
|
n = 0
|
|
for prov, rx in PATTERNS:
|
|
for m in rx.findall(text or ""):
|
|
n += db_add(cur, m, prov, src)
|
|
return n
|
|
|
|
|
|
def cycle(db):
|
|
cur = db.cursor()
|
|
total = 0
|
|
for q in QUERIES:
|
|
n = 0
|
|
for ep in ("search", "search_by_date"):
|
|
url = ("https://hn.algolia.com/api/v1/" + ep +
|
|
"?query=" + urllib.parse.quote(q) + "&hitsPerPage=50")
|
|
try:
|
|
data = fetch_json(url)
|
|
except Exception as e:
|
|
log(f" {q} {ep} err: {str(e)[:60]}")
|
|
continue
|
|
for h in data.get("hits", []):
|
|
text = " ".join(filter(None, [
|
|
h.get("title"), h.get("story_text"),
|
|
h.get("comment_text"), h.get("url")]))
|
|
n += scan_text(cur, text, f"hn:{h.get('objectID')}")
|
|
time.sleep(1.2)
|
|
db.commit()
|
|
total += n
|
|
log(f" '{q}': +{n} new")
|
|
return total
|
|
|
|
|
|
def main():
|
|
log(f"=== hn eternal START pid={__import__('os').getpid()} "
|
|
f"queries={len(QUERIES)}")
|
|
try:
|
|
con = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
cur = con.cursor()
|
|
cur.execute("SELECT val FROM keys WHERE found='hn'")
|
|
for r in cur.fetchall():
|
|
seen.add(r[0])
|
|
con.close()
|
|
except Exception:
|
|
pass
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
while True:
|
|
try:
|
|
n = cycle(db)
|
|
log(f"CYCLE done +{n}")
|
|
except Exception as e:
|
|
log(f"cycle err: {repr(e)[:100]}")
|
|
time.sleep(30)
|
|
time.sleep(90)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
sys.exit(0)
|