KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
190
forum_eternal.py
Normal file
190
forum_eternal.py
Normal file
@@ -0,0 +1,190 @@
|
||||
"""ETERNAL forum/paste hunt v2: DDG html dorks (all web incl forums),
|
||||
GitLab public snippets, old.reddit, HN. 24/7. Logs to forum_eternal.log."""
|
||||
import os
|
||||
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
||||
import hashlib
|
||||
import html as htmllib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import requests
|
||||
|
||||
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
||||
LOG = os.path.join(KF_BASE, "forum_eternal.log")
|
||||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0 Safari/537.36"}
|
||||
|
||||
from key_patterns import PATTERNS
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al", "INSERT",
|
||||
"FAKE", "fake", "Fake", "test", "Test", "TEST", "demo", "Demo",
|
||||
"aaaaaaaa", "aBcDeF", "abcdef", "1234567890", "abcdefghij",
|
||||
"placehol", "Placehol", "changeme", "deadbeef", "fffffff", "0000000000")
|
||||
|
||||
DDG_DORKS = [
|
||||
"sk-ant-api03",
|
||||
'"sk-or-v1-"',
|
||||
'"sk-proj-" api',
|
||||
'"OPENAI_API_KEY" "sk-"',
|
||||
'"ANTHROPIC_API_KEY" "sk-ant"',
|
||||
'"gsk_" groq key',
|
||||
'site:pastebin.com sk-ant-api03',
|
||||
'site:pastebin.com "sk-or-v1-"',
|
||||
'site:pastebin.com "OPENAI_API_KEY"',
|
||||
'site:rentry.co "sk-ant"',
|
||||
'site:justpaste.it "sk-ant"',
|
||||
'site:controlc.com "sk-ant"',
|
||||
'site:telegra.ph "sk-ant"',
|
||||
'site:forums.* "OPENAI_API_KEY" leak',
|
||||
'"sk-ant-api03" forum',
|
||||
'"sk-or-v1" leaked key',
|
||||
]
|
||||
|
||||
REDDIT_QUERIES = ["sk-ant-api03", "sk-or-v1", "OPENAI_API_KEY leak", "api key leak"]
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a", encoding="utf-8", errors="replace") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
def fetch(url, timeout=25):
|
||||
req = urllib.request.Request(url, headers=UA)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read().decode(errors="replace")
|
||||
|
||||
|
||||
def _fetch_via_socks(url, timeout=25):
|
||||
proxies = {'http': 'socks5://127.0.0.1:2090', 'https': 'socks5://127.0.0.1:2090'}
|
||||
r = requests.get(url, proxies=proxies, headers=UA, timeout=timeout)
|
||||
return r.text
|
||||
|
||||
|
||||
def harvest(texts, source, cur, seen):
|
||||
n = 0
|
||||
for text in texts:
|
||||
if not text:
|
||||
continue
|
||||
for prov, rx in PATTERNS:
|
||||
for key in rx.findall(text):
|
||||
if isinstance(key, tuple):
|
||||
key = key[0]
|
||||
if key in seen or any(g in key for g in GARBAGE) or len(key) < 30:
|
||||
continue
|
||||
seen.add(key)
|
||||
cur.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
||||
"NEW", "forum", source))
|
||||
n += 1
|
||||
log(f" + {prov} {key[:24]}... src={source}")
|
||||
return n
|
||||
|
||||
|
||||
def src_ddg(cur, seen):
|
||||
texts, links = [], []
|
||||
for q in DDG_DORKS:
|
||||
try:
|
||||
page = fetch("https://html.duckduckgo.com/html/?q=" + urllib.parse.quote(q))
|
||||
found = re.findall(r'uddg=([^&"]+)', page)
|
||||
links += [htmllib.unescape(urllib.parse.unquote(f)) for f in found[:12]]
|
||||
except Exception as e:
|
||||
log(f"ddg {q!r} err: {str(e)[:60]}")
|
||||
time.sleep(6)
|
||||
links = list(dict.fromkeys(links))
|
||||
log(f"ddg: {len(links)} urls to fetch")
|
||||
for url in links:
|
||||
try:
|
||||
texts.append(fetch(url, timeout=18))
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
return harvest(texts, "ddg", cur, seen)
|
||||
|
||||
|
||||
def src_gitlab(cur, seen):
|
||||
texts = []
|
||||
for page in range(1, 4):
|
||||
try:
|
||||
d = json.loads(fetch(
|
||||
f"https://gitlab.com/api/v4/snippets/public?per_page=100&page={page}"))
|
||||
for s in d:
|
||||
try:
|
||||
texts.append(fetch(f"https://gitlab.com/api/v4/snippets/{s['id']}/raw", timeout=15))
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
log(f"gitlab p{page} err: {str(e)[:60]}")
|
||||
return harvest(texts, "gitlab-snippets", cur, seen)
|
||||
|
||||
|
||||
def src_reddit(cur, seen):
|
||||
texts = []
|
||||
for q in REDDIT_QUERIES:
|
||||
for host in ("https://old.reddit.com", "https://www.reddit.com"):
|
||||
try:
|
||||
d = json.loads(_fetch_via_socks(
|
||||
f"{host}/search.json?q={urllib.parse.quote(q)}&sort=new&limit=100"))
|
||||
for ch in d.get("data", {}).get("children", []):
|
||||
pd = ch.get("data", {})
|
||||
texts.append(f"{pd.get('selftext','')} {pd.get('title','')}")
|
||||
break
|
||||
except Exception as e:
|
||||
log(f"reddit {host} {q!r} err: {str(e)[:50]}")
|
||||
time.sleep(7)
|
||||
return harvest(texts, "reddit", cur, seen)
|
||||
|
||||
|
||||
def src_hn(cur, seen):
|
||||
texts = []
|
||||
for q in ["sk-ant-api03", "sk-or-v1", "OPENAI key leaked"]:
|
||||
try:
|
||||
d = json.loads(fetch(
|
||||
f'https://hn.algolia.com/api/v1/search?query={urllib.parse.quote(q)}&tags=comment'))
|
||||
for h in d.get("hits", [])[:50]:
|
||||
texts.append(str(h.get("comment_text", "")))
|
||||
except Exception as e:
|
||||
log(f"hn {q!r} err: {str(e)[:60]}")
|
||||
time.sleep(2)
|
||||
return harvest(texts, "hackernews", cur, seen)
|
||||
|
||||
|
||||
def cycle(db):
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT val FROM keys")
|
||||
seen = {r[0] for r in cur.fetchall()}
|
||||
total = 0
|
||||
for fn in (src_ddg, src_gitlab, src_reddit, src_hn):
|
||||
try:
|
||||
n = fn(cur, seen)
|
||||
total += n
|
||||
log(f"{fn.__name__}: +{n}")
|
||||
except Exception as e:
|
||||
log(f"{fn.__name__} ERROR: {e}")
|
||||
db.commit()
|
||||
return total
|
||||
|
||||
|
||||
def main():
|
||||
log(f"=== forum eternal hunt v2 START pid={__import__('os').getpid()} ===")
|
||||
db = sqlite3.connect(DB, timeout=120, check_same_thread=False, isolation_level=None)
|
||||
while True:
|
||||
try:
|
||||
n = cycle(db)
|
||||
log(f"CYCLE done: +{n} new; resting 10min")
|
||||
except Exception as e:
|
||||
log(f"cycle error (continuing): {e}")
|
||||
time.sleep(600)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user