KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
304
iascholar_eternal.py
Normal file
304
iascholar_eternal.py
Normal file
@@ -0,0 +1,304 @@
|
||||
"""ETERNAL IA-text + OSF: Solr metadata search + OSF file walk.
|
||||
|
||||
Primary: Internet Archive Solr API — targeted queries for items with key
|
||||
patterns in metadata (description/title). Skips .bundle/software downloads.
|
||||
Secondary: OSF public nodes — shallow file download (3 nodes, 1 level).
|
||||
"""
|
||||
import os as _os
|
||||
_BASE = _os.path.dirname(_os.path.abspath(__file__))
|
||||
_os.makedirs(_os.path.join(_BASE, "data"), exist_ok=True)
|
||||
|
||||
import hashlib, sqlite3, sys, threading, time
|
||||
import requests
|
||||
|
||||
S = requests.Session()
|
||||
S.trust_env = False # contract 4
|
||||
|
||||
DB = _os.environ.get("KF_DB") or _os.path.join(_BASE, "data", "keys.db")
|
||||
LOG = _os.path.join(_BASE, "data", "iascholar_eternal.log")
|
||||
SEEN_FILE = _os.path.join(_BASE, "data", "iascholar_seen.txt")
|
||||
SLEEP = 180 # v7.5: smaller archive.org share (shared host with wayback)
|
||||
MAX_ITEMS = 20
|
||||
FILE_CAP = 500_000
|
||||
|
||||
from key_patterns import scan_text
|
||||
|
||||
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
||||
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
||||
"abcdefghijklmnopqrst", "1234567890abcdef")
|
||||
|
||||
UA = {"User-Agent": "keyfarm-iascholar/1.0"}
|
||||
|
||||
# IA queries: target description field where keys appear in metadata.
|
||||
# Full-text search catches keys in git bundles (unreadable) — description:
|
||||
# prefix limits to the metadata we can actually extract keys from.
|
||||
IA_QUERIES = [
|
||||
'description:(sk-proj OR "ghp_" OR AKIA) AND mediatype:software',
|
||||
'description:(xoxb OR sk_live OR pk_live) AND mediatype:software',
|
||||
'description:(api_key OR secret) AND collection:github',
|
||||
]
|
||||
_ia_qidx = 0
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = "[%s] %s\n" % (time.strftime('%Y-%m-%d %H:%M:%S'), msg)
|
||||
with open(LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
|
||||
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
||||
isolation_level=None)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
||||
plan TEXT, price TEXT, remaining TEXT,
|
||||
found TEXT, repo TEXT)""")
|
||||
conn.commit()
|
||||
lock = threading.Lock()
|
||||
total = 0
|
||||
|
||||
|
||||
def db_add(key, prov, repo):
|
||||
global total
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
||||
"VALUES (?,?,?,?,?,?) "
|
||||
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
||||
"found=excluded.found, repo=excluded.repo "
|
||||
"WHERE keys.status IN ('DEAD','ERR')",
|
||||
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
||||
key, prov, "NEW", "ia-text", repo))
|
||||
conn.commit()
|
||||
with lock:
|
||||
total += 1
|
||||
except Exception as e:
|
||||
log("db_add error: %s" % e)
|
||||
|
||||
|
||||
def load_seen():
|
||||
s = set()
|
||||
try:
|
||||
with open(SEEN_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
s.add(line)
|
||||
except OSError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
seen = load_seen()
|
||||
|
||||
|
||||
def mark_seen(p):
|
||||
seen.add(p)
|
||||
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def is_junk(v):
|
||||
if any(g in v for g in GARBAGE):
|
||||
return True
|
||||
if len(set(v)) < 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ── IA Solr ──────────────────────────────────────────────────────────────
|
||||
def ia_search(query, rows=MAX_ITEMS):
|
||||
params = {
|
||||
'q': query,
|
||||
'fl[]': 'identifier,title,description',
|
||||
'rows': min(rows, 200), 'page': 1, 'output': 'json',
|
||||
}
|
||||
for attempt in (2, 4, 8):
|
||||
try:
|
||||
r = S.get(
|
||||
'https://archive.org/advancedsearch.php',
|
||||
params=params, headers=UA, timeout=15,
|
||||
)
|
||||
if r.status_code == 200:
|
||||
return r.json().get('response', {}).get('docs', [])
|
||||
log("IA %d q=%s" % (r.status_code, query[:40]))
|
||||
if r.status_code == 429:
|
||||
time.sleep(min(int(r.headers.get("Retry-After", "30") or 30), 60))
|
||||
continue
|
||||
return []
|
||||
except requests.RequestException as e:
|
||||
log("IA err: %s" % e)
|
||||
time.sleep(attempt)
|
||||
return []
|
||||
|
||||
|
||||
def ia_process(doc):
|
||||
"""Scan Solr metadata for keys. No text file download (software/git)."""
|
||||
ident = doc.get('identifier', '').strip()
|
||||
if not ident or ident in seen:
|
||||
return 0
|
||||
mark_seen(ident)
|
||||
text = (doc.get('title') or '') + ' ' + (doc.get('description') or '')
|
||||
if not text.strip():
|
||||
return 0
|
||||
n = 0
|
||||
for prov, key in scan_text(text):
|
||||
if not is_junk(key):
|
||||
db_add(key, prov, "https://archive.org/details/" + ident)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def ia_cycle():
|
||||
global _ia_qidx
|
||||
query = IA_QUERIES[_ia_qidx % len(IA_QUERIES)]
|
||||
_ia_qidx += 1
|
||||
docs = ia_search(query)
|
||||
if not docs:
|
||||
log("IA q=%s: 0 docs" % query[:40])
|
||||
return 0
|
||||
scanned = 0
|
||||
found = 0
|
||||
for doc in docs:
|
||||
if scanned >= MAX_ITEMS:
|
||||
break
|
||||
scanned += 1
|
||||
try:
|
||||
found += ia_process(doc)
|
||||
except Exception as e:
|
||||
log("IA proc %s: %s" % (doc.get('identifier', '?'), e))
|
||||
log("IA q=%s: scanned=%d found=%d" % (query[:40], scanned, found))
|
||||
return found
|
||||
|
||||
|
||||
# ── OSF (fast, shallow) ─────────────────────────────────────────────────
|
||||
def osf_cycle():
|
||||
"""List 3 recent OSF nodes, download root-level text files."""
|
||||
try:
|
||||
r = S.get(
|
||||
'https://api.osf.io/v2/nodes/',
|
||||
params={'page[size]': 3, 'filter[public]': 'true',
|
||||
'sort': '-date_modified'},
|
||||
headers=UA, timeout=10,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
return 0
|
||||
nodes = r.json().get('data', [])
|
||||
except Exception as e:
|
||||
log("OSF list err: %s" % e)
|
||||
return 0
|
||||
|
||||
found = 0
|
||||
for node in nodes:
|
||||
nid = node['id']
|
||||
if nid in seen:
|
||||
continue
|
||||
mark_seen(nid)
|
||||
found += _osf_scan_node(nid)
|
||||
log("OSF: found=%d" % found)
|
||||
return found
|
||||
|
||||
|
||||
def _osf_scan_node(nid):
|
||||
"""Scan root-level files of one OSF node. One level deep only."""
|
||||
found = 0
|
||||
try:
|
||||
r = S.get(
|
||||
'https://api.osf.io/v2/nodes/%s/files/osfstorage/' % nid,
|
||||
params={'page[size]': 20}, headers=UA, timeout=5,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
return 0
|
||||
items = r.json().get('data', [])
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def _check(item):
|
||||
nonlocal found # folder branch accumulates into _osf_scan_node's scope
|
||||
# (without this, `found += _check(child)` raised UnboundLocalError —
|
||||
# silently eaten by the old except:pass = dead OSF-folder path)
|
||||
attr = item.get('attributes', {})
|
||||
kind = attr.get('kind', '')
|
||||
name = attr.get('name', '')
|
||||
if not name:
|
||||
return 0
|
||||
if kind == 'file':
|
||||
dl = item.get('links', {}).get('download')
|
||||
sz = attr.get('size', 0)
|
||||
if dl and 5 < sz < 100000:
|
||||
try:
|
||||
# §4 v7.14: stream + running cap — metadata sz can lie and
|
||||
# bare r2.content buffered arbitrary bodies (iascholar was
|
||||
# the last §4 offender: the hardening pass ran while the
|
||||
# lane was disabled, re-enable shipped it un-hardened).
|
||||
r2 = S.get(dl, headers=UA, timeout=15, stream=True,
|
||||
allow_redirects=True)
|
||||
try:
|
||||
if r2.status_code != 200:
|
||||
return 0
|
||||
cl = r2.headers.get("Content-Length")
|
||||
if cl and int(cl) > FILE_CAP:
|
||||
return 0
|
||||
chunks, got = [], 0
|
||||
for ch in r2.iter_content(65536):
|
||||
got += len(ch)
|
||||
if got > FILE_CAP:
|
||||
return 0
|
||||
chunks.append(ch)
|
||||
body = b"".join(chunks).decode("utf-8", "replace")
|
||||
finally:
|
||||
r2.close()
|
||||
for p, k in scan_text(body):
|
||||
if not is_junk(k):
|
||||
db_add(k, p, "osf:%s/%s" % (nid, name))
|
||||
return 1
|
||||
except Exception as e:
|
||||
log("osf file err: %s" % e) # voice: silent pass hid regressions
|
||||
elif kind == 'folder' and name != 'osfstorage':
|
||||
try:
|
||||
r2 = S.get(item['links']['info'],
|
||||
params={'page[size]': 20},
|
||||
headers=UA, timeout=5)
|
||||
if r2.status_code == 200:
|
||||
for child in r2.json().get('data', []):
|
||||
found += _check(child)
|
||||
except Exception as e:
|
||||
log("osf folder err: %s" % e)
|
||||
return 0
|
||||
|
||||
for item in items:
|
||||
found += _check(item)
|
||||
return found
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────
|
||||
def cycle():
|
||||
global total
|
||||
total = 0
|
||||
t0 = time.time()
|
||||
try:
|
||||
ia_cycle()
|
||||
except Exception as e:
|
||||
log("IA cycle err: %s" % e)
|
||||
try:
|
||||
osf_cycle()
|
||||
except Exception as e:
|
||||
log("OSF cycle err: %s" % e)
|
||||
log("cycle: +%d keys in %.1fs" % (total, time.time() - t0))
|
||||
|
||||
|
||||
def main():
|
||||
log("iascholar_eternal started")
|
||||
if "--once" in sys.argv:
|
||||
global MAX_ITEMS
|
||||
MAX_ITEMS = min(MAX_ITEMS, 10)
|
||||
cycle()
|
||||
log("--once done")
|
||||
return
|
||||
while True:
|
||||
cycle()
|
||||
time.sleep(SLEEP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user