304 lines
9.8 KiB
Python
304 lines
9.8 KiB
Python
"""ETERNAL IA-text + OSF: Solr metadata search + OSF file walk.
|
|
|
|
Primary: Internet Archive Solr API — targeted queries for items with key
|
|
patterns in metadata (description/title). Skips .bundle/software downloads.
|
|
Secondary: OSF public nodes — shallow file download (3 nodes, 1 level).
|
|
"""
|
|
import os as _os
|
|
_BASE = _os.path.dirname(_os.path.abspath(__file__))
|
|
_os.makedirs(_os.path.join(_BASE, "data"), exist_ok=True)
|
|
|
|
import hashlib, sqlite3, sys, threading, time
|
|
import requests
|
|
|
|
S = requests.Session()
|
|
S.trust_env = False # contract 4
|
|
|
|
DB = _os.environ.get("KF_DB") or _os.path.join(_BASE, "data", "keys.db")
|
|
LOG = _os.path.join(_BASE, "data", "iascholar_eternal.log")
|
|
SEEN_FILE = _os.path.join(_BASE, "data", "iascholar_seen.txt")
|
|
SLEEP = 180 # v7.5: smaller archive.org share (shared host with wayback)
|
|
MAX_ITEMS = 20
|
|
FILE_CAP = 500_000
|
|
|
|
from key_patterns import scan_text
|
|
|
|
GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
|
|
"placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
|
|
"abcdefghijklmnopqrst", "1234567890abcdef")
|
|
|
|
UA = {"User-Agent": "keyfarm-iascholar/1.0"}
|
|
|
|
# IA queries: target description field where keys appear in metadata.
|
|
# Full-text search catches keys in git bundles (unreadable) — description:
|
|
# prefix limits to the metadata we can actually extract keys from.
|
|
IA_QUERIES = [
|
|
'description:(sk-proj OR "ghp_" OR AKIA) AND mediatype:software',
|
|
'description:(xoxb OR sk_live OR pk_live) AND mediatype:software',
|
|
'description:(api_key OR secret) AND collection:github',
|
|
]
|
|
_ia_qidx = 0
|
|
|
|
|
|
def log(msg):
|
|
line = "[%s] %s\n" % (time.strftime('%Y-%m-%d %H:%M:%S'), msg)
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
|
|
|
|
conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
|
|
isolation_level=None)
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
|
|
plan TEXT, price TEXT, remaining TEXT,
|
|
found TEXT, repo TEXT)""")
|
|
conn.commit()
|
|
lock = threading.Lock()
|
|
total = 0
|
|
|
|
|
|
def db_add(key, prov, repo):
|
|
global total
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO keys (hash, val, prov, status, found, repo) "
|
|
"VALUES (?,?,?,?,?,?) "
|
|
"ON CONFLICT(hash) DO UPDATE SET status='NEW', "
|
|
"found=excluded.found, repo=excluded.repo "
|
|
"WHERE keys.status IN ('DEAD','ERR')",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16],
|
|
key, prov, "NEW", "ia-text", repo))
|
|
conn.commit()
|
|
with lock:
|
|
total += 1
|
|
except Exception as e:
|
|
log("db_add error: %s" % e)
|
|
|
|
|
|
def load_seen():
|
|
s = set()
|
|
try:
|
|
with open(SEEN_FILE, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
s.add(line)
|
|
except OSError:
|
|
pass
|
|
return s
|
|
|
|
|
|
seen = load_seen()
|
|
|
|
|
|
def mark_seen(p):
|
|
seen.add(p)
|
|
with open(SEEN_FILE, "a", encoding="utf-8") as f:
|
|
f.write(p + "\n")
|
|
|
|
|
|
def is_junk(v):
|
|
if any(g in v for g in GARBAGE):
|
|
return True
|
|
if len(set(v)) < 6:
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── IA Solr ──────────────────────────────────────────────────────────────
|
|
def ia_search(query, rows=MAX_ITEMS):
|
|
params = {
|
|
'q': query,
|
|
'fl[]': 'identifier,title,description',
|
|
'rows': min(rows, 200), 'page': 1, 'output': 'json',
|
|
}
|
|
for attempt in (2, 4, 8):
|
|
try:
|
|
r = S.get(
|
|
'https://archive.org/advancedsearch.php',
|
|
params=params, headers=UA, timeout=15,
|
|
)
|
|
if r.status_code == 200:
|
|
return r.json().get('response', {}).get('docs', [])
|
|
log("IA %d q=%s" % (r.status_code, query[:40]))
|
|
if r.status_code == 429:
|
|
time.sleep(min(int(r.headers.get("Retry-After", "30") or 30), 60))
|
|
continue
|
|
return []
|
|
except requests.RequestException as e:
|
|
log("IA err: %s" % e)
|
|
time.sleep(attempt)
|
|
return []
|
|
|
|
|
|
def ia_process(doc):
|
|
"""Scan Solr metadata for keys. No text file download (software/git)."""
|
|
ident = doc.get('identifier', '').strip()
|
|
if not ident or ident in seen:
|
|
return 0
|
|
mark_seen(ident)
|
|
text = (doc.get('title') or '') + ' ' + (doc.get('description') or '')
|
|
if not text.strip():
|
|
return 0
|
|
n = 0
|
|
for prov, key in scan_text(text):
|
|
if not is_junk(key):
|
|
db_add(key, prov, "https://archive.org/details/" + ident)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def ia_cycle():
|
|
global _ia_qidx
|
|
query = IA_QUERIES[_ia_qidx % len(IA_QUERIES)]
|
|
_ia_qidx += 1
|
|
docs = ia_search(query)
|
|
if not docs:
|
|
log("IA q=%s: 0 docs" % query[:40])
|
|
return 0
|
|
scanned = 0
|
|
found = 0
|
|
for doc in docs:
|
|
if scanned >= MAX_ITEMS:
|
|
break
|
|
scanned += 1
|
|
try:
|
|
found += ia_process(doc)
|
|
except Exception as e:
|
|
log("IA proc %s: %s" % (doc.get('identifier', '?'), e))
|
|
log("IA q=%s: scanned=%d found=%d" % (query[:40], scanned, found))
|
|
return found
|
|
|
|
|
|
# ── OSF (fast, shallow) ─────────────────────────────────────────────────
|
|
def osf_cycle():
|
|
"""List 3 recent OSF nodes, download root-level text files."""
|
|
try:
|
|
r = S.get(
|
|
'https://api.osf.io/v2/nodes/',
|
|
params={'page[size]': 3, 'filter[public]': 'true',
|
|
'sort': '-date_modified'},
|
|
headers=UA, timeout=10,
|
|
)
|
|
if r.status_code != 200:
|
|
return 0
|
|
nodes = r.json().get('data', [])
|
|
except Exception as e:
|
|
log("OSF list err: %s" % e)
|
|
return 0
|
|
|
|
found = 0
|
|
for node in nodes:
|
|
nid = node['id']
|
|
if nid in seen:
|
|
continue
|
|
mark_seen(nid)
|
|
found += _osf_scan_node(nid)
|
|
log("OSF: found=%d" % found)
|
|
return found
|
|
|
|
|
|
def _osf_scan_node(nid):
|
|
"""Scan root-level files of one OSF node. One level deep only."""
|
|
found = 0
|
|
try:
|
|
r = S.get(
|
|
'https://api.osf.io/v2/nodes/%s/files/osfstorage/' % nid,
|
|
params={'page[size]': 20}, headers=UA, timeout=5,
|
|
)
|
|
if r.status_code != 200:
|
|
return 0
|
|
items = r.json().get('data', [])
|
|
except Exception:
|
|
return 0
|
|
|
|
def _check(item):
|
|
nonlocal found # folder branch accumulates into _osf_scan_node's scope
|
|
# (without this, `found += _check(child)` raised UnboundLocalError —
|
|
# silently eaten by the old except:pass = dead OSF-folder path)
|
|
attr = item.get('attributes', {})
|
|
kind = attr.get('kind', '')
|
|
name = attr.get('name', '')
|
|
if not name:
|
|
return 0
|
|
if kind == 'file':
|
|
dl = item.get('links', {}).get('download')
|
|
sz = attr.get('size', 0)
|
|
if dl and 5 < sz < 100000:
|
|
try:
|
|
# §4 v7.14: stream + running cap — metadata sz can lie and
|
|
# bare r2.content buffered arbitrary bodies (iascholar was
|
|
# the last §4 offender: the hardening pass ran while the
|
|
# lane was disabled, re-enable shipped it un-hardened).
|
|
r2 = S.get(dl, headers=UA, timeout=15, stream=True,
|
|
allow_redirects=True)
|
|
try:
|
|
if r2.status_code != 200:
|
|
return 0
|
|
cl = r2.headers.get("Content-Length")
|
|
if cl and int(cl) > FILE_CAP:
|
|
return 0
|
|
chunks, got = [], 0
|
|
for ch in r2.iter_content(65536):
|
|
got += len(ch)
|
|
if got > FILE_CAP:
|
|
return 0
|
|
chunks.append(ch)
|
|
body = b"".join(chunks).decode("utf-8", "replace")
|
|
finally:
|
|
r2.close()
|
|
for p, k in scan_text(body):
|
|
if not is_junk(k):
|
|
db_add(k, p, "osf:%s/%s" % (nid, name))
|
|
return 1
|
|
except Exception as e:
|
|
log("osf file err: %s" % e) # voice: silent pass hid regressions
|
|
elif kind == 'folder' and name != 'osfstorage':
|
|
try:
|
|
r2 = S.get(item['links']['info'],
|
|
params={'page[size]': 20},
|
|
headers=UA, timeout=5)
|
|
if r2.status_code == 200:
|
|
for child in r2.json().get('data', []):
|
|
found += _check(child)
|
|
except Exception as e:
|
|
log("osf folder err: %s" % e)
|
|
return 0
|
|
|
|
for item in items:
|
|
found += _check(item)
|
|
return found
|
|
|
|
|
|
# ── Main ─────────────────────────────────────────────────────────────────
|
|
def cycle():
|
|
global total
|
|
total = 0
|
|
t0 = time.time()
|
|
try:
|
|
ia_cycle()
|
|
except Exception as e:
|
|
log("IA cycle err: %s" % e)
|
|
try:
|
|
osf_cycle()
|
|
except Exception as e:
|
|
log("OSF cycle err: %s" % e)
|
|
log("cycle: +%d keys in %.1fs" % (total, time.time() - t0))
|
|
|
|
|
|
def main():
|
|
log("iascholar_eternal started")
|
|
if "--once" in sys.argv:
|
|
global MAX_ITEMS
|
|
MAX_ITEMS = min(MAX_ITEMS, 10)
|
|
cycle()
|
|
log("--once done")
|
|
return
|
|
while True:
|
|
cycle()
|
|
time.sleep(SLEEP)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |