"""ETERNAL IA-text + OSF: Solr metadata search + OSF file walk.

Primary: Internet Archive Solr API — targeted queries for items with key
patterns in metadata (description/title).  Skips .bundle/software downloads.
Secondary: OSF public nodes — shallow file download (3 nodes, 1 level).
"""
import os as _os
_BASE = _os.path.dirname(_os.path.abspath(__file__))
_os.makedirs(_os.path.join(_BASE, "data"), exist_ok=True)

import hashlib, sqlite3, sys, threading, time
import requests

S = requests.Session()
S.trust_env = False  # contract 4

DB = _os.environ.get("KF_DB") or _os.path.join(_BASE, "data", "keys.db")
LOG = _os.path.join(_BASE, "data", "iascholar_eternal.log")
SEEN_FILE = _os.path.join(_BASE, "data", "iascholar_seen.txt")
SLEEP = 180   # v7.5: smaller archive.org share (shared host with wayback)
MAX_ITEMS = 20
FILE_CAP = 500_000

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

UA = {"User-Agent": "keyfarm-iascholar/1.0"}

# IA queries: target description field where keys appear in metadata.
# Full-text search catches keys in git bundles (unreadable) — description:
# prefix limits to the metadata we can actually extract keys from.
IA_QUERIES = [
    'description:(sk-proj OR "ghp_" OR AKIA) AND mediatype:software',
    'description:(xoxb OR sk_live OR pk_live) AND mediatype:software',
    'description:(api_key OR secret) AND collection:github',
]
_ia_qidx = 0


def log(msg):
    line = "[%s] %s\n" % (time.strftime('%Y-%m-%d %H:%M:%S'), msg)
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT "NEW",
  plan TEXT, price TEXT, remaining TEXT,
  found TEXT, repo TEXT)""")
conn.commit()
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    try:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "ia-text", repo))
        conn.commit()
        with lock:
            total += 1
    except Exception as e:
        log("db_add error: %s" % e)


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except OSError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


# ── IA Solr ──────────────────────────────────────────────────────────────
def ia_search(query, rows=MAX_ITEMS):
    params = {
        'q': query,
        'fl[]': 'identifier,title,description',
        'rows': min(rows, 200), 'page': 1, 'output': 'json',
    }
    for attempt in (2, 4, 8):
        try:
            r = S.get(
                'https://archive.org/advancedsearch.php',
                params=params, headers=UA, timeout=15,
            )
            if r.status_code == 200:
                return r.json().get('response', {}).get('docs', [])
            log("IA %d q=%s" % (r.status_code, query[:40]))
            if r.status_code == 429:
                time.sleep(min(int(r.headers.get("Retry-After", "30") or 30), 60))
                continue
            return []
        except requests.RequestException as e:
            log("IA err: %s" % e)
            time.sleep(attempt)
    return []


def ia_process(doc):
    """Scan Solr metadata for keys. No text file download (software/git)."""
    ident = doc.get('identifier', '').strip()
    if not ident or ident in seen:
        return 0
    mark_seen(ident)
    text = (doc.get('title') or '') + ' ' + (doc.get('description') or '')
    if not text.strip():
        return 0
    n = 0
    for prov, key in scan_text(text):
        if not is_junk(key):
            db_add(key, prov, "https://archive.org/details/" + ident)
            n += 1
    return n


def ia_cycle():
    global _ia_qidx
    query = IA_QUERIES[_ia_qidx % len(IA_QUERIES)]
    _ia_qidx += 1
    docs = ia_search(query)
    if not docs:
        log("IA q=%s: 0 docs" % query[:40])
        return 0
    scanned = 0
    found = 0
    for doc in docs:
        if scanned >= MAX_ITEMS:
            break
        scanned += 1
        try:
            found += ia_process(doc)
        except Exception as e:
            log("IA proc %s: %s" % (doc.get('identifier', '?'), e))
    log("IA q=%s: scanned=%d found=%d" % (query[:40], scanned, found))
    return found


# ── OSF (fast, shallow) ─────────────────────────────────────────────────
def osf_cycle():
    """List 3 recent OSF nodes, download root-level text files."""
    try:
        r = S.get(
            'https://api.osf.io/v2/nodes/',
            params={'page[size]': 3, 'filter[public]': 'true',
                    'sort': '-date_modified'},
            headers=UA, timeout=10,
        )
        if r.status_code != 200:
            return 0
        nodes = r.json().get('data', [])
    except Exception as e:
        log("OSF list err: %s" % e)
        return 0

    found = 0
    for node in nodes:
        nid = node['id']
        if nid in seen:
            continue
        mark_seen(nid)
        found += _osf_scan_node(nid)
    log("OSF: found=%d" % found)
    return found


def _osf_scan_node(nid):
    """Scan root-level files of one OSF node. One level deep only."""
    found = 0
    try:
        r = S.get(
            'https://api.osf.io/v2/nodes/%s/files/osfstorage/' % nid,
            params={'page[size]': 20}, headers=UA, timeout=5,
        )
        if r.status_code != 200:
            return 0
        items = r.json().get('data', [])
    except Exception:
        return 0

    def _check(item):
        nonlocal found  # folder branch accumulates into _osf_scan_node's scope
        # (without this, `found += _check(child)` raised UnboundLocalError —
        # silently eaten by the old except:pass = dead OSF-folder path)
        attr = item.get('attributes', {})
        kind = attr.get('kind', '')
        name = attr.get('name', '')
        if not name:
            return 0
        if kind == 'file':
            dl = item.get('links', {}).get('download')
            sz = attr.get('size', 0)
            if dl and 5 < sz < 100000:
                try:
                    # §4 v7.14: stream + running cap — metadata sz can lie and
                    # bare r2.content buffered arbitrary bodies (iascholar was
                    # the last §4 offender: the hardening pass ran while the
                    # lane was disabled, re-enable shipped it un-hardened).
                    r2 = S.get(dl, headers=UA, timeout=15, stream=True,
                               allow_redirects=True)
                    try:
                        if r2.status_code != 200:
                            return 0
                        cl = r2.headers.get("Content-Length")
                        if cl and int(cl) > FILE_CAP:
                            return 0
                        chunks, got = [], 0
                        for ch in r2.iter_content(65536):
                            got += len(ch)
                            if got > FILE_CAP:
                                return 0
                            chunks.append(ch)
                        body = b"".join(chunks).decode("utf-8", "replace")
                    finally:
                        r2.close()
                    for p, k in scan_text(body):
                        if not is_junk(k):
                            db_add(k, p, "osf:%s/%s" % (nid, name))
                            return 1
                except Exception as e:
                    log("osf file err: %s" % e)  # voice: silent pass hid regressions
        elif kind == 'folder' and name != 'osfstorage':
            try:
                r2 = S.get(item['links']['info'],
                                  params={'page[size]': 20},
                                  headers=UA, timeout=5)
                if r2.status_code == 200:
                    for child in r2.json().get('data', []):
                        found += _check(child)
            except Exception as e:
                log("osf folder err: %s" % e)
        return 0

    for item in items:
        found += _check(item)
    return found


# ── Main ─────────────────────────────────────────────────────────────────
def cycle():
    global total
    total = 0
    t0 = time.time()
    try:
        ia_cycle()
    except Exception as e:
        log("IA cycle err: %s" % e)
    try:
        osf_cycle()
    except Exception as e:
        log("OSF cycle err: %s" % e)
    log("cycle: +%d keys in %.1fs" % (total, time.time() - t0))


def main():
    log("iascholar_eternal started")
    if "--once" in sys.argv:
        global MAX_ITEMS
        MAX_ITEMS = min(MAX_ITEMS, 10)
        cycle()
        log("--once done")
        return
    while True:
        cycle()
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()