"""ETERNAL SourceForge: project discovery + file tree walk + text/archive scan.

Discovery: allura_sitemap (13 шардов × ~10k /p/-слагов, lastmod ежедневно, без
Cloudflare). /directory/?q= за Turnstile-challenge — мёртв для unattended loop.
Курсор шарда: data/sourceforge_sitemap_cursor.txt; пул: data/sourceforge_projects.txt.
Liveness: GET https://sourceforge.net/rest/p/<short>/ (trailing slash обязателен;
status "moved"/404 → skip). Мертвы: /api/*, /rest/search/, /rest/p/ (listing).
Files: https://sourceforge.net/projects/<p>/files/ (HTML parse of files_list table)
       https://sourceforge.net/projects/<p>/files/<path>/download (mirror redirect)
Auth: none
"""
import os as _os
BASE = _os.path.dirname(_os.path.abspath(__file__))
_os.makedirs(_os.path.join(BASE, "data"), exist_ok=True)


import hashlib
import io
import re
import sqlite3
import sys
import tarfile
import threading
import time
import zipfile

import requests

DB = _os.environ.get("KF_DB") or _os.path.join(BASE, "data", "keys.db")
LOG = BASE + r"/data/sourceforge_eternal.log"
CURSOR_FILE = BASE + r"/data/sourceforge_cursor.txt"
SEEN_FILE = BASE + r"/data/sourceforge_seen.txt"
PROJECTS_FILE = BASE + r"/data/sourceforge_projects.txt"

N_THREADS = 4
PROJECTS_PER_CYCLE = 5          # 5 projects per cycle → ~80 projects in 16 cycles (~24h)
MAX_DEPTH = 3                   # max subfolder depth to recurse
MAX_FOLDERS_PER_DEPTH = 10      # max folder listings per depth level (rate-limit guard)
MAX_ITEMS = 200                 # per cycle cap; --once uses min(MAX_ITEMS, 5)
FILE_CAP = 500_000              # max single text file
ARCHIVE_CAP = 5 * 1024 * 1024  # max source archive to unpack
SLEEP = 90

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "n0tr3al",
           "placeholder", "CHANGEME", "changeme", "dummy", "DUMMY",
           "abcdefghijklmnopqrst", "1234567890abcdef")

TEXT_EXT = (".txt", ".md", ".rst", ".cfg", ".ini", ".conf", ".env",
            ".xml", ".json", ".yaml", ".yml", ".properties",
            ".py", ".sh", ".bat", ".ps1", ".php", ".pl", ".rb",
            ".js", ".ts", ".java", ".c", ".h", ".cpp", ".hpp",
            ".sql", ".html", ".css", ".dockerfile", ".toml")

UA = {"User-Agent": "Wget/1.21.3"}

# Curated seed — popular open-source projects likely to bundle text/config files
SEED_PROJECTS = [
    # IDEs / Editors
    "notepadplusplus", "codeblocks", "geany", "dev-cpp",
    # Web / CMS / Database
    "xampp", "wampserver", "wordpress", "joomla", "drupal",
    "moodle", "phpmyadmin", "adminer", "mysqlcc",
    "postgresql", "sqlitebrowser", "heidisql",
    # Networking / Security
    "wireshark", "nmap", "putty", "openssh", "openvpn",
    "torproject", "truecrypt", "veracrypt", "gpg4win",
    "keepass", "password-safe",
    # File transfer / FTP
    "filezilla", "winscp", "smartftp", "cyberduck",
    # Multimedia
    "audacity", "vlc", "ffmpeg", "handbrake", "shotcut",
    "obsproject", "blender", "gimp", "inkscape",
    # Development tools
    "gitforwindows", "tortoisegit", "tortoisesvn", "mercurial",
    "cmake", "mingw", "cygwin", "nsis", "innosetup",
    "python", "rubyinstaller", "perl",
    # Utilities
    "sevenzip", "peazip", "ventoy", "rufus",
    "crystaldiskinfo", "hwmonitor", "speedfan",
    "sysinternals", "processhacker",
    # System
    "clamwin", "spybot", "ccleaner", "defraggler",
    "wiztree", "everything",
    # Emulators / Games
    "dosbox", "pcsx2", "dolphin-emu", "mame",
    "desmume", "visualboyadvance",
    # Science / Math
    "octave", "scilab", "geogebra",
    # Office
    "libreoffice", "openoffice", "abiword",
    "sumatrapdf", "foxitreader",
    # Chat / Comms
    "pidgin", "irssi", "hexchat", "miranda-ng",
    "thunderbird",
    # Other
    "scrcpy.mirror", "portableapps", "chocolate-doom",
    "golang", "rustdesk",
]

# Filter seed to unique non-empty strings
SEED_PROJECTS = sorted(set(s for s in SEED_PROJECTS if s and s.strip()))

# Discovery: allura_sitemap — полный живой список проектов SF (без Cloudflare)
SITEMAP_INDEX = "https://sourceforge.net/allura_sitemap/sitemap.xml"
_SHARD_RE = re.compile(
    r'<loc>(https://sourceforge\.net/allura_sitemap/sitemap-\d+\.xml)</loc>')
_P_SLUG_RE = re.compile(r'sourceforge\.net/p/([a-z0-9][a-z0-9._-]{0,48})/')
_PROJ_SLUG_RE = re.compile(
    r'sourceforge\.net/projects/([a-z0-9][a-z0-9._-]{0,48})/')
SITEMAP_CURSOR_FILE = BASE + r"/data/sourceforge_sitemap_cursor.txt"

# Session with trust_env=False to bypass local dashscope proxy
S = requests.Session()
S.trust_env = False


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False,
                       isolation_level=None)
lock = threading.Lock()
total = 0


def db_add(key, prov, repo):
    global total
    with lock:
        conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (hashlib.sha1(key.encode()).hexdigest()[:16],
             key, prov, "NEW", "sourceforge", repo))
        total += 1


def load_cursor():
    try:
        with open(CURSOR_FILE, encoding="utf-8") as f:
            return int(f.read().strip() or 0)
    except (FileNotFoundError, ValueError):
        return 0


def save_cursor(c):
    with open(CURSOR_FILE, "w", encoding="utf-8") as f:
        f.write(str(c))


def load_seen():
    s = set()
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    s.add(line)
    except FileNotFoundError:
        pass
    return s


seen = load_seen()


def mark_seen(p):
    seen.add(p)
    with open(SEEN_FILE, "a", encoding="utf-8") as f:
        f.write(p + "\n")


def is_junk(v):
    if any(g in v for g in GARBAGE):
        return True
    if len(set(v)) < 6:
        return True
    return False


def sf_get(url, timeout=20):
    """Thin wrapper with retry/backoff and 403 handling.

    Uses module session S (trust_env=False) to bypass local proxy.
    """
    for attempt in (1, 2, 3):
        try:
            r = S.get(url, headers=UA, timeout=timeout)
            if r.status_code in (429, 503):
                ra = int(r.headers.get("Retry-After", 10 * attempt))
                time.sleep(ra)
                continue
            if r.status_code == 403:
                # Rate-limited; brief wait then retry
                time.sleep(5 * attempt)
                continue
            if r.status_code == 404:
                return None
            if r.status_code != 200:
                raise RuntimeError(f"HTTP {r.status_code}")
            return r
        except (requests.ConnectionError, requests.Timeout, RuntimeError):
            if attempt == 3:
                raise
            time.sleep(5 * attempt)
    return None


def list_dir(project, folder=""):
    """Parse files_list table from project's files page.

    Yields (is_folder, name, display_name).
    is_folder=True: name is relative URL folder path
    is_folder=False: name is full relative file path (for /download)
    """
    if folder:
        url = f"https://sourceforge.net/projects/{project}/files/{folder}/"
    else:
        url = f"https://sourceforge.net/projects/{project}/files/"
    try:
        r = sf_get(url, timeout=20)
    except Exception:
        return
    if r is None:
        return
    html = r.text
    # Find tbody
    idx = html.find('<table id="files_list"')
    if idx == -1:
        idx = html.find("<table id='files_list'")
    if idx == -1:
        return
    tbody_start = html.find("<tbody>", idx)
    tbody_end = html.find("</tbody>", tbody_start)
    if tbody_start == -1 or tbody_end == -1:
        return
    tbody = html[tbody_start:tbody_end + 8]

    # Folders — match both relative and absolute href, attributes may span lines
    esc = re.escape(project)
    pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/"[^>]*title="Click to enter'
    for m in re.finditer(pattern, tbody, re.DOTALL):
        name = m.group(1)
        if not name or name.endswith("/stats/timeline"):
            continue
        yield True, name, ""

    # Files
    pattern = r'<a\s+href="(?:https://sourceforge\.net)?/projects/' + esc + r'/files/([^"]+)/download"[^>]*title="Click to download ([^"]+)"'
    for m in re.finditer(pattern, tbody, re.DOTALL):
        yield False, m.group(1), m.group(2)


def get_mirror_url(download_page_url):
    """Fetch the download landing page and extract direct mirror URL."""
    try:
        r = S.get(download_page_url, headers=UA, timeout=20, allow_redirects=False)
    except Exception:
        return None
    if r is None:
        return None
    # Follow redirect chain to get the final mirror
    location = r.headers.get("Location", "")
    if location:
        return location
    return None


def _read_capped(r, cap):
    """Stream-read r up to cap bytes; None if oversized. Closes r. Caps BEFORE
    the full body lands in RAM (Content-Length pre-check + stream-and-abort)."""
    cl = r.headers.get("Content-Length")
    if cl and cl.isdigit() and int(cl) > cap:
        r.close()
        return None
    chunks = []
    size = 0
    try:
        for chunk in r.iter_content(65536):
            size += len(chunk)
            if size > cap:
                return None
            chunks.append(chunk)
    finally:
        r.close()
    return b"".join(chunks)


def download_file(download_page_url, cap):
    """Download file from SourceForge via redirect chain, capped at `cap` bytes.

    The /download URL redirects (302) through mirrors to the actual file.
    Returns (content, final_url) or (None, None) on failure/oversize.
    """
    for attempt in (1, 2, 3):
        try:
            r = S.get(download_page_url, headers=UA, timeout=60,
                      allow_redirects=True, stream=True)
            if r.status_code == 200:
                data = _read_capped(r, cap)
                return (data, r.url) if data else (None, None)
            if r.status_code in (403, 404):
                r.close()
                return None, None
            if r.status_code == 302:
                loc = r.headers.get("Location", "")
                r.close()
                if loc:
                    r2 = S.get(loc, headers=UA, timeout=60, stream=True)
                    if r2.status_code == 200:
                        data = _read_capped(r2, cap)
                        return (data, loc) if data else (None, None)
                    r2.close()
                    return None, None
            r.close()
            raise RuntimeError(f"HTTP {r.status_code}")
        except (requests.ConnectionError, requests.Timeout, RuntimeError):
            if attempt == 3:
                return None, None
            time.sleep(5 * attempt)
    return None, None


def scan_content(text, repo_label):
    """Run pattern scan + junk filter, db_add hits."""
    n = 0
    for prov, key in scan_text(text):
        if isinstance(key, tuple):
            key = key[0]
        if is_junk(key) or len(key) < 30:
            continue
        db_add(key, prov, repo_label)
        n += 1
    return n


def scan_text_file(content, file_path, project):
    """Scan text content from a downloaded file."""
    return scan_content(content, f"sf:{project}:{file_path}")


def scan_archive(content, archive_path, project):
    """Extract and scan text files from a zip/tar archive."""
    n = 0
    repo_label = f"sf:{project}:{archive_path}"
    # Try zip
    try:
        zf = zipfile.ZipFile(io.BytesIO(content))
        for m in zf.infolist():
            if m.is_dir() or m.file_size > FILE_CAP:
                continue
            fn = m.filename.lower()
            if not any(fn.endswith(e) for e in TEXT_EXT):
                continue
            try:
                data = zf.read(m).decode("utf-8", errors="ignore")
            except Exception:
                continue
            n += scan_content(data, repo_label + "/" + m.filename)
        return n
    except (zipfile.BadZipFile, Exception):
        pass
    # Try tar.gz / tar.bz2
    try:
        tf = tarfile.open(fileobj=io.BytesIO(content), mode="r:*")
        for m in tf.getmembers():
            if m.isfile() and m.size <= FILE_CAP:
                fn = m.name.lower()
                if not any(fn.endswith(e) for e in TEXT_EXT):
                    continue
                try:
                    fobj = tf.extractfile(m)
                    if fobj is None:
                        continue
                    data = fobj.read().decode("utf-8", errors="ignore")
                except Exception:
                    continue
                n += scan_content(data, repo_label + "/" + m.name)
    except Exception:
        pass
    return n


def walk_project(project):
    """Walk a project's file tree and collect candidate files.

    Returns list of (is_archive, path_info) where:
      is_archive=True:  path_info = (archive_url, archive_ext, archive_path)
      is_archive=False: path_info = (text_url, file_path)

    Capped by MAX_FOLDERS_PER_DEPTH per level.
    """
    candidates = []
    stack = [""]  # folder paths relative to /files/
    depth = 0
    while stack and depth <= MAX_DEPTH:
        next_stack = []
        for folder in stack[:MAX_FOLDERS_PER_DEPTH]:
            for is_folder, name, _ in list_dir(project, folder):
                if is_folder:
                    if "/stats/" in name:
                        continue
                    next_stack.append(name)
                else:
                    ext = _os.path.splitext(name)[1].lower()
                    if ext in TEXT_EXT:
                        candidates.append((False, (name, name)))
                    elif ext in (".zip", ".tar.gz", ".tar.bz2", ".tgz", ".tar",
                                 ".gz", ".bz2", ".xz"):
                        candidates.append((True, (name, ext, name)))
        stack = next_stack
        depth += 1
    return candidates[:MAX_ITEMS]


def work(project):
    """Walk project, download candidates, scan. Returns (ok_keys, fail_count)."""
    if not project_alive(project):
        log(f"skip {project}: dead/moved")
        return 0, 0
    ok = 0
    fail = 0
    candidates = walk_project(project)
    for is_archive, info in candidates:
        if is_archive:
            path, ext, archive_path = info
        else:
            path, file_path = info

        sid = f"{project}:{path}"
        if sid in seen:
            continue

        # Download via redirect chain
        dl_page = f"https://sourceforge.net/projects/{project}/files/{path}/download"
        content, final_url = download_file(
            dl_page, ARCHIVE_CAP if is_archive else FILE_CAP)
        if content is None:
            fail += 1
            continue

        if is_archive:
            if len(content) > ARCHIVE_CAP:
                continue
            n = scan_archive(content, archive_path, project)
        else:
            if len(content) > FILE_CAP:
                continue
            text = content.decode("utf-8", errors="ignore")
            n = scan_text_file(text, path, project)

        if n >= 0:
            ok += n
            mark_seen(sid)

    return ok, fail


def load_sitemap_cursor():
    try:
        with open(SITEMAP_CURSOR_FILE, encoding="utf-8") as f:
            return int(f.read().strip() or 0)
    except (FileNotFoundError, ValueError):
        return 0


def save_sitemap_cursor(v):
    tmp = SITEMAP_CURSOR_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        f.write(str(v))
    _os.replace(tmp, SITEMAP_CURSOR_FILE)


def discover_projects():
    """Один шард allura_sitemap за вызов → слаг-пул (ротация по курсору).

    Курсор.advance только при успешном фетче шарда; ошибка → повтор того же.
    """
    try:
        r = sf_get(SITEMAP_INDEX, timeout=30)
        if r is None:
            return []
        shards = _SHARD_RE.findall(r.text)
        if not shards:
            return []
        ci = load_sitemap_cursor() % len(shards)
        r2 = sf_get(shards[ci], timeout=90)
        if r2 is None:
            return []
    except Exception:
        return []
    save_sitemap_cursor(ci + 1)
    out, seen_l = [], set()
    for rx in (_P_SLUG_RE, _PROJ_SLUG_RE):
        for s in rx.findall(r2.text):
            if s not in seen_l:
                seen_l.add(s)
                out.append(s)
    return out


def load_pool():
    """Seed + discovered pool, deduped, seed first."""
    pool = list(SEED_PROJECTS)
    known = set(pool)
    try:
        with open(PROJECTS_FILE, encoding="utf-8") as f:
            for line in f:
                s = line.strip()
                if s and s not in known:
                    known.add(s)
                    pool.append(s)
    except FileNotFoundError:
        pass
    return pool


def append_pool(new_slugs):
    """Append unseen slugs to PROJECTS_FILE. Returns count added."""
    if not new_slugs:
        return 0
    known = set(load_pool())
    fresh = [s for s in new_slugs if s not in known]
    if fresh:
        with open(PROJECTS_FILE, "a", encoding="utf-8") as f:
            f.write("\n".join(fresh) + "\n")
    return len(fresh)


def project_alive(project):
    """REST liveness: /rest/p/<short>/ (404 или status=moved → False)."""
    try:
        r = sf_get(f"https://sourceforge.net/rest/p/{project}/", timeout=15)
    except Exception:
        return True  # сеть/403 — даём шанс walk'у
    if r is None:
        return False
    try:
        j = r.json()
    except ValueError:
        return True
    return j.get("status") != "moved"


def cycle():
    global total
    total = 0
    idx = load_cursor()
    n_new = append_pool(discover_projects())
    if n_new:
        log(f"discovery sitemap shard: +{n_new} new projects")
    projects = load_pool()
    items = []
    for _ in range(PROJECTS_PER_CYCLE):
        p = projects[idx % len(projects)]
        idx += 1
        items.append(p)
    save_cursor(idx)

    ok = fail = 0
    from concurrent.futures import ThreadPoolExecutor, as_completed
    with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
        futs = {ex.submit(work, p): p for p in items}
        for fut in as_completed(futs):
            p = futs[fut]
            try:
                res_ok, res_fail = fut.result()
            except Exception:
                res_ok, res_fail = 0, 1
            ok += res_ok
            fail += res_fail

    log(f"cycle: idx={idx} projects={len(items)} ok={ok} fail={fail} +{total} keys")


def main():
    log("sourceforge_eternal started")
    if "--once" in sys.argv:
        global MAX_ITEMS, PROJECTS_PER_CYCLE
        MAX_ITEMS = min(MAX_ITEMS, 5)
        PROJECTS_PER_CYCLE = 1
        cycle()
        return
    while True:
        try:
            cycle()
        except Exception as e:
            log(f"cycle-err {type(e).__name__}: {e}")
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()