keyhunter-parser/baipiao_eternal.py

"""ETERNAL baipiao: BaiPiao.org free-API giveaway posts — community-shared
keys (DashScope/GLM/Claude/DeepSeek/SiliconFlow…) rendered server-side by
Astro, so a plain GET returns the keys inside HTML.

Endpoint(s): https://baipiao.org/free/api/            (index, probed 2026-10-06: 200, 301 KB)
             https://baipiao.org/free/api/<slug>/      (detail pages, keys in HTML)
Auth: none
"""
import os
BASE = os.path.dirname(os.path.abspath(__file__))
os.makedirs(os.path.join(BASE, "data"), exist_ok=True)

import hashlib
import re
import sqlite3
import sys
import threading
import time

import requests

DB = os.environ.get("KF_DB") or os.path.join(BASE, "data", "keys.db")
LOG = os.path.join(BASE, "data", "baipiao_eternal.log")
SEEN_FILE = os.path.join(BASE, "data", "baipiao_seen.txt")
INDEX_URL = "https://baipiao.org/free/api/"
N_THREADS = 3
SLEEP = 900               # giveaway site updates slowly
MAX_ITEMS = 15            # detail pages per cycle
PAGE_CAP = 2_000_000      # bytes per page (index is ~300 KB)
SEEN_CAP = 5_000

from key_patterns import scan_text

GARBAGE = ("xxxx", "your", "YOUR", "EXAMPLE", "example", "placeholder",
           "CHANGEME", "changeme", "dummy", "DUMMY", "abcdefghijklmnopqrst",
           "1234567890abcdef")
LINK_RE = re.compile(r'href="(/free/api/[a-z0-9][a-z0-9\-/]*/)"')

S = requests.Session()
S.trust_env = False       # LANE_CONTRACT §4
lock = threading.Lock()
seen = set()


def log(msg):
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n"
    try:
        with open(LOG, "a", encoding="utf-8", errors="replace") as f:
            f.write(line)
    except OSError:
        pass


def load_seen():
    global seen
    try:
        with open(SEEN_FILE, encoding="utf-8") as f:
            seen = {l.strip() for l in f if l.strip()}
    except OSError:
        seen = set()


def save_seen():
    items = list(seen)[-SEEN_CAP:]
    tmp = SEEN_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        f.write("\n".join(items))
    os.replace(tmp, SEEN_FILE)


conn = sqlite3.connect(DB, timeout=120, check_same_thread=False)
conn.execute("""CREATE TABLE IF NOT EXISTS keys (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    hash TEXT UNIQUE, val TEXT, prov TEXT, status TEXT DEFAULT 'NEW',
    plan TEXT, price TEXT, remaining TEXT, found TEXT, repo TEXT)""")
conn.commit()


def db_add(key, prov, repo):
    if any(g in key for g in GARBAGE) or len(set(key)) < 6:
        return 0
    h = hashlib.sha1(key.encode()).hexdigest()[:16]
    with lock:
        cur = conn.execute(
            "INSERT INTO keys (hash, val, prov, status, found, repo) "
            "VALUES (?,?,?,?,?,?) "
            "ON CONFLICT(hash) DO UPDATE SET status='NEW', "
            "found=excluded.found, repo=excluded.repo "
            "WHERE keys.status IN ('DEAD','ERR')",
            (h, key, prov, "NEW", "baipiao", repo[:300]))
        conn.commit()
        return cur.rowcount


def fetch(url):
    for attempt, wait in enumerate((2, 4, 8)):
        try:
            r = S.get(url, timeout=25, headers={
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) keyfarm/1.0"})
            if r.status_code == 200:
                return r.text[:PAGE_CAP]
            if r.status_code in (403, 429):
                ra = int(r.headers.get("Retry-After", "0") or 0)
                time.sleep(max(ra, wait))
                continue
            if r.status_code >= 500 and attempt < 2:
                time.sleep(wait)
                continue
            return None
        except requests.RequestException:
            if attempt < 2:
                time.sleep(wait)
    return None


def scan_page(html, url):
    n = 0
    for prov, key in scan_text(html or ""):
        if len(key) < 20:
            continue
        n += db_add(key, prov, url)
    return n


def work(url):
    with lock:
        if url in seen:
            return 0
        seen.add(url)
    html = fetch(url)
    if html is None:
        with lock:
            seen.discard(url)      # retry next cycle
        return 0
    return scan_page(html, url)


def cycle(max_items):
    index = fetch(INDEX_URL)
    if index is None:
        log("index fetch failed")
        return 0, 0
    links = []
    for m in LINK_RE.finditer(index):
        u = "https://baipiao.org" + m.group(1)
        if u not in seen and u not in links:
            links.append(u)
    fresh = links[:max_items]
    log(f"index: {len(links)} fresh links, fetching {len(fresh)}")
    found = 0
    if fresh:
        from concurrent.futures import ThreadPoolExecutor
        with ThreadPoolExecutor(max_workers=N_THREADS) as ex:
            for n in ex.map(work, fresh):
                found += n
    # re-scan the index itself (it embeds preview keys)
    found += scan_page(index, INDEX_URL)
    return len(fresh), found


def main():
    once = "--once" in sys.argv
    load_seen()
    log(f"=== baipiao START seen={len(seen)} once={once} ===")
    cycles = 0
    while True:
        try:
            budget = min(MAX_ITEMS, 5) if once else MAX_ITEMS
            pages, found = cycle(budget)
            cycles += 1
            log(f"cycle {cycles}: pages={pages} +{found} keys seen={len(seen)}")
            save_seen()
        except Exception as e:
            log(f"cycle error: {e!r}")
        if once:
            print(f"baipiao --once done: seen={len(seen)}")
            return
        time.sleep(SLEEP)


if __name__ == "__main__":
    main()