"""Searchcode.com scraper — tries both API and web UI for DashScope keys."""
import requests, re, sqlite3, hashlib, time

DB = r'C:\Users\User\Desktop\keyhunter\data\keys.db'
PATTERN = re.compile(r'sk-[0-9a-f]{32}', re.IGNORECASE)
new_keys = set()

s = requests.Session()
s.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
})

# Approach 1: web UI search
queries = [
    'sk- dashscope',
    'DASHSCOPE_API_KEY sk-',
    'sk- qwen',
]

for q in queries:
    print(f'\n=== searchcode: "{q}" ===')
    for page in range(0, 5):
        try:
            url = f'https://searchcode.com/?q={q}&p={page}'
            resp = s.get(url, timeout=15, allow_redirects=True)
            time.sleep(2)
            
            if resp.status_code != 200:
                print(f'  page {page}: HTTP {resp.status_code}')
                break
            
            text = resp.text
            keys = PATTERN.findall(text[:50000])
            for k in keys:
                if k not in new_keys:
                    new_keys.add(k)
                    print(f'  FOUND: {k}')
            
            # Extract repo/file info
            import bs4
            soup = bs4.BeautifulSoup(text, 'html.parser')
            results = soup.find_all('div', class_='result')
            for r in results:
                code = r.find('code')
                if code:
                    subkeys = PATTERN.findall(code.get_text())
                    for k in subkeys:
                        if k not in new_keys:
                            new_keys.add(k)
                            print(f'  FOUND (code): {k}')
            
            print(f'  page {page}: done, total={len(new_keys)}')
            
            # Check if more pages
            if 'next page' not in text.lower() and 'p=' not in text:
                break
        except Exception as e:
            print(f'  page {page}: ERR {str(e)[:80]}')
            break

print(f'\n=== Total new from searchcode: {len(new_keys)} ===')

if new_keys:
    conn = sqlite3.connect(DB)
    c = conn.cursor()
    added = 0
    for k in new_keys:
        h = hashlib.sha256(k.encode()).hexdigest()
        exists = c.execute('SELECT COUNT(*) FROM keys WHERE hash=?', (h,)).fetchone()[0]
        if not exists:
            try:
                c.execute(
                    'INSERT INTO keys (hash, val, prov, status, found, first_seen, last_seen) VALUES (?,?,?,?,?,?,?)',
                    (h, k, 'searchcode', 'NEW', 1, '2026-10-19T19:30:00', '2026-10-19T19:30:00')
                )
                added += 1
            except:
                pass
    conn.commit()
    conn.close()
    print(f'Added {added} to DB')