"""GitLab web search scraper — no API key needed, uses web UI search."""
import requests, re, sqlite3, hashlib, time
from bs4 import BeautifulSoup

DB = r'C:\Users\User\Desktop\keyhunter\data\keys.db'
PATTERN = re.compile(r'sk-[0-9a-f]{32}', re.IGNORECASE)

s = requests.Session()
s.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml',
    'Accept-Language': 'en-US,en;q=0.9',
})

queries = [
    'DASHSCOPE_API_KEY sk-',
    'sk- qwen-max dashscope',
    'sk- dashscope api_key',
]

new_keys = set()

for q in queries:
    print(f'\n=== GitLab web search: "{q}" ===')
    
    for page in range(1, 11):
        try:
            url = f'https://gitlab.com/search?search={q}&nav_source=navbar&page={page}'
            resp = s.get(url, timeout=15)
            time.sleep(2)
            
            if resp.status_code != 200:
                print(f'  page {page}: HTTP {resp.status_code}')
                break
            
            soup = BeautifulSoup(resp.text, 'html.parser')
            
            # Try to find code snippets in search results
            code_blocks = soup.find_all(['code', 'pre'])
            for block in code_blocks:
                text = block.get_text()
                keys = PATTERN.findall(text)
                for k in keys:
                    if k not in new_keys:
                        new_keys.add(k)
                        print(f'  FOUND: {k}')
            
            # Also check whole page text for inline keys
            keys = PATTERN.findall(resp.text[:50000])
            for k in keys:
                if k not in new_keys:
                    new_keys.add(k)
                    print(f'  FOUND (inline): {k}')
            
            print(f'  page {page}: done ({len(new_keys)} total new)')
            
            # Check if there are more pages
            if 'next page' not in resp.text.lower() and 'data-page=' not in resp.text:
                break
                
        except Exception as e:
            print(f'  page {page}: ERR {e}')
            break

print(f'\n=== Total new from GitLab: {len(new_keys)} ===')

if new_keys:
    conn = sqlite3.connect(DB)
    c = conn.cursor()
    added = 0
    for k in new_keys:
        h = hashlib.sha256(k.encode()).hexdigest()
        exists = c.execute('SELECT COUNT(*) FROM keys WHERE hash=?', (h,)).fetchone()[0]
        if not exists:
            try:
                c.execute(
                    'INSERT INTO keys (hash, val, prov, status, found, first_seen, last_seen) VALUES (?,?,?,?,?,?,?)',
                    (h, k, 'gitlab_web', 'NEW', 1, '2026-10-19T19:25:00', '2026-10-19T19:25:00')
                )
                added += 1
            except Exception as e:
                print(f'  DB insert error: {e}')
    conn.commit()
    conn.close()
    print(f'Added {added} to DB')
    for k in list(new_keys)[:20]:
        print(f'  {k}')