87 lines
3.0 KiB
Python
87 lines
3.0 KiB
Python
"""GitLab web search scraper — no API key needed, uses web UI search."""
|
|
import requests, re, sqlite3, hashlib, time
|
|
from bs4 import BeautifulSoup
|
|
|
|
DB = r'C:\Users\User\Desktop\keyhunter\data\keys.db'
|
|
PATTERN = re.compile(r'sk-[0-9a-f]{32}', re.IGNORECASE)
|
|
|
|
s = requests.Session()
|
|
s.headers.update({
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
})
|
|
|
|
queries = [
|
|
'DASHSCOPE_API_KEY sk-',
|
|
'sk- qwen-max dashscope',
|
|
'sk- dashscope api_key',
|
|
]
|
|
|
|
new_keys = set()
|
|
|
|
for q in queries:
|
|
print(f'\n=== GitLab web search: "{q}" ===')
|
|
|
|
for page in range(1, 11):
|
|
try:
|
|
url = f'https://gitlab.com/search?search={q}&nav_source=navbar&page={page}'
|
|
resp = s.get(url, timeout=15)
|
|
time.sleep(2)
|
|
|
|
if resp.status_code != 200:
|
|
print(f' page {page}: HTTP {resp.status_code}')
|
|
break
|
|
|
|
soup = BeautifulSoup(resp.text, 'html.parser')
|
|
|
|
# Try to find code snippets in search results
|
|
code_blocks = soup.find_all(['code', 'pre'])
|
|
for block in code_blocks:
|
|
text = block.get_text()
|
|
keys = PATTERN.findall(text)
|
|
for k in keys:
|
|
if k not in new_keys:
|
|
new_keys.add(k)
|
|
print(f' FOUND: {k}')
|
|
|
|
# Also check whole page text for inline keys
|
|
keys = PATTERN.findall(resp.text[:50000])
|
|
for k in keys:
|
|
if k not in new_keys:
|
|
new_keys.add(k)
|
|
print(f' FOUND (inline): {k}')
|
|
|
|
print(f' page {page}: done ({len(new_keys)} total new)')
|
|
|
|
# Check if there are more pages
|
|
if 'next page' not in resp.text.lower() and 'data-page=' not in resp.text:
|
|
break
|
|
|
|
except Exception as e:
|
|
print(f' page {page}: ERR {e}')
|
|
break
|
|
|
|
print(f'\n=== Total new from GitLab: {len(new_keys)} ===')
|
|
|
|
if new_keys:
|
|
conn = sqlite3.connect(DB)
|
|
c = conn.cursor()
|
|
added = 0
|
|
for k in new_keys:
|
|
h = hashlib.sha256(k.encode()).hexdigest()
|
|
exists = c.execute('SELECT COUNT(*) FROM keys WHERE hash=?', (h,)).fetchone()[0]
|
|
if not exists:
|
|
try:
|
|
c.execute(
|
|
'INSERT INTO keys (hash, val, prov, status, found, first_seen, last_seen) VALUES (?,?,?,?,?,?,?)',
|
|
(h, k, 'gitlab_web', 'NEW', 1, '2026-10-19T19:25:00', '2026-10-19T19:25:00')
|
|
)
|
|
added += 1
|
|
except Exception as e:
|
|
print(f' DB insert error: {e}')
|
|
conn.commit()
|
|
conn.close()
|
|
print(f'Added {added} to DB')
|
|
for k in list(new_keys)[:20]:
|
|
print(f' {k}') |