#!/usr/bin/env python3 """ Forum Scanner v6 template — drop-in stdlib-first parser for v2ex + nodeloc + linux.do + GitHub + HN + Reddit. Saves results to JSON, runs on a VPS without Telegram dependency. Usage: cp forum_scanner_v6_template.py /root/scanner/forum_scanner.py cd /root/scanner && python3 forum_scanner.py Expected output: forum_results.json """ import hashlib, json, re, os from datetime import datetime, timezone from pathlib import Path from urllib.parse import quote, urljoin from urllib import request as urllib_request try: from bs4 import BeautifulSoup HAS_BS4 = True except Exception: HAS_BS4 = False raise SystemExit("pip3 install beautifulsoup4 lxml") BASE_DIR = Path(__file__).parent STATE_FILE = BASE_DIR / "seen_forum.json" RESULT_FILE = BASE_DIR / "forum_results.json" TIMEOUT = 25 HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", } NODELOC_COOKIES = { "__stripe_mid": "2f6ae8f0-e223-4e41-94f5-80be154d524c888b32", "__stripe_sid": "dc820df4-abe2-44ff-b4a1-8c8cb7e12070679a05", "_bypass_cache": "true", } KEYWORDS = [ "api", "apikey", "token", "endpoint", "free", "бесплатн", "халяв", "gpt", "claude", "deepseek", "glm", "kimi", "gemini", "grok", "nvidia", "nim", "key", "ключ", "openai", "chatgpt", "copilot", "codex", "jailbreak", "bypass", "обход", "reverse", "逆向", "credits", "credit", "leak", "leaked", "утечк", "minimax", "qwen", "codebuddy", "opencode", "modelscope", "siliconflow", ] def load_state() -> set: if STATE_FILE.exists(): return set(json.loads(STATE_FILE.read_text())) return set() def save_state(s: set): STATE_FILE.write_text(json.dumps(list(s)[-20000:])) seen = load_state() def is_new(url: str) -> bool: return hashlib.md5(url.encode()).hexdigest() not in seen def mark(url: str): seen.add(hashlib.md5(url.encode()).hexdigest()) def fetch(url: str, headers=None, cookies=None) -> str: h = {**HEADERS, **(headers or {})} req = urllib_request.Request(url, headers=h) if cookies: cookie_str = "; ".join([f"{k}={v}" for k, v in cookies.items()]) req.add_header("Cookie", cookie_str) try: with urllib_request.urlopen(req, timeout=TIMEOUT) as r: data = r.read() return data.decode("utf-8", errors="ignore") except urllib_request.HTTPError as e: body = e.read().decode("utf-8", errors="ignore")[:200] print(f"[ERR fetch HTTP {e.code}] {url}: {body}") return "" except Exception as e: print(f"[ERR fetch] {url}: {e}") return "" def match(text: str) -> bool: t = text.lower() return any(kw in t for kw in KEYWORDS) def cat_emoji(title: str) -> str: t = title.lower() if any(w in t for w in ["nvidia","nim"]): return "🎮" if any(w in t for w in ["claude","opus","sonnet"]): return "🧠" if any(w in t for w in ["gpt","openai","chatgpt"]): return "🤖" if any(w in t for w in ["deepseek"]): return "🔍" if any(w in t for w in ["glm"]): return "🇨🇳" if any(w in t for w in ["kimi"]): return "🌙" if any(w in t for w in ["gemini","google"]): return "🌟" if any(w in t for w in ["grok"]): return "🐦" if any(w in t for w in ["jailbreak","bypass","reverse","exploit"]): return "🔓" if any(w in t for w in ["leak","leaked","утеч"]): return "💧" return "💎" def src_v2ex() -> list: res = [] nodes = ["ai", "python", "cloudflare", "vps"] for node in nodes: try: url = f"https://www.v2ex.com/api/topics/show.json?node_name={node}" txt = fetch(url) if not txt: continue data = json.loads(txt) for post in (data or [])[:15]: title = (post.get("title") or "").strip() purl = post.get("url", "") if not title or not purl: continue if not match(title) or not is_new(purl): continue res.append({"src":"v2ex","node":node,"title":title,"url":purl,"emoji":cat_emoji(title)}) mark(purl) except Exception as e: print(f"[ERR v2ex/{node}] {e}") return res def src_nodeloc() -> list: res = [] tags = ["ai", "羊毛", "推广", "free"] for tag in tags: for p in [1, 2]: try: url = f"https://www.nodeloc.com/tag/{quote(tag, safe='')}?page={p}" html = fetch(url, cookies=NODELOC_COOKIES) if not html: continue soup = BeautifulSoup(html, "lxml") for el in soup.select(".topic-list-item")[:10]: a = el.select_one("a[href*='/t/topic/']") if not a: continue title = a.get_text(strip=True) url = urljoin("https://www.nodeloc.com", a.get("href", "")) if not match(title) or not is_new(url): continue res.append({"src":"nodeloc","tag":tag,"title":title,"url":url,"emoji":cat_emoji(title)}) mark(url) except Exception as e: print(f"[ERR nodeloc/{tag}/{p}] {e}") return res def src_github() -> list: res = [] queries = ["free AI API 2026", "AI API key leaked", "free LLM API list 2026"] for q in queries: try: url = f"https://github.com/search?q={quote(q)}&type=repositories&s=updated" html = fetch(url) if not html: continue soup = BeautifulSoup(html, "lxml") for el in soup.select(".repo-list-item")[:5]: a = el.select_one("a.v-align-middle") if not a: continue title = a.get_text(strip=True) url = urljoin("https://github.com", a.get("href", "")) if not is_new(url): continue res.append({"src":"github","title":title,"url":url,"emoji":"💻"}) mark(url) except Exception as e: print(f"[ERR github/{q}] {e}") return res def src_hackernews() -> list: res = [] try: url = "https://hn.algolia.com/api/v1/search?query=free%20AI%20API&tags=story&hitsPerPage=10" data = json.loads(fetch(url)) for hit in data.get("hits", [])[:10]: title = hit.get("title", "") url_hn = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID')}" if not match(title) or not is_new(url_hn): continue res.append({"src":"hackernews","title":title,"url":url_hn,"emoji":"📡"}) mark(url_hn) except Exception as e: print(f"[ERR hackernews] {e}") return res def main(): print(f"[Forum Scanner v6] {datetime.now(timezone.utc).isoformat()}") results = [] for source in [src_v2ex, src_nodeloc, src_github, src_hackernews]: try: items = source() print(f" {source.__name__}: {len(items)} items") results.extend(items) except Exception as e: print(f"[ERR source] {source.__name__}: {e}") uniq, h = [], set() for r in results: hh = hashlib.md5(r["url"].encode()).hexdigest() if hh in h: continue h.add(hh); uniq.append(r) payload = { "timestamp": datetime.now(timezone.utc).isoformat(), "total": len(uniq), "by_src": {s: [i for i in uniq if i["src"] == s] for s in {i["src"] for i in uniq}}, "items": uniq, } RESULT_FILE.write_text(json.dumps(payload, ensure_ascii=False, indent=2)) save_state(seen) print(f"[DONE] {len(uniq)} items → {RESULT_FILE}") if __name__ == "__main__": main()