210 lines
7.7 KiB
Python
Executable File
210 lines
7.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Forum Scanner v6 template — drop-in stdlib-first parser for v2ex + nodeloc + linux.do + GitHub + HN + Reddit.
|
|
Saves results to JSON, runs on a VPS without Telegram dependency.
|
|
|
|
Usage:
|
|
cp forum_scanner_v6_template.py /root/scanner/forum_scanner.py
|
|
cd /root/scanner && python3 forum_scanner.py
|
|
|
|
Expected output: forum_results.json
|
|
"""
|
|
import hashlib, json, re, os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import quote, urljoin
|
|
from urllib import request as urllib_request
|
|
|
|
try:
|
|
from bs4 import BeautifulSoup
|
|
HAS_BS4 = True
|
|
except Exception:
|
|
HAS_BS4 = False
|
|
raise SystemExit("pip3 install beautifulsoup4 lxml")
|
|
|
|
BASE_DIR = Path(__file__).parent
|
|
STATE_FILE = BASE_DIR / "seen_forum.json"
|
|
RESULT_FILE = BASE_DIR / "forum_results.json"
|
|
TIMEOUT = 25
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
}
|
|
|
|
NODELOC_COOKIES = {
|
|
"__stripe_mid": "2f6ae8f0-e223-4e41-94f5-80be154d524c888b32",
|
|
"__stripe_sid": "dc820df4-abe2-44ff-b4a1-8c8cb7e12070679a05",
|
|
"_bypass_cache": "true",
|
|
}
|
|
|
|
KEYWORDS = [
|
|
"api", "apikey", "token", "endpoint", "free", "бесплатн", "халяв",
|
|
"gpt", "claude", "deepseek", "glm", "kimi", "gemini", "grok",
|
|
"nvidia", "nim", "key", "ключ", "openai", "chatgpt", "copilot",
|
|
"codex", "jailbreak", "bypass", "обход", "reverse", "逆向",
|
|
"credits", "credit", "leak", "leaked", "утечк", "minimax", "qwen",
|
|
"codebuddy", "opencode", "modelscope", "siliconflow",
|
|
]
|
|
|
|
def load_state() -> set:
|
|
if STATE_FILE.exists():
|
|
return set(json.loads(STATE_FILE.read_text()))
|
|
return set()
|
|
|
|
def save_state(s: set):
|
|
STATE_FILE.write_text(json.dumps(list(s)[-20000:]))
|
|
|
|
seen = load_state()
|
|
|
|
def is_new(url: str) -> bool:
|
|
return hashlib.md5(url.encode()).hexdigest() not in seen
|
|
|
|
def mark(url: str):
|
|
seen.add(hashlib.md5(url.encode()).hexdigest())
|
|
|
|
def fetch(url: str, headers=None, cookies=None) -> str:
|
|
h = {**HEADERS, **(headers or {})}
|
|
req = urllib_request.Request(url, headers=h)
|
|
if cookies:
|
|
cookie_str = "; ".join([f"{k}={v}" for k, v in cookies.items()])
|
|
req.add_header("Cookie", cookie_str)
|
|
try:
|
|
with urllib_request.urlopen(req, timeout=TIMEOUT) as r:
|
|
data = r.read()
|
|
return data.decode("utf-8", errors="ignore")
|
|
except urllib_request.HTTPError as e:
|
|
body = e.read().decode("utf-8", errors="ignore")[:200]
|
|
print(f"[ERR fetch HTTP {e.code}] {url}: {body}")
|
|
return ""
|
|
except Exception as e:
|
|
print(f"[ERR fetch] {url}: {e}")
|
|
return ""
|
|
|
|
def match(text: str) -> bool:
|
|
t = text.lower()
|
|
return any(kw in t for kw in KEYWORDS)
|
|
|
|
def cat_emoji(title: str) -> str:
|
|
t = title.lower()
|
|
if any(w in t for w in ["nvidia","nim"]): return "🎮"
|
|
if any(w in t for w in ["claude","opus","sonnet"]): return "🧠"
|
|
if any(w in t for w in ["gpt","openai","chatgpt"]): return "🤖"
|
|
if any(w in t for w in ["deepseek"]): return "🔍"
|
|
if any(w in t for w in ["glm"]): return "🇨🇳"
|
|
if any(w in t for w in ["kimi"]): return "🌙"
|
|
if any(w in t for w in ["gemini","google"]): return "🌟"
|
|
if any(w in t for w in ["grok"]): return "🐦"
|
|
if any(w in t for w in ["jailbreak","bypass","reverse","exploit"]): return "🔓"
|
|
if any(w in t for w in ["leak","leaked","утеч"]): return "💧"
|
|
return "💎"
|
|
|
|
def src_v2ex() -> list:
|
|
res = []
|
|
nodes = ["ai", "python", "cloudflare", "vps"]
|
|
for node in nodes:
|
|
try:
|
|
url = f"https://www.v2ex.com/api/topics/show.json?node_name={node}"
|
|
txt = fetch(url)
|
|
if not txt: continue
|
|
data = json.loads(txt)
|
|
for post in (data or [])[:15]:
|
|
title = (post.get("title") or "").strip()
|
|
purl = post.get("url", "")
|
|
if not title or not purl: continue
|
|
if not match(title) or not is_new(purl): continue
|
|
res.append({"src":"v2ex","node":node,"title":title,"url":purl,"emoji":cat_emoji(title)})
|
|
mark(purl)
|
|
except Exception as e:
|
|
print(f"[ERR v2ex/{node}] {e}")
|
|
return res
|
|
|
|
def src_nodeloc() -> list:
|
|
res = []
|
|
tags = ["ai", "羊毛", "推广", "free"]
|
|
for tag in tags:
|
|
for p in [1, 2]:
|
|
try:
|
|
url = f"https://www.nodeloc.com/tag/{quote(tag, safe='')}?page={p}"
|
|
html = fetch(url, cookies=NODELOC_COOKIES)
|
|
if not html: continue
|
|
soup = BeautifulSoup(html, "lxml")
|
|
for el in soup.select(".topic-list-item")[:10]:
|
|
a = el.select_one("a[href*='/t/topic/']")
|
|
if not a: continue
|
|
title = a.get_text(strip=True)
|
|
url = urljoin("https://www.nodeloc.com", a.get("href", ""))
|
|
if not match(title) or not is_new(url): continue
|
|
res.append({"src":"nodeloc","tag":tag,"title":title,"url":url,"emoji":cat_emoji(title)})
|
|
mark(url)
|
|
except Exception as e:
|
|
print(f"[ERR nodeloc/{tag}/{p}] {e}")
|
|
return res
|
|
|
|
def src_github() -> list:
|
|
res = []
|
|
queries = ["free AI API 2026", "AI API key leaked", "free LLM API list 2026"]
|
|
for q in queries:
|
|
try:
|
|
url = f"https://github.com/search?q={quote(q)}&type=repositories&s=updated"
|
|
html = fetch(url)
|
|
if not html: continue
|
|
soup = BeautifulSoup(html, "lxml")
|
|
for el in soup.select(".repo-list-item")[:5]:
|
|
a = el.select_one("a.v-align-middle")
|
|
if not a: continue
|
|
title = a.get_text(strip=True)
|
|
url = urljoin("https://github.com", a.get("href", ""))
|
|
if not is_new(url): continue
|
|
res.append({"src":"github","title":title,"url":url,"emoji":"💻"})
|
|
mark(url)
|
|
except Exception as e:
|
|
print(f"[ERR github/{q}] {e}")
|
|
return res
|
|
|
|
def src_hackernews() -> list:
|
|
res = []
|
|
try:
|
|
url = "https://hn.algolia.com/api/v1/search?query=free%20AI%20API&tags=story&hitsPerPage=10"
|
|
data = json.loads(fetch(url))
|
|
for hit in data.get("hits", [])[:10]:
|
|
title = hit.get("title", "")
|
|
url_hn = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID')}"
|
|
if not match(title) or not is_new(url_hn): continue
|
|
res.append({"src":"hackernews","title":title,"url":url_hn,"emoji":"📡"})
|
|
mark(url_hn)
|
|
except Exception as e:
|
|
print(f"[ERR hackernews] {e}")
|
|
return res
|
|
|
|
def main():
|
|
print(f"[Forum Scanner v6] {datetime.now(timezone.utc).isoformat()}")
|
|
results = []
|
|
for source in [src_v2ex, src_nodeloc, src_github, src_hackernews]:
|
|
try:
|
|
items = source()
|
|
print(f" {source.__name__}: {len(items)} items")
|
|
results.extend(items)
|
|
except Exception as e:
|
|
print(f"[ERR source] {source.__name__}: {e}")
|
|
|
|
uniq, h = [], set()
|
|
for r in results:
|
|
hh = hashlib.md5(r["url"].encode()).hexdigest()
|
|
if hh in h: continue
|
|
h.add(hh); uniq.append(r)
|
|
|
|
payload = {
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"total": len(uniq),
|
|
"by_src": {s: [i for i in uniq if i["src"] == s] for s in {i["src"] for i in uniq}},
|
|
"items": uniq,
|
|
}
|
|
RESULT_FILE.write_text(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
save_state(seen)
|
|
print(f"[DONE] {len(uniq)} items → {RESULT_FILE}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|