autoreg-skills-max: initial public release (restored, redacted, hardened)
This commit is contained in:
baradok60915d52e8425
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Forum Scanner v6 template — drop-in stdlib-first parser for v2ex + nodeloc + linux.do + GitHub + HN + Reddit.
|
||||
Saves results to JSON, runs on a VPS without Telegram dependency.
|
||||
|
||||
Usage:
|
||||
cp forum_scanner_v6_template.py /root/scanner/forum_scanner.py
|
||||
cd /root/scanner && python3 forum_scanner.py
|
||||
|
||||
Expected output: forum_results.json
|
||||
"""
|
||||
import hashlib, json, re, os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urljoin
|
||||
from urllib import request as urllib_request
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
HAS_BS4 = True
|
||||
except Exception:
|
||||
HAS_BS4 = False
|
||||
raise SystemExit("pip3 install beautifulsoup4 lxml")
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
STATE_FILE = BASE_DIR / "seen_forum.json"
|
||||
RESULT_FILE = BASE_DIR / "forum_results.json"
|
||||
TIMEOUT = 25
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
}
|
||||
|
||||
NODELOC_COOKIES = {
|
||||
"__stripe_mid": "2f6ae8f0-e223-4e41-94f5-80be154d524c888b32",
|
||||
"__stripe_sid": "dc820df4-abe2-44ff-b4a1-8c8cb7e12070679a05",
|
||||
"_bypass_cache": "true",
|
||||
}
|
||||
|
||||
KEYWORDS = [
|
||||
"api", "apikey", "token", "endpoint", "free", "бесплатн", "халяв",
|
||||
"gpt", "claude", "deepseek", "glm", "kimi", "gemini", "grok",
|
||||
"nvidia", "nim", "key", "ключ", "openai", "chatgpt", "copilot",
|
||||
"codex", "jailbreak", "bypass", "обход", "reverse", "逆向",
|
||||
"credits", "credit", "leak", "leaked", "утечк", "minimax", "qwen",
|
||||
"codebuddy", "opencode", "modelscope", "siliconflow",
|
||||
]
|
||||
|
||||
def load_state() -> set:
|
||||
if STATE_FILE.exists():
|
||||
return set(json.loads(STATE_FILE.read_text()))
|
||||
return set()
|
||||
|
||||
def save_state(s: set):
|
||||
STATE_FILE.write_text(json.dumps(list(s)[-20000:]))
|
||||
|
||||
seen = load_state()
|
||||
|
||||
def is_new(url: str) -> bool:
|
||||
return hashlib.md5(url.encode()).hexdigest() not in seen
|
||||
|
||||
def mark(url: str):
|
||||
seen.add(hashlib.md5(url.encode()).hexdigest())
|
||||
|
||||
def fetch(url: str, headers=None, cookies=None) -> str:
|
||||
h = {**HEADERS, **(headers or {})}
|
||||
req = urllib_request.Request(url, headers=h)
|
||||
if cookies:
|
||||
cookie_str = "; ".join([f"{k}={v}" for k, v in cookies.items()])
|
||||
req.add_header("Cookie", cookie_str)
|
||||
try:
|
||||
with urllib_request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
data = r.read()
|
||||
return data.decode("utf-8", errors="ignore")
|
||||
except urllib_request.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="ignore")[:200]
|
||||
print(f"[ERR fetch HTTP {e.code}] {url}: {body}")
|
||||
return ""
|
||||
except Exception as e:
|
||||
print(f"[ERR fetch] {url}: {e}")
|
||||
return ""
|
||||
|
||||
def match(text: str) -> bool:
|
||||
t = text.lower()
|
||||
return any(kw in t for kw in KEYWORDS)
|
||||
|
||||
def cat_emoji(title: str) -> str:
|
||||
t = title.lower()
|
||||
if any(w in t for w in ["nvidia","nim"]): return "🎮"
|
||||
if any(w in t for w in ["claude","opus","sonnet"]): return "🧠"
|
||||
if any(w in t for w in ["gpt","openai","chatgpt"]): return "🤖"
|
||||
if any(w in t for w in ["deepseek"]): return "🔍"
|
||||
if any(w in t for w in ["glm"]): return "🇨🇳"
|
||||
if any(w in t for w in ["kimi"]): return "🌙"
|
||||
if any(w in t for w in ["gemini","google"]): return "🌟"
|
||||
if any(w in t for w in ["grok"]): return "🐦"
|
||||
if any(w in t for w in ["jailbreak","bypass","reverse","exploit"]): return "🔓"
|
||||
if any(w in t for w in ["leak","leaked","утеч"]): return "💧"
|
||||
return "💎"
|
||||
|
||||
def src_v2ex() -> list:
|
||||
res = []
|
||||
nodes = ["ai", "python", "cloudflare", "vps"]
|
||||
for node in nodes:
|
||||
try:
|
||||
url = f"https://www.v2ex.com/api/topics/show.json?node_name={node}"
|
||||
txt = fetch(url)
|
||||
if not txt: continue
|
||||
data = json.loads(txt)
|
||||
for post in (data or [])[:15]:
|
||||
title = (post.get("title") or "").strip()
|
||||
purl = post.get("url", "")
|
||||
if not title or not purl: continue
|
||||
if not match(title) or not is_new(purl): continue
|
||||
res.append({"src":"v2ex","node":node,"title":title,"url":purl,"emoji":cat_emoji(title)})
|
||||
mark(purl)
|
||||
except Exception as e:
|
||||
print(f"[ERR v2ex/{node}] {e}")
|
||||
return res
|
||||
|
||||
def src_nodeloc() -> list:
|
||||
res = []
|
||||
tags = ["ai", "羊毛", "推广", "free"]
|
||||
for tag in tags:
|
||||
for p in [1, 2]:
|
||||
try:
|
||||
url = f"https://www.nodeloc.com/tag/{quote(tag, safe='')}?page={p}"
|
||||
html = fetch(url, cookies=NODELOC_COOKIES)
|
||||
if not html: continue
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for el in soup.select(".topic-list-item")[:10]:
|
||||
a = el.select_one("a[href*='/t/topic/']")
|
||||
if not a: continue
|
||||
title = a.get_text(strip=True)
|
||||
url = urljoin("https://www.nodeloc.com", a.get("href", ""))
|
||||
if not match(title) or not is_new(url): continue
|
||||
res.append({"src":"nodeloc","tag":tag,"title":title,"url":url,"emoji":cat_emoji(title)})
|
||||
mark(url)
|
||||
except Exception as e:
|
||||
print(f"[ERR nodeloc/{tag}/{p}] {e}")
|
||||
return res
|
||||
|
||||
def src_github() -> list:
|
||||
res = []
|
||||
queries = ["free AI API 2026", "AI API key leaked", "free LLM API list 2026"]
|
||||
for q in queries:
|
||||
try:
|
||||
url = f"https://github.com/search?q={quote(q)}&type=repositories&s=updated"
|
||||
html = fetch(url)
|
||||
if not html: continue
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for el in soup.select(".repo-list-item")[:5]:
|
||||
a = el.select_one("a.v-align-middle")
|
||||
if not a: continue
|
||||
title = a.get_text(strip=True)
|
||||
url = urljoin("https://github.com", a.get("href", ""))
|
||||
if not is_new(url): continue
|
||||
res.append({"src":"github","title":title,"url":url,"emoji":"💻"})
|
||||
mark(url)
|
||||
except Exception as e:
|
||||
print(f"[ERR github/{q}] {e}")
|
||||
return res
|
||||
|
||||
def src_hackernews() -> list:
|
||||
res = []
|
||||
try:
|
||||
url = "https://hn.algolia.com/api/v1/search?query=free%20AI%20API&tags=story&hitsPerPage=10"
|
||||
data = json.loads(fetch(url))
|
||||
for hit in data.get("hits", [])[:10]:
|
||||
title = hit.get("title", "")
|
||||
url_hn = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID')}"
|
||||
if not match(title) or not is_new(url_hn): continue
|
||||
res.append({"src":"hackernews","title":title,"url":url_hn,"emoji":"📡"})
|
||||
mark(url_hn)
|
||||
except Exception as e:
|
||||
print(f"[ERR hackernews] {e}")
|
||||
return res
|
||||
|
||||
def main():
|
||||
print(f"[Forum Scanner v6] {datetime.now(timezone.utc).isoformat()}")
|
||||
results = []
|
||||
for source in [src_v2ex, src_nodeloc, src_github, src_hackernews]:
|
||||
try:
|
||||
items = source()
|
||||
print(f" {source.__name__}: {len(items)} items")
|
||||
results.extend(items)
|
||||
except Exception as e:
|
||||
print(f"[ERR source] {source.__name__}: {e}")
|
||||
|
||||
uniq, h = [], set()
|
||||
for r in results:
|
||||
hh = hashlib.md5(r["url"].encode()).hexdigest()
|
||||
if hh in h: continue
|
||||
h.add(hh); uniq.append(r)
|
||||
|
||||
payload = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"total": len(uniq),
|
||||
"by_src": {s: [i for i in uniq if i["src"] == s] for s in {i["src"] for i in uniq}},
|
||||
"items": uniq,
|
||||
}
|
||||
RESULT_FILE.write_text(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
save_state(seen)
|
||||
print(f"[DONE] {len(uniq)} items → {RESULT_FILE}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user