151 lines
6.5 KiB
Python
151 lines
6.5 KiB
Python
"""Форумы без ключей: Discourse, HN (Algolia), Lobsters, XenForo, Telegram-веб."""
|
|
import html
|
|
import json
|
|
import re
|
|
import urllib.parse
|
|
|
|
from .. import net
|
|
from ..model import Item
|
|
|
|
DISCOURSE = [
|
|
"meta.discourse.org", "discuss.python.org", "community.openai.com", "forum.cursor.com",
|
|
"discuss.huggingface.co", "community.cloudflare.com", "forum.gitlab.com", "discuss.elastic.co",
|
|
]
|
|
XENFORO = ["www.blackhatworld.com", "xenforo.com/community"]
|
|
TG_CHANNELS = ["Abuz_ai_soft", "AlStack", "iishkogatewaybot"]
|
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
|
|
|
|
|
|
def available(name="forums"):
|
|
return True, "Discourse/HN/Lobsters/TG-веб, ключи не нужны"
|
|
|
|
|
|
def _strip(s):
|
|
return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip()
|
|
|
|
|
|
def _discourse(host, query, limit=8):
|
|
out = []
|
|
url = f"https://{host}/search.json?q={urllib.parse.quote(query)}"
|
|
r = net.get(url, headers=UA, json_mode=True)
|
|
if r.status != 200:
|
|
return out
|
|
d = r.json() or {}
|
|
for t in (d.get("topics") or [])[:limit]:
|
|
slug = t.get("slug") or "t"
|
|
tid = t.get("id")
|
|
out.append(Item("forums", "thread", url=f"https://{host}/t/{slug}/{tid}",
|
|
title=t.get("title", ""), text=_strip(t.get("blurb") or ""),
|
|
source=host, published=str(t.get("created_at") or ""),
|
|
score=float(t.get("posts_count") or 0) ** 0.4 +
|
|
float(t.get("like_count") or 0) ** 0.3,
|
|
extra={"posts": t.get("posts_count"), "views": t.get("views")}))
|
|
for p in (d.get("posts") or [])[:limit]:
|
|
if len(out) >= limit:
|
|
break
|
|
out.append(Item("forums", "post", url=f"https://{host}/t/{p.get('topic_id')}/{p.get('post_number', 1)}",
|
|
title=(p.get("topic_title") or "")[:200], text=_strip(p.get("blurb") or ""),
|
|
source=host, author=p.get("username", ""), published=str(p.get("created_at") or "")))
|
|
return out
|
|
|
|
|
|
def _hn(query, limit=10):
|
|
out = []
|
|
url = ("https://hn.algolia.com/api/v1/search?query=" + urllib.parse.quote(query)
|
|
+ f"&tags=story&hitsPerPage={min(limit, 30)}")
|
|
r = net.get(url, headers=UA, json_mode=True)
|
|
for h in ((r.json() or {}).get("hits") or []):
|
|
title = h.get("title") or h.get("story_title") or ""
|
|
if not title:
|
|
continue
|
|
it = Item("forums", "thread", url=h.get("url") or f"https://news.ycombinator.com/item?id={h.get('objectID')}",
|
|
title=title, text=(h.get("story_text") or "")[:1500], source="news.ycombinator.com",
|
|
author=h.get("author", ""), published=h.get("created_at", ""),
|
|
extra={"points": h.get("points"), "comments": h.get("num_comments")})
|
|
it.score = float(h.get("points") or 0) ** 0.5 + float(h.get("num_comments") or 0) ** 0.4
|
|
out.append(it)
|
|
return out
|
|
|
|
|
|
def _lobsters(query, limit=8):
|
|
out = []
|
|
r = net.get("https://lobste.rs/search.json?q=" + urllib.parse.quote(query) + "&what=stories&order=newest",
|
|
headers=UA, json_mode=True)
|
|
for s in ((r.json() if r.status == 200 else []) or [])[:limit]:
|
|
it = Item("forums", "thread", url=s.get("comments_url") or s.get("url") or "",
|
|
title=s.get("title", ""), text=_strip(s.get("description") or ""),
|
|
source="lobste.rs", author=s.get("submitter_user", "") if isinstance(s.get("submitter_user"), str)
|
|
else (s.get("submitter_user") or {}).get("username", ""),
|
|
published=s.get("created_at", ""),
|
|
extra={"score": s.get("score"), "comments": s.get("comment_count")})
|
|
it.score = float(s.get("score") or 0) ** 0.5
|
|
out.append(it)
|
|
return out
|
|
|
|
|
|
def _xenforo(host, query, limit=6):
|
|
out = []
|
|
base = f"https://{host}" if "//" not in host else host
|
|
url = f"{base}/search/?q={urllib.parse.quote(query)}&o=date"
|
|
r = net.get(url, headers=UA)
|
|
if r.status != 200:
|
|
return out
|
|
for m in re.finditer(r'<h3 class="contentRow-title">\s*<a href="([^"]+)"[^>]*>(.*?)</a>', r.text, re.S):
|
|
href, title = m.group(1), _strip(m.group(2))
|
|
if not title:
|
|
continue
|
|
link = href if href.startswith("http") else base + href
|
|
out.append(Item("forums", "thread", url=link, title=title, source=host))
|
|
if len(out) >= limit:
|
|
break
|
|
return out
|
|
|
|
|
|
def _tg(channel, query, limit=8):
|
|
out = []
|
|
r = net.get(f"https://t.me/s/{channel}", headers=UA)
|
|
if r.status != 200:
|
|
return out
|
|
blocks = re.split(r'<div class="tgme_widget_message_wrap', r.text)[1:]
|
|
ql = query.lower()
|
|
for b in reversed(blocks):
|
|
mtext = re.search(r'<div class="tgme_widget_message_text[^"]*"[^>]*>(.*?)</div>', b, re.S)
|
|
if not mtext:
|
|
continue
|
|
text = _strip(mtext.group(1))
|
|
if ql not in text.lower():
|
|
continue
|
|
mlink = re.search(r'data-post="([^"]+)"', b)
|
|
link = f"https://t.me/{mlink.group(1)}" if mlink else f"https://t.me/s/{channel}"
|
|
it = Item("forums", "tg_post", url=link, title=text[:120], text=text, source="t.me/" + channel,
|
|
published=(re.search(r'datetime="([^"]+)"', b).group(1) if re.search(r'datetime="([^"]+)"', b) else ""))
|
|
out.append(it)
|
|
if len(out) >= limit:
|
|
break
|
|
return out
|
|
|
|
|
|
def _relevant(query, item):
|
|
"""Дискурс любит отдавать мусор по одному совпавшему слову: требуем покрытие терминов."""
|
|
toks = [t for t in re.findall(r"\w{3,}", (query or "").lower())]
|
|
if not toks:
|
|
return True
|
|
body = ((item.title or "") + " " + (item.text or "")).lower()
|
|
hit = sum(1 for t in toks if t in body)
|
|
title_hit = sum(1 for t in toks if t in (item.title or "").lower())
|
|
return (hit >= max(2, int(len(toks) * 0.7))) and title_hit >= 1 if len(toks) > 1 else hit == 1
|
|
|
|
|
|
def search_forums(query, limit=20, hosts=None, channels=None):
|
|
items = []
|
|
for host in (hosts or DISCOURSE)[:5]:
|
|
items += _discourse(host, query, limit=6)
|
|
items += _hn(query, limit=10)
|
|
items += _lobsters(query, limit=6)
|
|
for ch in (channels or TG_CHANNELS)[:3]:
|
|
items += _tg(ch, query, limit=6)
|
|
if len(items) < limit:
|
|
for host in (hosts or []) and XENFORO[:1]:
|
|
items += _xenforo(host, query, limit=5)
|
|
items = [i for i in items if _relevant(query, i) or i.kind == "tg_post"]
|
|
return items[:limit * 2] |