omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности
This commit is contained in:
dev
151
omni/providers/forums.py
Normal file
151
omni/providers/forums.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Форумы без ключей: Discourse, HN (Algolia), Lobsters, XenForo, Telegram-веб."""
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .. import net
|
||||
from ..model import Item
|
||||
|
||||
DISCOURSE = [
|
||||
"meta.discourse.org", "discuss.python.org", "community.openai.com", "forum.cursor.com",
|
||||
"discuss.huggingface.co", "community.cloudflare.com", "forum.gitlab.com", "discuss.elastic.co",
|
||||
]
|
||||
XENFORO = ["www.blackhatworld.com", "xenforo.com/community"]
|
||||
TG_CHANNELS = ["Abuz_ai_soft", "AlStack", "iishkogatewaybot"]
|
||||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36"}
|
||||
|
||||
|
||||
def available(name="forums"):
|
||||
return True, "Discourse/HN/Lobsters/TG-веб, ключи не нужны"
|
||||
|
||||
|
||||
def _strip(s):
|
||||
return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip()
|
||||
|
||||
|
||||
def _discourse(host, query, limit=8):
|
||||
out = []
|
||||
url = f"https://{host}/search.json?q={urllib.parse.quote(query)}"
|
||||
r = net.get(url, headers=UA, json_mode=True)
|
||||
if r.status != 200:
|
||||
return out
|
||||
d = r.json() or {}
|
||||
for t in (d.get("topics") or [])[:limit]:
|
||||
slug = t.get("slug") or "t"
|
||||
tid = t.get("id")
|
||||
out.append(Item("forums", "thread", url=f"https://{host}/t/{slug}/{tid}",
|
||||
title=t.get("title", ""), text=_strip(t.get("blurb") or ""),
|
||||
source=host, published=str(t.get("created_at") or ""),
|
||||
score=float(t.get("posts_count") or 0) ** 0.4 +
|
||||
float(t.get("like_count") or 0) ** 0.3,
|
||||
extra={"posts": t.get("posts_count"), "views": t.get("views")}))
|
||||
for p in (d.get("posts") or [])[:limit]:
|
||||
if len(out) >= limit:
|
||||
break
|
||||
out.append(Item("forums", "post", url=f"https://{host}/t/{p.get('topic_id')}/{p.get('post_number', 1)}",
|
||||
title=(p.get("topic_title") or "")[:200], text=_strip(p.get("blurb") or ""),
|
||||
source=host, author=p.get("username", ""), published=str(p.get("created_at") or "")))
|
||||
return out
|
||||
|
||||
|
||||
def _hn(query, limit=10):
|
||||
out = []
|
||||
url = ("https://hn.algolia.com/api/v1/search?query=" + urllib.parse.quote(query)
|
||||
+ f"&tags=story&hitsPerPage={min(limit, 30)}")
|
||||
r = net.get(url, headers=UA, json_mode=True)
|
||||
for h in ((r.json() or {}).get("hits") or []):
|
||||
title = h.get("title") or h.get("story_title") or ""
|
||||
if not title:
|
||||
continue
|
||||
it = Item("forums", "thread", url=h.get("url") or f"https://news.ycombinator.com/item?id={h.get('objectID')}",
|
||||
title=title, text=(h.get("story_text") or "")[:1500], source="news.ycombinator.com",
|
||||
author=h.get("author", ""), published=h.get("created_at", ""),
|
||||
extra={"points": h.get("points"), "comments": h.get("num_comments")})
|
||||
it.score = float(h.get("points") or 0) ** 0.5 + float(h.get("num_comments") or 0) ** 0.4
|
||||
out.append(it)
|
||||
return out
|
||||
|
||||
|
||||
def _lobsters(query, limit=8):
|
||||
out = []
|
||||
r = net.get("https://lobste.rs/search.json?q=" + urllib.parse.quote(query) + "&what=stories&order=newest",
|
||||
headers=UA, json_mode=True)
|
||||
for s in ((r.json() if r.status == 200 else []) or [])[:limit]:
|
||||
it = Item("forums", "thread", url=s.get("comments_url") or s.get("url") or "",
|
||||
title=s.get("title", ""), text=_strip(s.get("description") or ""),
|
||||
source="lobste.rs", author=s.get("submitter_user", "") if isinstance(s.get("submitter_user"), str)
|
||||
else (s.get("submitter_user") or {}).get("username", ""),
|
||||
published=s.get("created_at", ""),
|
||||
extra={"score": s.get("score"), "comments": s.get("comment_count")})
|
||||
it.score = float(s.get("score") or 0) ** 0.5
|
||||
out.append(it)
|
||||
return out
|
||||
|
||||
|
||||
def _xenforo(host, query, limit=6):
|
||||
out = []
|
||||
base = f"https://{host}" if "//" not in host else host
|
||||
url = f"{base}/search/?q={urllib.parse.quote(query)}&o=date"
|
||||
r = net.get(url, headers=UA)
|
||||
if r.status != 200:
|
||||
return out
|
||||
for m in re.finditer(r'<h3 class="contentRow-title">\s*<a href="([^"]+)"[^>]*>(.*?)</a>', r.text, re.S):
|
||||
href, title = m.group(1), _strip(m.group(2))
|
||||
if not title:
|
||||
continue
|
||||
link = href if href.startswith("http") else base + href
|
||||
out.append(Item("forums", "thread", url=link, title=title, source=host))
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _tg(channel, query, limit=8):
|
||||
out = []
|
||||
r = net.get(f"https://t.me/s/{channel}", headers=UA)
|
||||
if r.status != 200:
|
||||
return out
|
||||
blocks = re.split(r'<div class="tgme_widget_message_wrap', r.text)[1:]
|
||||
ql = query.lower()
|
||||
for b in reversed(blocks):
|
||||
mtext = re.search(r'<div class="tgme_widget_message_text[^"]*"[^>]*>(.*?)</div>', b, re.S)
|
||||
if not mtext:
|
||||
continue
|
||||
text = _strip(mtext.group(1))
|
||||
if ql not in text.lower():
|
||||
continue
|
||||
mlink = re.search(r'data-post="([^"]+)"', b)
|
||||
link = f"https://t.me/{mlink.group(1)}" if mlink else f"https://t.me/s/{channel}"
|
||||
it = Item("forums", "tg_post", url=link, title=text[:120], text=text, source="t.me/" + channel,
|
||||
published=(re.search(r'datetime="([^"]+)"', b).group(1) if re.search(r'datetime="([^"]+)"', b) else ""))
|
||||
out.append(it)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _relevant(query, item):
|
||||
"""Дискурс любит отдавать мусор по одному совпавшему слову: требуем покрытие терминов."""
|
||||
toks = [t for t in re.findall(r"\w{3,}", (query or "").lower())]
|
||||
if not toks:
|
||||
return True
|
||||
body = ((item.title or "") + " " + (item.text or "")).lower()
|
||||
hit = sum(1 for t in toks if t in body)
|
||||
title_hit = sum(1 for t in toks if t in (item.title or "").lower())
|
||||
return (hit >= max(2, int(len(toks) * 0.7))) and title_hit >= 1 if len(toks) > 1 else hit == 1
|
||||
|
||||
|
||||
def search_forums(query, limit=20, hosts=None, channels=None):
|
||||
items = []
|
||||
for host in (hosts or DISCOURSE)[:5]:
|
||||
items += _discourse(host, query, limit=6)
|
||||
items += _hn(query, limit=10)
|
||||
items += _lobsters(query, limit=6)
|
||||
for ch in (channels or TG_CHANNELS)[:3]:
|
||||
items += _tg(ch, query, limit=6)
|
||||
if len(items) < limit:
|
||||
for host in (hosts or []) and XENFORO[:1]:
|
||||
items += _xenforo(host, query, limit=5)
|
||||
items = [i for i in items if _relevant(query, i) or i.kind == "tg_post"]
|
||||
return items[:limit * 2]
|
||||
Reference in New Issue
Block a user