99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
"""Reddit без ключей: поиск по всему реддиту и по сабам, сортировка new/top."""
|
|
import urllib.parse
|
|
|
|
from .. import net
|
|
from ..model import Item
|
|
|
|
DEFAULT_SUBS = ["netsec", "cybersecurity", "webscraping", "selfhosted", "webdev", "Python"]
|
|
UA = {"User-Agent": "omni-parse/1.0 (research; contact: local)"}
|
|
|
|
|
|
def available(name="reddit"):
|
|
return True, "pullpush (архив) + публичный json, ключи не нужны"
|
|
|
|
|
|
def _pullpush(query, limit=25, subreddit=None, kind="submission"):
|
|
q = urllib.parse.quote(query or "")
|
|
url = f"https://api.pullpush.io/reddit/search/{kind}/?q={q}&size={min(limit, 100)}&sort=desc"
|
|
if subreddit:
|
|
url += f"&subreddit={subreddit}"
|
|
r = net.get(url, headers=UA, json_mode=True)
|
|
data = (r.json() or {}).get("data") or []
|
|
out = []
|
|
for d in data:
|
|
if not (d.get("title") or d.get("body")):
|
|
continue
|
|
it = _mk(d, subreddit)
|
|
it.kind = "thread" if kind == "submission" else "comment"
|
|
if kind == "comment":
|
|
it.title = (d.get("link_title") or it.text[:100])
|
|
out.append(it)
|
|
return out
|
|
|
|
|
|
def _rows(payload):
|
|
out = []
|
|
for ch in (payload or {}).get("data", {}).get("children", []):
|
|
d = ch.get("data") or {}
|
|
if not d.get("title"):
|
|
continue
|
|
out.append(d)
|
|
return out
|
|
|
|
|
|
def _mk(d, sub=None):
|
|
perm = d.get("permalink") or ""
|
|
url = "https://www.reddit.com" + perm if perm else d.get("url", "")
|
|
body = (d.get("selftext") or "")[:1800]
|
|
it = Item("reddit", "thread", url=url, title=d.get("title", ""), text=body,
|
|
source="r/" + (d.get("subreddit") or sub or ""), author=d.get("author", ""),
|
|
published=str(int(d.get("created_utc") or 0)),
|
|
extra={"ups": d.get("ups"), "comments": d.get("num_comments"),
|
|
"sub": d.get("subreddit"), "ext": d.get("url", "")[:300]})
|
|
it.score = float(d.get("ups") or 0) ** 0.5 + float(d.get("num_comments") or 0) ** 0.4
|
|
return it
|
|
|
|
|
|
def _reddit_native(query, limit=20):
|
|
q = urllib.parse.quote(query)
|
|
r = net.get(f"https://www.reddit.com/search.json?q={q}&limit={min(limit, 25)}&sort=relevance&t=month",
|
|
headers={**UA, "Accept": "application/json"}, json_mode=True)
|
|
if r.status != 200:
|
|
return []
|
|
return [_mk(d) for d in _rows(r.json())]
|
|
|
|
|
|
def _relevant(query, item):
|
|
toks = [t for t in __import__("re").findall(r"\w{3,}", (query or "").lower())]
|
|
if not toks:
|
|
return True
|
|
body = ((item.title or "") + " " + (item.text or "")).lower()
|
|
hit = sum(1 for t in toks if t in body)
|
|
title_hit = sum(1 for t in toks if t in (item.title or "").lower())
|
|
return hit >= max(2, int(len(toks) * 0.7)) and title_hit >= 1
|
|
|
|
|
|
def search_reddit(query, limit=20, subs=None, include_comments=True):
|
|
items = _pullpush(query, limit=limit)
|
|
if len(items) < 5:
|
|
items += _reddit_native(query, limit=limit)
|
|
for sub in (subs or [])[:3]:
|
|
items += _pullpush(query, limit=10, subreddit=sub)
|
|
if include_comments and len(items) < limit:
|
|
items += _pullpush(query, limit=10, kind="comment")[:8]
|
|
items = [i for i in items if _relevant(query, i)]
|
|
uniq, seen = [], set()
|
|
for it in items:
|
|
k = it.url or it.title
|
|
if k in seen:
|
|
continue
|
|
seen.add(k)
|
|
uniq.append(it)
|
|
return uniq[:limit * 2]
|
|
|
|
|
|
def hot(subs=None, limit=20):
|
|
out = []
|
|
for sub in (subs or DEFAULT_SUBS)[:6]:
|
|
out += _pullpush("", limit=min(limit, 15), subreddit=sub)
|
|
return out |