omni-parse: 12 провайдеров, sqlite+fts5, simhash-дедуп, фильтр релевантности

omni/providers/reddit.py

@@ -0,0 +1,99 @@
+"""Reddit без ключей: поиск по всему реддиту и по сабам, сортировка new/top."""
+import urllib.parse
+
+from .. import net
+from ..model import Item
+
+DEFAULT_SUBS = ["netsec", "cybersecurity", "webscraping", "selfhosted", "webdev", "Python"]
+UA = {"User-Agent": "omni-parse/1.0 (research; contact: local)"}
+
+
+def available(name="reddit"):
+    return True, "pullpush (архив) + публичный json, ключи не нужны"
+
+
+def _pullpush(query, limit=25, subreddit=None, kind="submission"):
+    q = urllib.parse.quote(query or "")
+    url = f"https://api.pullpush.io/reddit/search/{kind}/?q={q}&size={min(limit, 100)}&sort=desc"
+    if subreddit:
+        url += f"&subreddit={subreddit}"
+    r = net.get(url, headers=UA, json_mode=True)
+    data = (r.json() or {}).get("data") or []
+    out = []
+    for d in data:
+        if not (d.get("title") or d.get("body")):
+            continue
+        it = _mk(d, subreddit)
+        it.kind = "thread" if kind == "submission" else "comment"
+        if kind == "comment":
+            it.title = (d.get("link_title") or it.text[:100])
+        out.append(it)
+    return out
+
+
+def _rows(payload):
+    out = []
+    for ch in (payload or {}).get("data", {}).get("children", []):
+        d = ch.get("data") or {}
+        if not d.get("title"):
+            continue
+        out.append(d)
+    return out
+
+
+def _mk(d, sub=None):
+    perm = d.get("permalink") or ""
+    url = "https://www.reddit.com" + perm if perm else d.get("url", "")
+    body = (d.get("selftext") or "")[:1800]
+    it = Item("reddit", "thread", url=url, title=d.get("title", ""), text=body,
+              source="r/" + (d.get("subreddit") or sub or ""), author=d.get("author", ""),
+              published=str(int(d.get("created_utc") or 0)),
+              extra={"ups": d.get("ups"), "comments": d.get("num_comments"),
+                     "sub": d.get("subreddit"), "ext": d.get("url", "")[:300]})
+    it.score = float(d.get("ups") or 0) ** 0.5 + float(d.get("num_comments") or 0) ** 0.4
+    return it
+
+
+def _reddit_native(query, limit=20):
+    q = urllib.parse.quote(query)
+    r = net.get(f"https://www.reddit.com/search.json?q={q}&limit={min(limit, 25)}&sort=relevance&t=month",
+                headers={**UA, "Accept": "application/json"}, json_mode=True)
+    if r.status != 200:
+        return []
+    return [_mk(d) for d in _rows(r.json())]
+
+
+def _relevant(query, item):
+    toks = [t for t in __import__("re").findall(r"\w{3,}", (query or "").lower())]
+    if not toks:
+        return True
+    body = ((item.title or "") + " " + (item.text or "")).lower()
+    hit = sum(1 for t in toks if t in body)
+    title_hit = sum(1 for t in toks if t in (item.title or "").lower())
+    return hit >= max(2, int(len(toks) * 0.7)) and title_hit >= 1
+
+
+def search_reddit(query, limit=20, subs=None, include_comments=True):
+    items = _pullpush(query, limit=limit)
+    if len(items) < 5:
+        items += _reddit_native(query, limit=limit)
+    for sub in (subs or [])[:3]:
+        items += _pullpush(query, limit=10, subreddit=sub)
+    if include_comments and len(items) < limit:
+        items += _pullpush(query, limit=10, kind="comment")[:8]
+    items = [i for i in items if _relevant(query, i)]
+    uniq, seen = [], set()
+    for it in items:
+        k = it.url or it.title
+        if k in seen:
+            continue
+        seen.add(k)
+        uniq.append(it)
+    return uniq[:limit * 2]
+
+
+def hot(subs=None, limit=20):
+    out = []
+    for sub in (subs or DEFAULT_SUBS)[:6]:
+        out += _pullpush("", limit=min(limit, 15), subreddit=sub)
+    return out