1081 lines
43 KiB
Python
1081 lines
43 KiB
Python
"""Забор источников из реестра sources.py.
|
||
|
||
Один вход: fetch(source, limit) -> list[Item]. Адаптер выбирается по source.adapter.
|
||
Всё без ключей, кроме опционального GITHUB_TOKEN (поднимает лимит api.github.com).
|
||
"""
|
||
import html
|
||
import json
|
||
import re
|
||
import urllib.parse
|
||
import xml.etree.ElementTree as ET
|
||
|
||
from bs4 import BeautifulSoup
|
||
|
||
from . import config, net
|
||
from .dates import parse_published
|
||
from .model import Item
|
||
|
||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36",
|
||
"Accept-Language": "en,ru;q=0.8"}
|
||
|
||
|
||
def _txt(x):
|
||
if x is None:
|
||
return ""
|
||
return html.unescape(re.sub(r"<[^>]+>", " ", str(x))).strip()
|
||
|
||
|
||
def _item(src, kind, url, title, text="", author="", published="", extra=None):
|
||
ex = dict(extra or {})
|
||
ex["tags"] = list(src.tags)
|
||
t = _txt(title)
|
||
body = _txt(text)
|
||
if body and (not t or getattr(src, "adapter", "").startswith("tg_")) and sum(ch.isalpha() for ch in t) < 12:
|
||
cand = [ln for ln in body.splitlines() if sum(ch.isalpha() for ch in ln) >= 12]
|
||
if cand:
|
||
t = max(cand, key=len)[:140]
|
||
# CVE-каналы постят тело advisory без идентификатора в заголовке — выносим его вперёд
|
||
if "CVE-" not in t:
|
||
m = re.search(r"CVE-\d{4}-\d{4,7}", body or "")
|
||
if m:
|
||
t = "%s · %s" % (m.group(0), t[:120])
|
||
dt = parse_published(published)
|
||
return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000],
|
||
author=_txt(author)[:80], published=dt.isoformat(timespec="seconds") if dt else str(published or ""), source=src.name,
|
||
extra=ex)
|
||
|
||
|
||
# ---------------- rss ----------------
|
||
def _arxiv_api(src, limit):
|
||
"""arxiv RSS часто пустой -> официальный API (Atom), тот же формат."""
|
||
m = re.search(r"rss/([A-Za-z\-.]+)$", src.url)
|
||
cat = m.group(1) if m else "cs.AI"
|
||
url = ("https://export.arxiv.org/api/query?search_query=cat:" + cat +
|
||
"&sortBy=submittedDate&sortOrder=descending&max_results=" + str(max(limit, 5)))
|
||
r = net.get(url, headers=UA)
|
||
if r.status != 200 or "<entry" not in (r.text or ""):
|
||
return []
|
||
out = []
|
||
try:
|
||
root = ET.fromstring((r.text or "").encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return []
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
for e in root.findall(".//a:entry", ns)[:limit]:
|
||
title = " ".join((e.findtext("a:title", namespaces=ns) or "").split())
|
||
summ = (e.findtext("a:summary", namespaces=ns) or "").strip()
|
||
link = e.findtext("a:id", namespaces=ns) or ""
|
||
pub = e.findtext("a:published", namespaces=ns) or ""
|
||
auth = e.findtext("a:author/a:name", namespaces=ns) or ""
|
||
if title:
|
||
out.append(_item(src, "paper", str(link), " ".join(title.split()), summ, auth, pub,
|
||
{"cat": cat}))
|
||
return out
|
||
|
||
|
||
_FEED_CACHE = None
|
||
|
||
|
||
def _cache_url(src):
|
||
global _FEED_CACHE
|
||
if _FEED_CACHE is None:
|
||
try:
|
||
import pathlib as _pl
|
||
f = _pl.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
|
||
_FEED_CACHE = __import__("json").loads(f.read_text(encoding="utf-8")) if f.exists() else {}
|
||
except Exception:
|
||
_FEED_CACHE = {}
|
||
hit = _FEED_CACHE.get(src.name)
|
||
return hit.get("url") if isinstance(hit, dict) and hit.get("url") else src.url
|
||
|
||
|
||
def _rss(src, limit):
|
||
r = net.get(_cache_url(src), headers=UA)
|
||
if r.status in (429, 503):
|
||
import time
|
||
time.sleep(2.5)
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200 or not r.text.strip():
|
||
return _arxiv_api(src, limit) if "arxiv.org" in src.url else []
|
||
out = []
|
||
try:
|
||
root = ET.fromstring(r.text.encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return _rss_regex(src, r.text, limit)
|
||
ns = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/elements/1.1/"}
|
||
entries = root.findall(".//item") or root.findall(".//a:entry", ns)
|
||
for e in entries[:limit]:
|
||
def g(tag):
|
||
direct = e.findtext(tag, namespaces=ns) or ""
|
||
if direct or ":" in tag:
|
||
return direct
|
||
return e.findtext("a:" + tag, namespaces=ns) or ""
|
||
|
||
link = g("link")
|
||
if not link:
|
||
links = list(e.findall("a:link", ns)) + list(e.findall("link"))
|
||
le = next((el for el in links if el.get("rel", "alternate") == "alternate" and el.get("href")), None)
|
||
link = le.get("href", "") if le is not None else ""
|
||
if link:
|
||
link = urllib.parse.urljoin(src.url, link)
|
||
body = g("description") or g("summary") or g("content")
|
||
author = e.findtext("a:author/a:name", namespaces=ns) or g("author") or g("dc:creator")
|
||
out.append(_item(src, "entry", link, g("title"), body,
|
||
author, g("pubDate") or g("published") or g("updated")))
|
||
if not out and "arxiv.org" in src.url:
|
||
return _arxiv_api(src, limit)
|
||
return out
|
||
|
||
|
||
def _rss_regex(src, text, limit):
|
||
out = []
|
||
for m in re.finditer(r"<item>(.*?)</item>", text, re.S | re.I):
|
||
blob = m.group(1)
|
||
g = lambda t: (re.search(rf"<{t}[^>]*>(.*?)</{t}>", blob, re.S | re.I) or [None, ""])[1]
|
||
out.append(_item(src, "entry", _txt(g("link")), g("title"), g("description"), g("author"), g("pubDate")))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
# ---------------- json adapters ----------------
|
||
def _hn_top(src, limit):
|
||
ids = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for sid in ids[:limit]:
|
||
d = net.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", headers=UA, json_mode=True).json() or {}
|
||
if not d.get("title"):
|
||
continue
|
||
out.append(_item(src, "post", d.get("url") or f"https://news.ycombinator.com/item?id={sid}",
|
||
d.get("title"), d.get("text", ""), d.get("by", ""),
|
||
str(d.get("time", "")), {"points": d.get("score"), "comments": d.get("descendants"),
|
||
"hn": f"https://news.ycombinator.com/item?id={sid}"}))
|
||
return out
|
||
|
||
|
||
def _reddit_listing(src, limit):
|
||
"""Reddit без ключей: .json часто 403 с серверов, тогда .rss (Atom), тогда pullpush."""
|
||
sub_m = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||
rss_first = f"https://www.reddit.com/r/{sub_m.group(1)}/hot.rss" if sub_m else None
|
||
import time
|
||
children = []
|
||
if rss_first:
|
||
rr = net.get(rss_first, headers=UA)
|
||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||
return _reddit_atom(src, rr.text, limit)
|
||
if rr.status in (429, 503):
|
||
time.sleep(4)
|
||
rr = net.get(rss_first, headers=UA)
|
||
if rr.status == 200 and "<entry" in (rr.text or ""):
|
||
return _reddit_atom(src, rr.text, limit)
|
||
r = net.get(src.url, headers={**UA, "Accept": "application/json"}, json_mode=True)
|
||
out = []
|
||
if r.status == 200 and isinstance(r.json(), dict):
|
||
children = (((r.json() or {}).get("data") or {}).get("children") or [])
|
||
for ch in children[:limit]:
|
||
d = ch.get("data") or {}
|
||
if not d.get("title"):
|
||
continue
|
||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||
str(d.get("created_utc", "")),
|
||
{"ups": d.get("ups") or d.get("score"), "comments": d.get("num_comments")}))
|
||
if out:
|
||
return out
|
||
|
||
rss_url = (rss_first or re.sub(r"/\.json.*$", "/.rss", src.url))
|
||
r2 = net.get(rss_url, headers=UA) if not rss_first else None
|
||
if r2 is not None and r2.status == 200 and "<entry" in (r2.text or ""):
|
||
return _reddit_atom(src, r2.text, limit)
|
||
|
||
sub = re.search(r"reddit\.com/r/([^/]+)", src.url)
|
||
if sub:
|
||
alt = net.get(f"https://api.pullpush.io/reddit/search/submission/?subreddit={sub.group(1)}"
|
||
f"&size={limit}&sort=desc", headers=UA, json_mode=True)
|
||
for d in ((alt.json() or {}).get("data") or [])[:limit]:
|
||
if d.get("title"):
|
||
out.append(_item(src, "post", "https://www.reddit.com" + (d.get("permalink") or ""),
|
||
d.get("title"), d.get("selftext", ""), d.get("author", ""),
|
||
str(d.get("created_utc", "")), {"ups": d.get("score")}))
|
||
return out
|
||
|
||
|
||
def _reddit_atom(src, text, limit):
|
||
out = []
|
||
try:
|
||
root = ET.fromstring(text.encode("utf-8", "ignore"))
|
||
except Exception:
|
||
return out
|
||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||
for e in root.findall(".//a:entry", ns)[:limit]:
|
||
title = e.findtext("a:title", namespaces=ns) or ""
|
||
link_el = e.find("a:link", ns)
|
||
link = link_el.get("href") if link_el is not None else ""
|
||
content = e.findtext("a:content", namespaces=ns) or ""
|
||
author = e.findtext("a:author/a:name", namespaces=ns) or ""
|
||
upd = e.findtext("a:updated", namespaces=ns) or ""
|
||
if title:
|
||
out.append(_item(src, "post", str(link or ""), title, content, author, upd, {}))
|
||
return out
|
||
|
||
|
||
def _hf_generic(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("recentlyTrending") or d.get("models")
|
||
or d.get("spaces") or d.get("datasets") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
if "repoData" in x: # trending: {repoType, repoData:{...}}
|
||
x = {**x.get("repoData", {}), "repoType": x.get("repoType")}
|
||
rid = x.get("id") or x.get("repoId") or x.get("name") or ""
|
||
if not rid:
|
||
continue
|
||
kind = "space" if x.get("sdk") else ("dataset" if "dataset" in src.url else "model")
|
||
url = f"https://huggingface.co/{'datasets/' if kind == 'dataset' else ''}{rid}"
|
||
out.append(_item(src, kind, url, rid, json.dumps({k: x[k] for k in list(x)[:8]}, ensure_ascii=False)[:800],
|
||
"", x.get("lastModified", ""), {"likes": x.get("likes"), "downloads": x.get("downloads")}))
|
||
return out
|
||
|
||
|
||
def _hf_models(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
out.append(_item(src, "model", f"https://huggingface.co/{x.get('id', '')}", x.get("id", ""),
|
||
"", "", x.get("createdAt", ""),
|
||
{"likes": x.get("likes"), "downloads": x.get("downloads"), "pipeline": x.get("pipeline_tag")}))
|
||
return out
|
||
|
||
|
||
def _hf_daily_papers(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
p = x.get("paper") or x
|
||
pid = p.get("id") or ""
|
||
out.append(_item(src, "paper", f"https://huggingface.co/papers/{pid}", p.get("title", ""),
|
||
p.get("summary", ""), ", ".join(a.get("name", "") for a in (p.get("authors") or [])[:4]),
|
||
p.get("publishedAt", ""), {"upvotes": p.get("upvotes")}))
|
||
return out
|
||
|
||
|
||
def _hf_rows(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for row in (d.get("rows") or [])[:limit]:
|
||
r = row.get("row") or {}
|
||
name = r.get("fullname") or r.get("model") or r.get("Model") or ""
|
||
if not name:
|
||
continue
|
||
out.append(_item(src, "benchmark", f"https://huggingface.co/{name}", name, "",
|
||
"", "", {k: v for k, v in list(r.items())[:6] if isinstance(v, (int, float, str))}))
|
||
return out
|
||
|
||
|
||
def _openalex(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for w in (d.get("results") or [])[:limit]:
|
||
out.append(_item(src, "paper", w.get("doi") or w.get("id", ""), w.get("title", ""),
|
||
w.get("abstract_inverted_index") and "" or "", "", str(w.get("publication_date", "")),
|
||
{"cited": w.get("cited_by_count")}))
|
||
return out
|
||
|
||
|
||
def _s2(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
out = []
|
||
for p in (d.get("data") or [])[:limit]:
|
||
out.append(_item(src, "paper", p.get("url", ""), p.get("title", ""), p.get("abstract", ""),
|
||
", ".join(a.get("name", "") for a in (p.get("authors") or [])[:3]),
|
||
str(p.get("year", "")), {"citations": p.get("citationCount")}))
|
||
return out
|
||
|
||
|
||
def _samsung(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or d.get("result") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
out.append(_item(src, "blog", x.get("link") or x.get("url", "https://research.samsung.com/blog"),
|
||
x.get("title", ""), x.get("summary") or x.get("content", ""),
|
||
"", str(x.get("regDate") or x.get("date") or ""), {}))
|
||
return out
|
||
|
||
|
||
def _lg_ai(src, limit):
|
||
d = net.get(src.url, headers={**UA, "Referer": "https://www.lgresearch.ai/"}, json_mode=True).json() or {}
|
||
rows = d if isinstance(d, list) else (d.get("list") or d.get("data") or [])
|
||
out = []
|
||
for x in rows[:limit]:
|
||
if not isinstance(x, dict):
|
||
continue
|
||
out.append(_item(src, "blog", x.get("url") or "https://www.lgresearch.ai/blog", x.get("title", ""),
|
||
x.get("summary") or x.get("description", ""), "", str(x.get("regDate") or ""), {}))
|
||
return out
|
||
|
||
|
||
def _github_org(src, limit):
|
||
hdr = dict(UA)
|
||
tok = config.KEYS.get("GITHUB_TOKEN")
|
||
if tok:
|
||
hdr["Authorization"] = f"token {tok}"
|
||
url = f"https://api.github.com/orgs/{src.url}/repos?sort=updated&per_page={min(limit, 50)}"
|
||
d = net.get(url, headers=hdr, json_mode=True).json() or []
|
||
out = []
|
||
for x in (d if isinstance(d, list) else [])[:limit]:
|
||
out.append(_item(src, "repo", x.get("html_url", ""), x.get("full_name", ""),
|
||
x.get("description", ""), x.get("owner", {}).get("login", ""),
|
||
x.get("pushed_at", ""),
|
||
{"stars": x.get("stargazers_count"), "lang": x.get("language")}))
|
||
return out
|
||
|
||
|
||
def _discourse(src, limit):
|
||
d = net.get(src.url, headers=UA, json_mode=True).json() or {}
|
||
base = src.url.split("/latest.json")[0]
|
||
out = []
|
||
for t in ((d.get("topic_list") or {}).get("topics") or [])[:limit]:
|
||
out.append(_item(src, "thread", f"{base}/t/{t.get('slug', 'x')}/{t.get('id')}",
|
||
t.get("title", ""), t.get("excerpt", "")[:1500] if t.get("excerpt") else "",
|
||
"", t.get("created_at", ""), {"posts": t.get("posts_count"), "views": t.get("views")}))
|
||
return out
|
||
|
||
|
||
def _xenforo_latest(src, limit):
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
out = []
|
||
for a in soup.select("a[href*='/threads/']"):
|
||
title = a.get_text(" ", strip=True)
|
||
href = str(a.get("href", "") or "")
|
||
if len(title) < 12 or href.rstrip("/").endswith("/threads"):
|
||
continue
|
||
if not href.startswith("http"):
|
||
parts = urllib.parse.urlsplit(src.url)
|
||
href = "%s://%s%s" % (parts.scheme, parts.netloc, href)
|
||
out.append(_item(src, "thread", href, title))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
SERVICE_RE = re.compile(
|
||
r"^(channel (created|photo|name|description|username|deleted)|pinned|"
|
||
r"message pinned|channel photo (updated|removed)|joined|left|invite link)", re.I)
|
||
|
||
|
||
def _tg_web(src, limit):
|
||
"""Публичное веб-превью t.me/s/<канал>.
|
||
|
||
Отсекаем служебное (Channel created / photo updated / pinned) и пустыши:
|
||
раньше в базу уезжали заголовки "@upenbullet" и "Channel photo removed"."""
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
out = []
|
||
for msg in soup.select("div.tgme_widget_message_wrap")[-max(limit * 2, 20):]:
|
||
if msg.select_one("div.tgme_widget_message_service"):
|
||
continue
|
||
text_el = msg.select_one("div.tgme_widget_message_text")
|
||
if not text_el:
|
||
continue
|
||
text = text_el.get_text("\n", strip=True)
|
||
text = re.sub(r"\n{2,}", "\n", text).strip()
|
||
# лив-каналы пишут коротко («<домен> 2M users»), им порог ниже
|
||
min_len = 12 if "leak" in src.tags else 25
|
||
min_alpha = 8 if "leak" in src.tags else 15
|
||
if len(text) < min_len:
|
||
continue
|
||
if SERVICE_RE.match(text):
|
||
continue
|
||
if sum(ch.isalpha() for ch in text) < min_alpha: # только ссылки/эмодзи/цифры
|
||
continue
|
||
link_el = msg.select_one("a.tgme_widget_message_date")
|
||
link = str(link_el.get("href") or src.url) if link_el else src.url
|
||
dt = ""
|
||
t_el = msg.select_one("time[datetime]")
|
||
if t_el:
|
||
dt = str(t_el.get("datetime") or "")
|
||
views = ""
|
||
v_el = msg.select_one("span.tgme_widget_message_views")
|
||
if v_el:
|
||
views = v_el.get_text(" ", strip=True)
|
||
via = ""
|
||
vv = msg.select_one("span.tgme_widget_message_via")
|
||
if vv:
|
||
via = vv.get_text(" ", strip=True)
|
||
lines = [ln.strip() for ln in text.split("\n")
|
||
if ln.strip() and not ln.strip().startswith(("http", "t.co/"))]
|
||
lines = [ln for ln in lines if sum(ch.isalpha() for ch in ln) >= 12]
|
||
if not lines:
|
||
continue
|
||
first = [ln for ln in lines[:6] if len(ln) >= 20]
|
||
head = (first[0] if first else max(lines[:6], key=len)).strip(' ,.;:-——')
|
||
out.append(_item(src, "post", link, head[:140], text, via, dt[:10],
|
||
{"views": views, "handle": src.url.rsplit("/", 1)[-1], "lang": ""}))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
def _html_listing(src, limit):
|
||
r = net.get(src.url, headers=UA)
|
||
if r.status != 200:
|
||
return []
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
inc = src.extra.get("include_prefixes") or []
|
||
inc_c = src.extra.get("include_contains") or []
|
||
exc = set(src.extra.get("exclude_exact") or [])
|
||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||
seen, links = set(), []
|
||
for a in soup.find_all("a", href=True):
|
||
href = urllib.parse.urljoin(base, str(a["href"]))
|
||
path = urllib.parse.urlsplit(href).path
|
||
if path in exc or len(path) < 6:
|
||
continue
|
||
if inc and not any(path.startswith(p) for p in inc):
|
||
continue
|
||
if inc_c and not any(c in href for c in inc_c):
|
||
continue
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
links.append((href, a.get_text(" ", strip=True)))
|
||
if len(links) >= limit:
|
||
break
|
||
out = []
|
||
for href, title in links:
|
||
text = ""
|
||
if src.detail:
|
||
d = net.get(href, headers=UA)
|
||
if d.status == 200:
|
||
ds = BeautifulSoup(d.text, "lxml")
|
||
for bad in ds(["script", "style", "nav", "footer", "header"]):
|
||
bad.decompose()
|
||
paras = [p.get_text(" ", strip=True) for p in ds.find_all("p")]
|
||
text = " ".join(paras)[:4000]
|
||
if not title:
|
||
t = ds.find("h1") or ds.title
|
||
title = t.get_text(" ", strip=True) if t else href
|
||
out.append(_item(src, "entry", href, title or href, text))
|
||
return out
|
||
|
||
|
||
def _json_of(url, timeout=35):
|
||
"""JSON без зависимостей от конкретного клиента: net.get + json.loads."""
|
||
try:
|
||
r = net.get(url, json_mode=True, timeout=timeout)
|
||
except Exception:
|
||
return None
|
||
if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
|
||
return None
|
||
try:
|
||
return json.loads(r.text)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _generic_json(src, limit):
|
||
"""Любой JSON-эндпоинт: список объектов (+extra: rows/title_key/url_key/text_key/date_key/base/slug_key/deref)."""
|
||
d = _json_of(src.url)
|
||
ex = src.extra or {}
|
||
rows = []
|
||
if isinstance(d, list):
|
||
rows = d
|
||
elif isinstance(d, dict):
|
||
path = ex.get("rows")
|
||
if path:
|
||
cur = d
|
||
for part in str(path).split("."):
|
||
cur = cur.get(part) if isinstance(cur, dict) else None
|
||
if isinstance(cur, list):
|
||
rows = cur
|
||
if not rows:
|
||
for key in ("data", "items", "results", "papers", "models", "hits", "notes",
|
||
"posts", "records", "objects", "crates", "hackathons"):
|
||
v = d.get(key)
|
||
if isinstance(v, list):
|
||
rows = v
|
||
break
|
||
if not rows:
|
||
for v in d.values():
|
||
if isinstance(v, list) and v and isinstance(v[0], dict):
|
||
rows = v
|
||
break
|
||
out = []
|
||
deref = ex.get("deref")
|
||
base = str(ex.get("base") or "")
|
||
title_keys = ([ex["title_key"]] if ex.get("title_key") else []) + \
|
||
["title", "name", "paper_title", "full_name", "id"]
|
||
url_keys = ([ex["url_key"]] if ex.get("url_key") else []) + \
|
||
["url", "link", "html_url", "paper_url", "web_url"]
|
||
text_keys = ([ex["text_key"]] if ex.get("text_key") else []) + \
|
||
["abstract", "summary", "description", "body", "content"]
|
||
date_keys = ([ex["date_key"]] if ex.get("date_key") else []) + \
|
||
["published", "created", "created_at", "date", "published_at"]
|
||
for r in rows[:limit]:
|
||
if not isinstance(r, dict):
|
||
continue
|
||
if deref and isinstance(r.get(deref), dict):
|
||
r = r[deref]
|
||
title = ""
|
||
for k in title_keys:
|
||
v = r.get(k)
|
||
if isinstance(v, str) and v.strip():
|
||
title = v.strip()
|
||
break
|
||
url = ""
|
||
for k in url_keys:
|
||
v = r.get(k)
|
||
if isinstance(v, str) and v.strip():
|
||
url = v.strip()
|
||
break
|
||
if base and (not url or not url.startswith("http")):
|
||
slug = ""
|
||
if ex.get("slug_key"):
|
||
slug = str(r.get(ex["slug_key"]) or "")
|
||
url = base.rstrip("/") + "/" + (url or slug).lstrip("/")
|
||
if isinstance(url, str) and url and not url.startswith("http"):
|
||
if "github.com" in src.url:
|
||
url = "https://github.com/" + url.lstrip("/")
|
||
elif "huggingface.co" in src.url:
|
||
url = "https://huggingface.co/" + url.lstrip("/")
|
||
text = ""
|
||
for k in text_keys:
|
||
v = r.get(k)
|
||
if isinstance(v, str) and v.strip():
|
||
text = v
|
||
break
|
||
if isinstance(v, (dict, list)):
|
||
text = json.dumps(v, ensure_ascii=False)[:400]
|
||
break
|
||
published = ""
|
||
for k in date_keys:
|
||
v = r.get(k)
|
||
if isinstance(v, str) and v.strip():
|
||
published = v.strip()
|
||
break
|
||
if not title:
|
||
continue
|
||
out.append(_item(src, "entry", str(url or ""), _txt(title)[:180], _txt(text)[:800],
|
||
str(r.get("author") or r.get("owner") or r.get("by") or ""),
|
||
str(published)[:24]))
|
||
return out
|
||
|
||
|
||
def _bsky_search(src, limit):
|
||
"""Bluesky: публичный поиск постов без ключа (api.bsky.app)."""
|
||
d = _json_of(src.url)
|
||
out = []
|
||
for p in (d.get("posts") or [])[:limit]:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
rec = p.get("record") or {}
|
||
txt = _txt(str(rec.get("text") or ""))
|
||
author = str((p.get("author") or {}).get("handle") or "")
|
||
uri = str(p.get("uri") or "")
|
||
rkey = uri.rsplit("/", 1)[-1] if uri else ""
|
||
url = "https://bsky.app/profile/%s/post/%s" % (author, rkey) if author and rkey else uri
|
||
out.append(_item(src, "post", url, txt[:150], txt[:800], author,
|
||
str(rec.get("createdAt") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _mastodon_tag(src, limit):
|
||
"""Mastodon: публичная лента по тегу, без токена."""
|
||
d = _json_of(src.url)
|
||
rows = d if isinstance(d, list) else []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
if not isinstance(r, dict):
|
||
continue
|
||
txt = _txt(str(r.get("content") or ""))
|
||
out.append(_item(src, "post", str(r.get("url") or ""), txt[:150], txt[:800],
|
||
str((r.get("account") or {}).get("acct") or ""),
|
||
str(r.get("created_at") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _or_models(src, limit):
|
||
d = (_json_of(src.url) or {}).get("data", [])
|
||
out = []
|
||
for m in d[:limit]:
|
||
mid = m.get("id") or ""
|
||
pricing = m.get("pricing") or {}
|
||
price = None
|
||
try:
|
||
price = round(float(pricing.get("prompt", 0)) * 1_000_000, 3)
|
||
except Exception:
|
||
price = None
|
||
name = m.get("name") or mid
|
||
ctx = m.get("context_length") or 0
|
||
out.append(_item(src, "model", f"https://openrouter.ai/models/{mid}", f"{name}",
|
||
f"context {ctx} · prompt ${price}/1M", "", str(m.get("created") or ""),
|
||
extra={"price": price}))
|
||
return out
|
||
|
||
|
||
def _cisa_kev(src, limit):
|
||
d = _json_of(src.url, timeout=45) or {}
|
||
vulns = d.get("vulnerabilities", [])
|
||
vulns = sorted(vulns, key=lambda v: v.get("dateAdded") or "", reverse=True)
|
||
out = []
|
||
for v in vulns[:limit]:
|
||
cve = v.get("cveID") or ""
|
||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cve}",
|
||
f"{cve} · {v.get('vendorProject','')} {v.get('product','')}".strip(),
|
||
(v.get("shortDescription") or "")[:600],
|
||
v.get("vendorProject") or "", v.get("dateAdded") or ""))
|
||
return out
|
||
|
||
|
||
def _nvd(src, limit):
|
||
d = _json_of(src.url, timeout=45) or {}
|
||
out = []
|
||
for v in d.get("vulnerabilities", [])[:limit]:
|
||
c = v.get("cve", {})
|
||
cid = c.get("id") or ""
|
||
desc = ""
|
||
for dd in c.get("descriptions", []):
|
||
if dd.get("lang") == "en":
|
||
desc = dd.get("value") or ""
|
||
break
|
||
sev = ""
|
||
metrics = c.get("metrics", {})
|
||
for key in ("cvssMetricV31", "cvssMetricV40", "cvssMetricV2"):
|
||
if metrics.get(key):
|
||
try:
|
||
sev = str(metrics[key][0]["cvssData"].get("baseScore"))
|
||
except Exception:
|
||
sev = ""
|
||
break
|
||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cid}",
|
||
f"{cid}{(' · CVSS ' + sev) if sev else ''}", desc[:600],
|
||
"", c.get("published") or ""))
|
||
return out
|
||
|
||
|
||
|
||
def _biorxiv(src, limit):
|
||
import json as _j
|
||
d = _json_of(src.url)
|
||
rows = d.get("collection") if isinstance(d, dict) else d
|
||
out = []
|
||
for r in (rows or [])[:limit]:
|
||
title = r.get("title") or ""
|
||
doi = r.get("doi") or ""
|
||
if not title:
|
||
continue
|
||
ver = r.get("version") or "1"
|
||
host = "medrxiv" if "medrxiv" in src.url else "biorxiv"
|
||
url = "https://www.%s.org/content/%sv%s" % (host, doi, ver) if doi else src.url
|
||
out.append(_item(src, "paper", url, title, str(r.get("abstract") or r.get("category") or "")[:800],
|
||
str(r.get("authors") or ""), str(r.get("date") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _dblp(src, limit):
|
||
d = _json_of(src.url)
|
||
hits = (((d or {}).get("result") or {}).get("hits") or {}).get("hit") or []
|
||
out = []
|
||
for h in hits[:limit]:
|
||
info = h.get("info") or {}
|
||
title = info.get("title") or ""
|
||
if not title:
|
||
continue
|
||
auth = info.get("authors") or {}
|
||
if isinstance(auth, dict):
|
||
a = auth.get("author")
|
||
if isinstance(a, list):
|
||
auth = ", ".join(x.get("text", "") if isinstance(x, dict) else str(x) for x in a)
|
||
elif isinstance(a, dict):
|
||
auth = a.get("text", "")
|
||
out.append(_item(src, "paper", str(info.get("url") or ""), title, str(info.get("venue") or ""),
|
||
str(auth or ""), str(info.get("year") or "")))
|
||
return out
|
||
|
||
|
||
def _hal(src, limit):
|
||
d = _json_of(src.url)
|
||
docs = ((d or {}).get("response") or {}).get("docs") or []
|
||
out = []
|
||
for r in docs[:limit]:
|
||
t = r.get("title_s")
|
||
title = (t[0] if isinstance(t, list) and t else t) or ""
|
||
if not title:
|
||
continue
|
||
ab = r.get("abstract_s")
|
||
if isinstance(ab, list):
|
||
ab = ab[0] if ab else ""
|
||
out.append(_item(src, "paper", str(r.get("uri_s") or ""), title, str(ab or "")[:800], "",
|
||
str(r.get("producedDateY_i") or "")))
|
||
return out
|
||
|
||
|
||
def _zenodo(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = ((d or {}).get("hits") or {}).get("hits") or []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
md = r.get("metadata") or {}
|
||
title = md.get("title") or ""
|
||
if not title:
|
||
continue
|
||
url = (r.get("links") or {}).get("self_html") or md.get("doi_url") or ""
|
||
out.append(_item(src, "paper", str(url), title, str(md.get("description") or "")[:800],
|
||
str((md.get("creators") or [{}])[0].get("name") if md.get("creators") else ""),
|
||
str(md.get("publication_date") or "")))
|
||
return out
|
||
|
||
|
||
def _cve_circl(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = d if isinstance(d, list) else []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
cid = r.get("id") or ""
|
||
if not cid:
|
||
continue
|
||
out.append(_item(src, "cve", "https://nvd.nist.gov/vuln/detail/" + cid, cid + " — " + str(r.get("summary") or "")[:140],
|
||
str(r.get("summary") or "")[:800], "", str(r.get("Published") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _ghsa(src, limit):
|
||
d = _json_of(src.url)
|
||
rows = d if isinstance(d, list) else (d.get("advisories") or []) if isinstance(d, dict) else []
|
||
out = []
|
||
for r in rows[:limit]:
|
||
gid = r.get("ghsa_id") or ""
|
||
title = r.get("summary") or gid
|
||
if not title:
|
||
continue
|
||
sev = r.get("severity") or ""
|
||
out.append(_item(src, "advisory", str(r.get("html_url") or ""), str(title),
|
||
(sev + " | " + (r.get("description") or ""))[:800], "",
|
||
str(r.get("published_at") or "")[:24]))
|
||
return out
|
||
|
||
|
||
def _models_dev(src, limit):
|
||
d = _json_of(src.url)
|
||
out = []
|
||
if not isinstance(d, dict):
|
||
return out
|
||
for pid, prov in d.items():
|
||
if not isinstance(prov, dict):
|
||
continue
|
||
pname = prov.get("name") or pid
|
||
models = prov.get("models") or {}
|
||
if isinstance(models, dict):
|
||
iterable = models.items()
|
||
elif isinstance(models, list):
|
||
iterable = [(m.get("id", ""), m) for m in models if isinstance(m, dict)]
|
||
else:
|
||
continue
|
||
for mid, m in iterable:
|
||
if not isinstance(m, dict):
|
||
continue
|
||
name = m.get("name") or mid
|
||
out.append(_item(src, "model", "https://models.dev/#" + str(pid), "%s / %s" % (pname, name),
|
||
"ctx %s | in $%s | out $%s" % (m.get("limit", {}).get("context", "?"),
|
||
(m.get("cost") or {}).get("input", "?"),
|
||
(m.get("cost") or {}).get("output", "?")),
|
||
str(pid), ""))
|
||
if len(out) >= limit:
|
||
return out
|
||
return out
|
||
|
||
|
||
def _x_user(src, limit=12):
|
||
"""Лента X-аккаунта через api.fxtwitter.com (без ключей)."""
|
||
from .providers import x as xp
|
||
handle = (src.extra or {}).get("handle") or src.url.rsplit("/", 1)[-1]
|
||
return xp.profile(handle, limit=max(4, min(limit, 25)))
|
||
|
||
|
||
# ---------------- sitemap_recent: сайты без ленты, но со свежими url ----------------
|
||
def _sitemap_recent(src, limit):
|
||
"""sitemap.xml/sitemap_index.xml -> свежие url по lastmod.
|
||
|
||
extra: {"days": 14, "sitemaps": ["https://x/sitemap.xml"], "must": ["/blog/","/news/"]}
|
||
Даёт доступ к сайтам, где нет ни rss, ни списка статей в html.
|
||
"""
|
||
import datetime as _d
|
||
days = int((src.extra or {}).get("days") or 21)
|
||
cut = (_d.datetime.utcnow() - _d.timedelta(days=days)).strftime("%Y-%m-%d")
|
||
must = tuple((src.extra or {}).get("must") or ())
|
||
starts = list((src.extra or {}).get("sitemaps") or [])
|
||
if not starts:
|
||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||
starts = [base + "/sitemap.xml", base + "/sitemap_index.xml",
|
||
base + "/sitemap-index.xml", base + "/wp-sitemap.xml"]
|
||
urls, seen_maps = [], set()
|
||
|
||
def load(u, depth=0):
|
||
if u in seen_maps or depth > 2 or len(seen_maps) > 12:
|
||
return
|
||
seen_maps.add(u)
|
||
r = net.get(u, headers=UA)
|
||
if r.status != 200 or "<" not in (r.text or ""):
|
||
return
|
||
sx = BeautifulSoup(r.text, "xml")
|
||
for sm in sx.find_all("sitemap"):
|
||
loc = (sm.find("loc").get_text(strip=True) if sm.find("loc") else "")
|
||
lm = sm.find("lastmod").get_text(strip=True) if sm.find("lastmod") else ""
|
||
if loc and (not lm or lm[:10] >= cut):
|
||
load(loc, depth + 1)
|
||
for ue in sx.find_all("url"):
|
||
loc = ue.find("loc").get_text(strip=True) if ue.find("loc") else ""
|
||
lm = ue.find("lastmod").get_text(strip=True) if ue.find("lastmod") else ""
|
||
if not loc:
|
||
continue
|
||
if lm and lm[:10] < cut:
|
||
continue
|
||
if must and not any(m in loc for m in must):
|
||
continue
|
||
urls.append((loc, lm))
|
||
|
||
for u in starts:
|
||
try:
|
||
load(u)
|
||
except Exception:
|
||
continue
|
||
# свежие сверху, без подстраниц-дублей
|
||
urls.sort(key=lambda x: x[1] or "", reverse=True)
|
||
out, seen = [], set()
|
||
for loc, lm in urls:
|
||
k = loc.rstrip("/")
|
||
if k in seen:
|
||
continue
|
||
seen.add(k)
|
||
tail = urllib.parse.urlsplit(loc).path.strip("/").rsplit("/", 1)[-1]
|
||
title = urllib.parse.unquote(tail).replace("-", " ").replace("_", " ")[:160] or loc
|
||
out.append(_item(src, "page", loc, title, "", "", lm))
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
# ---------------- newapi_pricing: панели new-api -> бесплатные модели ----------------
|
||
def _newapi_pricing(src, limit):
|
||
"""Панель new-api: /api/pricing -> модели с нулевой ценой = бесплатный доступ.
|
||
|
||
Ищет не статьи про абузы, а сами раздачи: free-модели на живых панелях.
|
||
"""
|
||
base = src.url.rstrip("/")
|
||
js = None
|
||
# часть панелей отдаёт каталог только простому UA (полный Chrome-UA -> пустое тело)
|
||
base_hdr = dict(UA) if isinstance(UA, dict) else {"User-Agent": str(UA)}
|
||
for ua in (base_hdr.get("User-Agent", ""), "Mozilla/5.0", "curl/8.5.0"):
|
||
r = net.get(base + "/api/pricing", headers={"User-Agent": ua, "Accept": "application/json"})
|
||
if r.status != 200 or not (r.text or "").strip().startswith(("{", "[")):
|
||
continue
|
||
try:
|
||
js = json.loads(r.text)
|
||
break
|
||
except Exception:
|
||
continue
|
||
if js is None:
|
||
return []
|
||
data = js.get("data") if isinstance(js, dict) else None
|
||
if not isinstance(data, list):
|
||
return []
|
||
free, paid = [], 0
|
||
for m in data:
|
||
if not isinstance(m, dict):
|
||
continue
|
||
name = m.get("model_name") or m.get("model") or ""
|
||
ratio = m.get("model_ratio")
|
||
cratio = m.get("completion_ratio")
|
||
try:
|
||
zero = (float(ratio or 0) == 0) and (float(cratio or 0) == 0)
|
||
except Exception:
|
||
zero = False
|
||
if name and zero:
|
||
free.append(name)
|
||
elif name:
|
||
paid += 1
|
||
if not free:
|
||
return []
|
||
host = urllib.parse.urlsplit(base).netloc or base
|
||
title = "панель %s: %d бесплатных моделей" % (host, len(free))
|
||
body = ("Бесплатные модели (цена 0): " + ", ".join(free[:40])
|
||
+ ("\nвсего моделей на панели: %d" % (len(free) + paid)))
|
||
return [_item(src, "panel", base + "/api/pricing", title, body, "", "",
|
||
{"free_models": free[:60], "free_count": len(free), "paid_count": paid})]
|
||
|
||
|
||
def _github_search(src, limit):
|
||
"""GitHub search API: свежие репозитории по запросу (free api, jailbreak, autoreg)."""
|
||
q = (src.extra or {}).get("q") or "free api"
|
||
sort = (src.extra or {}).get("sort") or "updated"
|
||
url = ("https://api.github.com/search/repositories?q=" + urllib.parse.quote(str(q))
|
||
+ "&sort=" + str(sort) + "&order=desc&per_page=" + str(max(1, min(int(limit), 50))))
|
||
hdr = {"Accept": "application/vnd.github+json", "User-Agent": "omni-parse"}
|
||
tok = ((config.CONFIG.get("keys") or {}).get("github") or [])
|
||
if not tok:
|
||
single = config.KEYS.get("GITHUB_TOKEN")
|
||
tok = [single] if single else []
|
||
if isinstance(tok, list) and tok:
|
||
hdr["Authorization"] = "Bearer " + str(tok[0])
|
||
r = net.get(url, headers=hdr)
|
||
if r.status != 200:
|
||
return []
|
||
try:
|
||
js = json.loads(r.text)
|
||
except Exception:
|
||
return []
|
||
out = []
|
||
for it in (js.get("items") or [])[:limit]:
|
||
title = it.get("full_name") or it.get("name") or ""
|
||
desc = it.get("description") or ""
|
||
url_ = it.get("html_url") or ""
|
||
upd = (it.get("updated_at") or "")[:10]
|
||
out.append(_item(src, "github", url_, title, desc, upd, "", {}))
|
||
return out
|
||
|
||
|
||
def _hn_search(src, limit):
|
||
"""Hacker News (Algolia): поиск по свежим историям — бесплатные API, сливы, LLM-абузы."""
|
||
q = (src.extra or {}).get("q") or "free api"
|
||
url = ("https://hn.algolia.com/api/v1/search_by_date?query=" + urllib.parse.quote(str(q))
|
||
+ "&tags=story&hitsPerPage=" + str(max(1, min(int(limit), 50))))
|
||
r = net.get(url, headers={"User-Agent": "omni-parse", "Accept": "application/json"})
|
||
if r.status != 200:
|
||
return []
|
||
try:
|
||
js = json.loads(r.text)
|
||
except Exception:
|
||
return []
|
||
out = []
|
||
for h in (js.get("hits") or [])[:limit]:
|
||
title = h.get("title") or h.get("story_title") or ""
|
||
if not title:
|
||
continue
|
||
link = h.get("url") or ("https://news.ycombinator.com/item?id=%s" % h.get("objectID"))
|
||
ts = (h.get("created_at") or "")[:10]
|
||
body = (h.get("story_text") or "")[:400]
|
||
out.append(_item(src, "hn", link, title, body, ts, "", {}))
|
||
return out
|
||
|
||
|
||
ADAPTERS = {
|
||
"x_user": _x_user,
|
||
"rss": _rss,
|
||
"generic_json": _generic_json,
|
||
"or_models": _or_models,
|
||
"cisa_kev": _cisa_kev,
|
||
"nvd": _nvd,
|
||
"hn_top": _hn_top,
|
||
"reddit_listing": _reddit_listing,
|
||
"hf_trending": _hf_generic,
|
||
"hf_generic": _hf_generic,
|
||
"hf_models": _hf_models,
|
||
"hf_daily_papers": _hf_daily_papers,
|
||
"hf_rows": _hf_rows,
|
||
"openalex": _openalex,
|
||
"s2": _s2,
|
||
"samsung": _samsung,
|
||
"lg_ai": _lg_ai,
|
||
"github_org": _github_org,
|
||
"discourse": _discourse,
|
||
"xenforo_latest": _xenforo_latest,
|
||
"tg_web": _tg_web,
|
||
"html_listing": _html_listing,
|
||
"biorxiv": _biorxiv,
|
||
"dblp": _dblp,
|
||
"hal": _hal,
|
||
"zenodo": _zenodo,
|
||
"cve_circl": _cve_circl,
|
||
"ghsa": _ghsa,
|
||
"models_dev": _models_dev,
|
||
"sitemap_recent": _sitemap_recent,
|
||
"newapi_pricing": _newapi_pricing,
|
||
"github_search": _github_search,
|
||
"hn_search": _hn_search,
|
||
"bsky_search": _bsky_search,
|
||
"mastodon_tag": _mastodon_tag,
|
||
|
||
}
|
||
|
||
|
||
_CHALLENGE_TITLE = re.compile(
|
||
r"(?:just a moment|access denied|403 forbidden|404(?: not found)?|not found|"
|
||
r"verify (?:that )?you are human|checking your browser|"
|
||
r"attention required!?(?:\s*\|\s*cloudflare)?|please enable javascript)[.!…\s]*",
|
||
re.I,
|
||
)
|
||
|
||
|
||
def _valid_items(items):
|
||
"""Drop broken records and obvious challenge pages, not short real posts."""
|
||
out, seen = [], set()
|
||
for it in items:
|
||
if not isinstance(it, Item) or any(not isinstance(getattr(it, field, None), str)
|
||
for field in ("url", "title", "text")):
|
||
continue
|
||
try:
|
||
u = urllib.parse.urlsplit(it.url)
|
||
valid_url = u.scheme in ("http", "https") and bool(u.hostname)
|
||
except ValueError:
|
||
valid_url = False
|
||
if not valid_url or not (it.title.strip() or it.text.strip()):
|
||
continue
|
||
if _CHALLENGE_TITLE.fullmatch(it.title.strip()) and (
|
||
len(it.text.strip()) < 160 or re.search(
|
||
r"cloudflare|captcha|verify you are human|checking your browser|just a moment|attention required",
|
||
it.text, re.I)):
|
||
continue
|
||
if it.url in seen:
|
||
continue
|
||
seen.add(it.url)
|
||
out.append(it)
|
||
return out
|
||
|
||
|
||
def fetch(src, limit=None):
|
||
fn = ADAPTERS.get(src.adapter)
|
||
if not fn:
|
||
return []
|
||
n = limit or src.limit
|
||
try:
|
||
return _valid_items(fn(src, n) or [])
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co", "services.nvd.nist.gov")
|
||
|
||
|
||
def fetch_many(sources, limit=None, workers=8):
|
||
"""Параллельный обход, но медленные хосты (reddit/github/hf) — по очереди с паузой."""
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
slow = [s for s in sources if any(h in s.url for h in SLOW_HOSTS)]
|
||
fast = [s for s in sources if s not in slow]
|
||
items, report = [], []
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
futs = [(ex.submit(fetch, s, limit), s) for s in fast]
|
||
for f, s in futs:
|
||
got = f.result()
|
||
report.append((s.name, len(got)))
|
||
items.extend(got)
|
||
for s in slow:
|
||
got = fetch(s, limit)
|
||
if not got and "reddit.com" in s.url:
|
||
time.sleep(7)
|
||
got = fetch(s, limit)
|
||
report.append((s.name, len(got)))
|
||
items.extend(got)
|
||
delay = 6.5 if "services.nvd.nist.gov" in s.url else (3.5 if "reddit.com" in s.url else 1.0)
|
||
time.sleep(delay)
|
||
return items, report |