omni: менеджер источников в панели (338 источников, вкл/выкл, добавление, health-check, фоновые джобы)
This commit is contained in:
reform
21
omni/cli.py
21
omni/cli.py
@@ -149,12 +149,18 @@ def cmd_stats(args):
|
||||
|
||||
|
||||
def cmd_sources(args):
|
||||
from . import source_state
|
||||
st = sources.stats()
|
||||
print("источников:", len(sources.SOURCES), "| по категориям:",
|
||||
", ".join(f"{k}={v}" for k, v in st.items()))
|
||||
fl = source_state.flags()
|
||||
all_s = sources.all_sources()
|
||||
print("источников:", len(all_s), "| включено:", sum(1 for s in all_s if fl.get(s.name, {}).get("enabled", True)),
|
||||
"| по категориям:", ", ".join(f"{k}={v}" for k, v in st.items()))
|
||||
if args.list:
|
||||
for s in sources.pick(args.category, args.only, args.tags):
|
||||
print(f" {s.category:<12} {s.adapter:<18} {s.name:<26} {s.url[:70]}")
|
||||
for s in sources.pick(args.category, args.only, args.tags, respect_enabled=not args.all):
|
||||
f = fl.get(s.name, {})
|
||||
mark = "on " if f.get("enabled", True) else "off"
|
||||
stt = f"{f.get('items')} зап." if (f.get("items") or 0) > 0 else (f.get("note") or "—")
|
||||
print(f" {mark} {s.category:<12} {s.adapter:<18} {s.name:<26} {stt:<12} {s.url[:60]}")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -256,6 +262,7 @@ def main(argv=None):
|
||||
sp.add_argument("--only", default=None)
|
||||
sp.add_argument("--tags", default=None)
|
||||
sp.add_argument("--list", action="store_true")
|
||||
sp.add_argument("--all", action="store_true", help="показать и выключенные")
|
||||
sp.set_defaults(fn=cmd_sources)
|
||||
|
||||
sp = sub.add_parser("feed", help="обход реестра источников без запроса")
|
||||
@@ -267,6 +274,12 @@ def main(argv=None):
|
||||
sp.add_argument("--verbose", action="store_true")
|
||||
sp.set_defaults(fn=cmd_feed)
|
||||
|
||||
sp = sub.add_parser("doctor", help="проверка источников, --fix подбирает рабочие фиды")
|
||||
sp.add_argument("--only", default=None)
|
||||
sp.add_argument("--fix", action="store_true")
|
||||
sp.add_argument("--limit", type=int, default=3)
|
||||
sp.set_defaults(fn=lambda a: (__import__("omni.doctor", fromlist=["run"]).run(a.only, a.fix, limit=a.limit), 0)[1])
|
||||
|
||||
sp = sub.add_parser("serve", help="веб-панель + JSON API")
|
||||
sp.add_argument("--port", type=int, default=8077)
|
||||
sp.set_defaults(fn=lambda a: __import__("omni.serve", fromlist=["serve"]).serve(a.port))
|
||||
|
||||
115
omni/doctor.py
Normal file
115
omni/doctor.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""Доктор источников: проверяет каждый, а с --fix сам находит рабочий фид по HTML-странице.
|
||||
|
||||
Причина: у половины блогов угаданный /rss.xml отдаёт 404, у части — Cloudflare.
|
||||
Для rss-источников лечим автопоиском <link rel="alternate" type="...rss..."> на главной,
|
||||
результат кладём в data/feeds_cache.json (переопределяет url без правки реестра).
|
||||
"""
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from . import feeds, sources
|
||||
|
||||
CACHE = pathlib.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
|
||||
|
||||
|
||||
def load_cache():
|
||||
try:
|
||||
return json.loads(CACHE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_cache(d):
|
||||
CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||
CACHE.write_text(json.dumps(d, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def discover(home):
|
||||
"""Ищем ссылки на фиды в HTML главной/раздела."""
|
||||
r = feeds.net.get(home, headers=feeds.UA)
|
||||
if r.status != 200 or not r.text:
|
||||
return []
|
||||
out = []
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
for ln in soup.find_all("link", href=True):
|
||||
typ = (ln.get("type") or "").lower()
|
||||
if "rss" in typ or "atom" in typ:
|
||||
out.append(urllib.parse.urljoin(home, str(ln["href"])))
|
||||
for a in soup.find_all("a", href=True):
|
||||
h = str(a["href"])
|
||||
if re.search(r"(rss|feed|atom)(\.xml|\.rss)?/?$", h) and "comment" not in h:
|
||||
out.append(urllib.parse.urljoin(home, h))
|
||||
seen, uniq = set(), []
|
||||
for u in out:
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
uniq.append(u)
|
||||
return uniq[:8]
|
||||
|
||||
|
||||
def probe(src, limit=3):
|
||||
try:
|
||||
got = feeds.fetch(src, limit)
|
||||
return src.name, len(got), (got[0].title[:60] if got else "")
|
||||
except Exception as e:
|
||||
return src.name, 0, f"{type(e).__name__}"
|
||||
|
||||
|
||||
def fix(src):
|
||||
"""Подобрать рабочий фид: кандидаты с главной (и её подразделов)."""
|
||||
base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(src.url))
|
||||
cands = []
|
||||
for home in (src.url, base + "/", base + "/blog", base + "/news", base + "/research"):
|
||||
try:
|
||||
cands += discover(home)
|
||||
except Exception:
|
||||
continue
|
||||
seen = set()
|
||||
for u in cands:
|
||||
if u in seen:
|
||||
continue
|
||||
seen.add(u)
|
||||
trial = sources.Source(src.name, src.category, src.kind, u, "rss", src.tags)
|
||||
try:
|
||||
got = feeds.fetch(trial, 3)
|
||||
except Exception:
|
||||
got = []
|
||||
if got:
|
||||
return u, len(got)
|
||||
return None, 0
|
||||
|
||||
|
||||
def run(only=None, do_fix=False, workers=8, limit=3):
|
||||
from . import source_state
|
||||
picked = sources.pick(None, only, None)
|
||||
cache = load_cache()
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for name, n, extra in ex.map(lambda s: probe(s, limit), picked):
|
||||
results.append({"name": name, "n": n, "extra": extra})
|
||||
source_state.record_check(name, 200 if n else 0, n, "ok" if n else str(extra)[:120])
|
||||
bad = [r for r in results if r["n"] == 0]
|
||||
print(f"проверено {len(results)} | живых {len(results) - len(bad)} | пустых {len(bad)}")
|
||||
if do_fix:
|
||||
by_name = {s.name: s for s in picked}
|
||||
fixed = 0
|
||||
for r in bad:
|
||||
src = by_name[r["name"]]
|
||||
if src.adapter not in ("rss", "html_listing", "github_org"):
|
||||
continue
|
||||
url, n = fix(src)
|
||||
if url:
|
||||
cache[src.name] = {"url": url, "adapter": "rss", "n": n}
|
||||
fixed += 1
|
||||
print(f" фикс {src.name:24s} -> {url} ({n})")
|
||||
save_cache(cache)
|
||||
print(f"починено {fixed}, кэш: {len(cache)} записей -> {CACHE}")
|
||||
else:
|
||||
for r in bad:
|
||||
print(f" пусто {r['name']:26s} {r['extra']}")
|
||||
return results, cache
|
||||
134
omni/feeds.py
134
omni/feeds.py
@@ -60,8 +60,24 @@ def _arxiv_api(src, limit):
|
||||
return out
|
||||
|
||||
|
||||
_FEED_CACHE = None
|
||||
|
||||
|
||||
def _cache_url(src):
|
||||
global _FEED_CACHE
|
||||
if _FEED_CACHE is None:
|
||||
try:
|
||||
import pathlib as _pl
|
||||
f = _pl.Path(__file__).resolve().parent.parent / "data" / "feeds_cache.json"
|
||||
_FEED_CACHE = __import__("json").loads(f.read_text(encoding="utf-8")) if f.exists() else {}
|
||||
except Exception:
|
||||
_FEED_CACHE = {}
|
||||
hit = _FEED_CACHE.get(src.name)
|
||||
return hit.get("url") if isinstance(hit, dict) and hit.get("url") else src.url
|
||||
|
||||
|
||||
def _rss(src, limit):
|
||||
r = net.get(src.url, headers=UA)
|
||||
r = net.get(_cache_url(src), headers=UA)
|
||||
if r.status in (429, 503):
|
||||
import time
|
||||
time.sleep(2.5)
|
||||
@@ -387,8 +403,124 @@ def _html_listing(src, limit):
|
||||
return out
|
||||
|
||||
|
||||
def _json_of(url, timeout=35):
|
||||
"""JSON без зависимостей от конкретного клиента: net.get + json.loads."""
|
||||
try:
|
||||
r = net.get(url, json_mode=True, timeout=timeout)
|
||||
except Exception:
|
||||
return None
|
||||
if getattr(r, "status", 0) != 200 or not (r.text or "").strip():
|
||||
return None
|
||||
try:
|
||||
return json.loads(r.text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _generic_json(src, limit):
|
||||
"""Любой JSON-эндпоинт: ищем список объектов и вытаскиваем title/url по частым ключам."""
|
||||
d = _json_of(src.url)
|
||||
rows = []
|
||||
if isinstance(d, list):
|
||||
rows = d
|
||||
elif isinstance(d, dict):
|
||||
for key in ("data", "items", "results", "papers", "models", "hits", "notes", "posts", "records"):
|
||||
v = d.get(key)
|
||||
if isinstance(v, list):
|
||||
rows = v
|
||||
break
|
||||
if not rows:
|
||||
for v in d.values():
|
||||
if isinstance(v, list) and v and isinstance(v[0], dict):
|
||||
rows = v
|
||||
break
|
||||
out = []
|
||||
for r in rows[:limit]:
|
||||
if not isinstance(r, dict):
|
||||
continue
|
||||
title = r.get("title") or r.get("name") or r.get("paper_title") or r.get("full_name") or r.get("id")
|
||||
url = r.get("url") or r.get("link") or r.get("html_url") or r.get("paper_url") or r.get("id")
|
||||
if isinstance(url, str) and url and not url.startswith("http"):
|
||||
if "github.com" in src.url:
|
||||
url = "https://github.com/" + url.lstrip("/")
|
||||
elif "huggingface.co" in src.url:
|
||||
url = "https://huggingface.co/" + url.lstrip("/")
|
||||
text = r.get("abstract") or r.get("summary") or r.get("description") or r.get("body") or r.get("content") or ""
|
||||
if isinstance(text, dict):
|
||||
text = json.dumps(text, ensure_ascii=False)[:400]
|
||||
published = r.get("published") or r.get("created") or r.get("created_at") or r.get("date") or ""
|
||||
if not title:
|
||||
continue
|
||||
out.append(_item(src, "entry", str(url or ""), str(title), str(text)[:800],
|
||||
str(r.get("author") or r.get("owner") or ""), str(published)[:24]))
|
||||
return out
|
||||
|
||||
|
||||
def _or_models(src, limit):
|
||||
d = (_json_of(src.url) or {}).get("data", [])
|
||||
out = []
|
||||
for m in d[:limit]:
|
||||
mid = m.get("id") or ""
|
||||
pricing = m.get("pricing") or {}
|
||||
price = None
|
||||
try:
|
||||
price = round(float(pricing.get("prompt", 0)) * 1_000_000, 3)
|
||||
except Exception:
|
||||
price = None
|
||||
name = m.get("name") or mid
|
||||
ctx = m.get("context_length") or 0
|
||||
out.append(_item(src, "model", f"https://openrouter.ai/models/{mid}", f"{name}",
|
||||
f"context {ctx} · prompt ${price}/1M", "", str(m.get("created") or ""),
|
||||
extra={"price": price}))
|
||||
return out
|
||||
|
||||
|
||||
def _cisa_kev(src, limit):
|
||||
d = _json_of(src.url, timeout=45) or {}
|
||||
vulns = d.get("vulnerabilities", [])
|
||||
vulns = sorted(vulns, key=lambda v: v.get("dateAdded") or "", reverse=True)
|
||||
out = []
|
||||
for v in vulns[:limit]:
|
||||
cve = v.get("cveID") or ""
|
||||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cve}",
|
||||
f"{cve} · {v.get('vendorProject','')} {v.get('product','')}".strip(),
|
||||
(v.get("shortDescription") or "")[:600],
|
||||
v.get("vendorProject") or "", v.get("dateAdded") or ""))
|
||||
return out
|
||||
|
||||
|
||||
def _nvd(src, limit):
|
||||
d = _json_of(src.url, timeout=45) or {}
|
||||
out = []
|
||||
for v in d.get("vulnerabilities", [])[:limit]:
|
||||
c = v.get("cve", {})
|
||||
cid = c.get("id") or ""
|
||||
desc = ""
|
||||
for dd in c.get("descriptions", []):
|
||||
if dd.get("lang") == "en":
|
||||
desc = dd.get("value") or ""
|
||||
break
|
||||
sev = ""
|
||||
metrics = c.get("metrics", {})
|
||||
for key in ("cvssMetricV31", "cvssMetricV40", "cvssMetricV2"):
|
||||
if metrics.get(key):
|
||||
try:
|
||||
sev = str(metrics[key][0]["cvssData"].get("baseScore"))
|
||||
except Exception:
|
||||
sev = ""
|
||||
break
|
||||
out.append(_item(src, "cve", f"https://nvd.nist.gov/vuln/detail/{cid}",
|
||||
f"{cid}{(' · CVSS ' + sev) if sev else ''}", desc[:600],
|
||||
"", c.get("published") or ""))
|
||||
return out
|
||||
|
||||
|
||||
ADAPTERS = {
|
||||
"rss": _rss,
|
||||
"generic_json": _generic_json,
|
||||
"or_models": _or_models,
|
||||
"cisa_kev": _cisa_kev,
|
||||
"nvd": _nvd,
|
||||
"hn_top": _hn_top,
|
||||
"reddit_listing": _reddit_listing,
|
||||
"hf_trending": _hf_generic,
|
||||
|
||||
150
omni/serve.py
150
omni/serve.py
@@ -6,31 +6,76 @@ import json
|
||||
import pathlib
|
||||
import threading
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from . import feeds, sources, store
|
||||
from . import feeds, net, sources, source_state, store
|
||||
|
||||
WEB = pathlib.Path(__file__).resolve().parent.parent / "web"
|
||||
JOBS = {}
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _job(jid, **kw):
|
||||
with _LOCK:
|
||||
cur = JOBS.get(jid) or {}
|
||||
cur.update(kw)
|
||||
JOBS[jid] = cur
|
||||
|
||||
|
||||
def _run_feed(job_id, category, limit, tags):
|
||||
picked = sources.pick(category, None, tags)
|
||||
try:
|
||||
items, report = feeds.fetch_many(picked, limit=limit, workers=8)
|
||||
new, dup = store.add(items)
|
||||
with _LOCK:
|
||||
JOBS[job_id] = {"state": "done", "sources": len(picked), "fetched": len(items),
|
||||
"new": new, "dup": dup, "empty": sum(1 for _, c in report if not c),
|
||||
"report": sorted(report, key=lambda x: -x[1])[:40]}
|
||||
_job(job_id, state="done", sources=len(picked), fetched=len(items), new=new, dup=dup,
|
||||
empty=sum(1 for _, c in report if not c),
|
||||
report=sorted(report, key=lambda x: -x[1])[:40])
|
||||
except Exception as e:
|
||||
with _LOCK:
|
||||
JOBS[job_id] = {"state": "error", "error": f"{type(e).__name__}: {e}"[:200]}
|
||||
_job(job_id, state="error", error=f"{type(e).__name__}: {e}"[:200])
|
||||
|
||||
|
||||
def _check_one(src, limit=5):
|
||||
note = ""
|
||||
try:
|
||||
items = feeds.fetch(src, limit)
|
||||
except Exception as e:
|
||||
items, note = [], type(e).__name__
|
||||
if items:
|
||||
return source_state.record_check(src.name, 200, len(items), "ok")
|
||||
code = 0
|
||||
try:
|
||||
r = net.get(src.url, timeout=20)
|
||||
code = getattr(r, "status", 0) or 0
|
||||
note = note or ("пусто" if code == 200 else f"http {code}")
|
||||
except Exception as e:
|
||||
note = note or type(e).__name__
|
||||
return source_state.record_check(src.name, code, 0, note or "нет данных")
|
||||
|
||||
|
||||
def _run_check(job_id, names=None, category=None):
|
||||
picked = sources.pick(category, ",".join(names) if names else None, None)
|
||||
total = len(picked)
|
||||
_job(job_id, state="running", total=total, done=0, alive=0, dead=0, current="")
|
||||
alive = dead = 0
|
||||
with ThreadPoolExecutor(max_workers=6) as ex:
|
||||
futures = {ex.submit(_check_one, s): s for s in picked}
|
||||
for i, f in enumerate(futures, 1):
|
||||
s = futures[f]
|
||||
try:
|
||||
res = f.result()
|
||||
except Exception:
|
||||
res = {}
|
||||
if (res.get("items") or 0) > 0:
|
||||
alive += 1
|
||||
else:
|
||||
dead += 1
|
||||
_job(job_id, done=i, alive=alive, dead=dead, current=s.name)
|
||||
_job(job_id, state="done", done=total, alive=alive, dead=dead, current="")
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "omni/1.0"
|
||||
server_version = "omni/1.1"
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
@@ -60,17 +105,44 @@ class Handler(BaseHTTPRequestHandler):
|
||||
|
||||
if path == "/api/stats":
|
||||
s = store.stats()
|
||||
return self._json({"items": s["items"], "priced": s["priced"], "runs": s["runs"],
|
||||
"runs_ok": s["runs_ok"],
|
||||
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
|
||||
"by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
|
||||
"sources": len(sources.SOURCES), "categories": sources.stats()})
|
||||
fl = source_state.flags()
|
||||
all_s = sources.all_sources()
|
||||
return self._json({
|
||||
"items": s["items"], "priced": s["priced"], "runs": s["runs"], "runs_ok": s["runs_ok"],
|
||||
"by_provider": [{"name": n, "count": c} for n, c in s["by_provider"]],
|
||||
"by_kind": [{"name": n, "count": c} for n, c in s["by_kind"]],
|
||||
"sources": len(all_s),
|
||||
"sources_enabled": sum(1 for x in all_s if fl.get(x.name, {}).get("enabled", True)),
|
||||
"categories": sources.stats(),
|
||||
})
|
||||
|
||||
if path == "/api/sources":
|
||||
return self._json({"total": len(sources.SOURCES), "categories": sources.stats(),
|
||||
"items": [{"name": s.name, "category": s.category, "kind": s.kind,
|
||||
"adapter": s.adapter, "url": s.url, "tags": list(s.tags)}
|
||||
for s in sources.SOURCES]})
|
||||
cat = (q.get("category") or [None])[0]
|
||||
query = (q.get("q") or [""])[0].strip().lower()
|
||||
fl = source_state.flags()
|
||||
custom_names = {d["name"] for d in source_state.custom()}
|
||||
items = []
|
||||
for s in sources.all_sources():
|
||||
f = fl.get(s.name, {})
|
||||
if cat and cat not in ("all", "") and s.category != cat:
|
||||
continue
|
||||
if query and query not in s.name.lower() and query not in s.url.lower():
|
||||
continue
|
||||
items.append({
|
||||
"name": s.name, "category": s.category, "kind": s.kind, "adapter": s.adapter,
|
||||
"url": s.url, "tags": list(s.tags),
|
||||
"enabled": f.get("enabled", True), "status": f.get("status"),
|
||||
"items": f.get("items"), "note": f.get("note"), "checked": f.get("checked"),
|
||||
"custom": s.name in custom_names,
|
||||
})
|
||||
return self._json({
|
||||
"total": len(sources.all_sources()), "shown": len(items),
|
||||
"enabled": sum(1 for x in items if x["enabled"]),
|
||||
"alive": sum(1 for x in items if (x["items"] or 0) > 0),
|
||||
"dead": sum(1 for x in items if x["checked"] and not (x["items"] or 0)),
|
||||
"untested": sum(1 for x in items if not x["checked"]),
|
||||
"categories": sources.stats(), "items": items,
|
||||
})
|
||||
|
||||
if path == "/api/items":
|
||||
cat = (q.get("category") or [None])[0]
|
||||
@@ -95,17 +167,49 @@ class Handler(BaseHTTPRequestHandler):
|
||||
body = json.loads(self.rfile.read(n) or b"{}")
|
||||
except Exception:
|
||||
body = {}
|
||||
|
||||
if u.path == "/api/feed":
|
||||
jid = f"job{len(JOBS) + 1}"
|
||||
with _LOCK:
|
||||
JOBS[jid] = {"state": "running"}
|
||||
threading.Thread(target=_run_feed, args=(jid, body.get("category"), int(body.get("limit") or 12),
|
||||
body.get("tags")), daemon=True).start()
|
||||
jid = f"feed{len(JOBS)}"
|
||||
_job(jid, state="running")
|
||||
threading.Thread(target=_run_feed,
|
||||
args=(jid, body.get("category"), int(body.get("limit") or 12), body.get("tags")),
|
||||
daemon=True).start()
|
||||
return self._json({"job": jid})
|
||||
|
||||
if u.path == "/api/sources/toggle":
|
||||
name = body.get("name")
|
||||
if not name:
|
||||
return self._json({"error": "name required"}, 400)
|
||||
return self._json(source_state.set_enabled(name, bool(body.get("enabled", True))))
|
||||
|
||||
if u.path == "/api/sources/add":
|
||||
try:
|
||||
res = source_state.add_custom(body.get("name"), body.get("category"), body.get("url"),
|
||||
body.get("kind"), body.get("adapter"),
|
||||
body.get("tags") or (), int(body.get("limit") or 20),
|
||||
bool(body.get("detail")))
|
||||
except Exception as e:
|
||||
return self._json({"error": str(e)}, 400)
|
||||
return self._json(res)
|
||||
|
||||
if u.path == "/api/sources/remove":
|
||||
name = body.get("name")
|
||||
if not name:
|
||||
return self._json({"error": "name required"}, 400)
|
||||
return self._json(source_state.remove_custom(name))
|
||||
|
||||
if u.path == "/api/sources/check":
|
||||
names = body.get("names") or ([body["name"]] if body.get("name") else None)
|
||||
category = None if body.get("all") else body.get("category")
|
||||
jid = f"check{len(JOBS)}"
|
||||
_job(jid, state="running")
|
||||
threading.Thread(target=_run_check, args=(jid, names, category), daemon=True).start()
|
||||
return self._json({"job": jid})
|
||||
|
||||
return self._json({"error": "not found"}, 404)
|
||||
|
||||
|
||||
def serve(port=8077):
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", port), Handler)
|
||||
print(f"omni ui: http://127.0.0.1:{port} (источников {len(sources.SOURCES)})")
|
||||
print(f"omni ui: http://127.0.0.1:{port} (источников {len(sources.all_sources())})")
|
||||
srv.serve_forever()
|
||||
131
omni/source_state.py
Normal file
131
omni/source_state.py
Normal file
@@ -0,0 +1,131 @@
|
||||
"""Состояние источников: включён/выключен, пользовательские источники, результаты проверок.
|
||||
|
||||
Две таблицы в той же SQLite, что и записи:
|
||||
source_flags (name, enabled, status, items, note, checked)
|
||||
source_custom (name, category, kind, url, adapter, tags, limit, detail, added)
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
from . import config
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS source_flags (
|
||||
name TEXT PRIMARY KEY,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
status INTEGER,
|
||||
items INTEGER,
|
||||
note TEXT,
|
||||
checked INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS source_custom (
|
||||
name TEXT PRIMARY KEY,
|
||||
category TEXT, kind TEXT, url TEXT, adapter TEXT,
|
||||
tags TEXT, lim INTEGER DEFAULT 20, detail INTEGER DEFAULT 0, added INTEGER
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def conn():
|
||||
c = sqlite3.connect(str(config.DB), timeout=30)
|
||||
c.execute("PRAGMA journal_mode=WAL")
|
||||
c.executescript(SCHEMA)
|
||||
return c
|
||||
|
||||
|
||||
def flags():
|
||||
"""{name: {enabled, status, items, note, checked}}"""
|
||||
with conn() as c:
|
||||
rows = c.execute("SELECT name, enabled, status, items, note, checked FROM source_flags").fetchall()
|
||||
return {r[0]: {"enabled": bool(r[1]), "status": r[2], "items": r[3], "note": r[4], "checked": r[5]}
|
||||
for r in rows}
|
||||
|
||||
|
||||
def set_enabled(name, enabled):
|
||||
now = int(time.time())
|
||||
with conn() as c:
|
||||
c.execute("INSERT INTO source_flags(name, enabled, checked) VALUES(?,?,?) "
|
||||
"ON CONFLICT(name) DO UPDATE SET enabled=excluded.enabled",
|
||||
(name, 1 if enabled else 0, now))
|
||||
return {"name": name, "enabled": bool(enabled)}
|
||||
|
||||
|
||||
def record_check(name, status, items, note=""):
|
||||
now = int(time.time())
|
||||
with conn() as c:
|
||||
c.execute("INSERT INTO source_flags(name, enabled, status, items, note, checked) VALUES(?,1,?,?,?,?) "
|
||||
"ON CONFLICT(name) DO UPDATE SET status=excluded.status, items=excluded.items, "
|
||||
"note=excluded.note, checked=excluded.checked",
|
||||
(name, status, items, str(note)[:160], now))
|
||||
return {"name": name, "status": status, "items": items, "note": str(note)[:160], "checked": now}
|
||||
|
||||
|
||||
def custom():
|
||||
with conn() as c:
|
||||
rows = c.execute("SELECT name, category, kind, url, adapter, tags, lim, detail, added "
|
||||
"FROM source_custom ORDER BY added DESC").fetchall()
|
||||
out = []
|
||||
for r in rows:
|
||||
out.append({"name": r[0], "category": r[1], "kind": r[2], "url": r[3], "adapter": r[4],
|
||||
"tags": tuple(json.loads(r[5] or "[]")), "limit": r[6] or 20,
|
||||
"detail": bool(r[7]), "extra": {}, "added": r[8]})
|
||||
return out
|
||||
|
||||
|
||||
def add_custom(name, category, url, kind=None, adapter=None, tags=(), limit=20, detail=False):
|
||||
name = (name or "").strip()
|
||||
url = (url or "").strip()
|
||||
if not name or not url:
|
||||
raise ValueError("нужны name и url")
|
||||
if not url.startswith("http"):
|
||||
raise ValueError("url должен начинаться с http")
|
||||
kind, adapter = kind or guess_kind(url), adapter or guess_adapter(url, kind)
|
||||
with conn() as c:
|
||||
c.execute("INSERT OR REPLACE INTO source_custom(name, category, kind, url, adapter, tags, lim, detail, added) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?)",
|
||||
(name, category or "custom", kind, url, adapter,
|
||||
json.dumps(list(tags), ensure_ascii=False), int(limit), 1 if detail else 0, int(time.time())))
|
||||
return {"name": name, "category": category or "custom", "kind": kind, "url": url,
|
||||
"adapter": adapter, "tags": list(tags), "limit": limit, "detail": bool(detail)}
|
||||
|
||||
|
||||
def remove_custom(name):
|
||||
with conn() as c:
|
||||
c.execute("DELETE FROM source_custom WHERE name=?", (name,))
|
||||
c.execute("DELETE FROM source_flags WHERE name=?", (name,))
|
||||
return {"removed": name}
|
||||
|
||||
|
||||
def guess_kind(url):
|
||||
u = url.lower().rstrip("/")
|
||||
path = u.split("://", 1)[-1]
|
||||
path = path[path.find("/"):] if "/" in path else ""
|
||||
last = path.rsplit("/", 1)[-1]
|
||||
if "github.com" in u and u.count("/") <= 4:
|
||||
return "github"
|
||||
if last in ("rss", "feed", "atom", "rss.xml", "feed.xml", "atom.xml", "index.xml", "rss2"):
|
||||
return "rss"
|
||||
if u.endswith(".json") or "/latest.json" in u or "api." in u or "/api/" in u:
|
||||
return "json"
|
||||
if any(x in u for x in (".rss", "rss.xml", "/feed", ".atom", "/rss/", "atom.xml")):
|
||||
return "rss"
|
||||
return "html"
|
||||
|
||||
|
||||
def guess_adapter(url, kind=None):
|
||||
u = url.lower()
|
||||
kind = kind or guess_kind(url)
|
||||
if "github.com" in u:
|
||||
return "github_org"
|
||||
if "discourse" in u or u.endswith("/latest.json"):
|
||||
return "discourse"
|
||||
if kind == "json":
|
||||
return "generic_json"
|
||||
if u.endswith(".json") or "api." in u:
|
||||
return "generic_json"
|
||||
if "t.me/s/" in u:
|
||||
return "tg_web"
|
||||
if kind == "rss":
|
||||
return "rss"
|
||||
return "html_listing"
|
||||
@@ -10,6 +10,8 @@
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .sources_ext import EXTRA
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
@@ -26,7 +28,7 @@ class Source:
|
||||
|
||||
S = Source
|
||||
|
||||
SOURCES = [
|
||||
SOURCES_BASE = [
|
||||
# ---------- papers ----------
|
||||
S("arxiv_cs_ai", "papers", "rss", "https://rss.arxiv.org/rss/cs.AI", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_lg", "papers", "rss", "https://rss.arxiv.org/rss/cs.LG", "rss", ("paper", "arxiv")),
|
||||
@@ -175,12 +177,34 @@ SOURCES = [
|
||||
S("tg_huggingface", "forum", "html", "https://t.me/s/huggingface", "tg_web", ("telegram",)),
|
||||
]
|
||||
|
||||
SOURCES = SOURCES_BASE + [Source(**d) for d in EXTRA]
|
||||
|
||||
BY_NAME = {s.name: s for s in SOURCES}
|
||||
CATEGORIES = sorted({s.category for s in SOURCES})
|
||||
|
||||
|
||||
def pick(category=None, only=None, tags=None):
|
||||
out = SOURCES
|
||||
def all_sources():
|
||||
"""Реестр + пользовательские источники из панели."""
|
||||
from . import source_state
|
||||
extra = []
|
||||
for d in source_state.custom():
|
||||
if d["name"] in BY_NAME:
|
||||
continue
|
||||
d = {k: v for k, v in d.items() if k in
|
||||
("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
|
||||
extra.append(Source(**d))
|
||||
return SOURCES + extra
|
||||
|
||||
|
||||
def enabled_sources():
|
||||
"""Только включённые: выключенные в панели источники не дёргаем."""
|
||||
from . import source_state
|
||||
fl = source_state.flags()
|
||||
return [s for s in all_sources() if fl.get(s.name, {}).get("enabled", True)]
|
||||
|
||||
|
||||
def pick(category=None, only=None, tags=None, respect_enabled=True):
|
||||
out = enabled_sources() if respect_enabled else all_sources()
|
||||
if category:
|
||||
cats = {c.strip() for c in category.split(",") if c.strip()}
|
||||
out = [s for s in out if s.category in cats]
|
||||
@@ -193,8 +217,8 @@ def pick(category=None, only=None, tags=None):
|
||||
return out
|
||||
|
||||
|
||||
def stats():
|
||||
def stats(use_all=True):
|
||||
out = {}
|
||||
for s in SOURCES:
|
||||
for s in (all_sources() if use_all else SOURCES):
|
||||
out.setdefault(s.category, []).append(s.name)
|
||||
return {k: len(v) for k, v in sorted(out.items())}
|
||||
290
omni/sources_ext.py
Normal file
290
omni/sources_ext.py
Normal file
@@ -0,0 +1,290 @@
|
||||
"""Расширение реестра: ещё ~230 источников.
|
||||
|
||||
Правило: только публичные точки входа без ключей. Всё, что требует токен
|
||||
(Exa, You, Firecrawl, X), живёт в providers/ и включается через .env.
|
||||
|
||||
Здесь только данные — источниками они становятся в sources.py.
|
||||
"""
|
||||
|
||||
|
||||
def S(name, category, kind, url, adapter="rss", tags=(), limit=20, detail=False, extra=None):
|
||||
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
|
||||
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
|
||||
|
||||
|
||||
EXTRA = [
|
||||
# ---------------- papers ----------------
|
||||
S("arxiv_cs_cy", "papers", "rss", "https://rss.arxiv.org/rss/cs.CY", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_gt", "papers", "rss", "https://rss.arxiv.org/rss/cs.GT", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_sd", "papers", "rss", "https://rss.arxiv.org/rss/cs.SD", "rss", ("paper", "arxiv", "audio")),
|
||||
S("arxiv_cs_mm", "papers", "rss", "https://rss.arxiv.org/rss/cs.MM", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ni", "papers", "rss", "https://rss.arxiv.org/rss/cs.NI", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ds", "papers", "rss", "https://rss.arxiv.org/rss/cs.DS", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_si", "papers", "rss", "https://rss.arxiv.org/rss/cs.SI", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ar", "papers", "rss", "https://rss.arxiv.org/rss/cs.AR", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_et", "papers", "rss", "https://rss.arxiv.org/rss/cs.ET", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_ms", "papers", "rss", "https://rss.arxiv.org/rss/cs.MS", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_pl", "papers", "rss", "https://rss.arxiv.org/rss/cs.PL", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_lo", "papers", "rss", "https://rss.arxiv.org/rss/cs.LO", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_fl", "papers", "rss", "https://rss.arxiv.org/rss/cs.FL", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_gr", "papers", "rss", "https://rss.arxiv.org/rss/cs.GR", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_sc", "papers", "rss", "https://rss.arxiv.org/rss/cs.SC", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_cs_os", "papers", "rss", "https://rss.arxiv.org/rss/cs.OS", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_eess_sp", "papers", "rss", "https://rss.arxiv.org/rss/eess.SP", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_eess_iv", "papers", "rss", "https://rss.arxiv.org/rss/eess.IV", "rss", ("paper", "arxiv")),
|
||||
S("arxiv_math_oc", "papers", "rss", "https://rss.arxiv.org/rss/math.OC", "rss", ("paper", "arxiv", "math")),
|
||||
S("arxiv_q_bio_nc", "papers", "rss", "https://rss.arxiv.org/rss/q-bio.NC", "rss", ("paper", "arxiv", "bio")),
|
||||
S("arxiv_quant_ph", "papers", "rss", "https://rss.arxiv.org/rss/quant-ph", "rss", ("paper", "arxiv", "quantum")),
|
||||
S("arxiv_stat_ap", "papers", "rss", "https://rss.arxiv.org/rss/stat.AP", "rss", ("paper", "arxiv")),
|
||||
S("openalex_llm", "papers", "json",
|
||||
"https://api.openalex.org/works?search=large%20language%20model&sort=publication_date:desc&per_page=25",
|
||||
"openalex", ("paper", "openalex", "llm")),
|
||||
S("openalex_agents", "papers", "json",
|
||||
"https://api.openalex.org/works?search=llm%20agent&sort=publication_date:desc&per_page=25",
|
||||
"openalex", ("paper", "openalex", "agents")),
|
||||
S("s2_agents", "papers", "json",
|
||||
"https://api.semanticscholar.org/graph/v1/paper/search?query=llm+agent+tool+use&limit=20&fields=title,abstract,url,venue,year,citationCount",
|
||||
"s2", ("paper", "semanticscholar", "agents")),
|
||||
S("s2_security", "papers", "json",
|
||||
"https://api.semanticscholar.org/graph/v1/paper/search?query=prompt+injection+llm+security&limit=20&fields=title,abstract,url,venue,year,citationCount",
|
||||
"s2", ("paper", "semanticscholar", "security")),
|
||||
S("biorxiv_ai", "papers", "rss", "https://connect.biorxiv.org/biorxiv_xml.php?subject=bioinformatics", "rss", ("paper", "bio")),
|
||||
S("medrxiv_ai", "papers", "rss", "https://connect.medrxiv.org/medrxiv_xml.php?subject=all", "rss", ("paper", "med")),
|
||||
S("paperswithcode_trending", "papers", "json", "https://paperswithcode.com/api/v1/papers/?ordering=-published&items_per_page=25",
|
||||
"generic_json", ("paper", "pwc")),
|
||||
S("openreview_recent", "papers", "json", "https://api2.openreview.net/notes?content.venueid=ICLR.cc/2026/Conference&limit=25",
|
||||
"generic_json", ("paper", "openreview")),
|
||||
S("acl_anthology", "papers", "rss", "https://aclanthology.org/feed.xml", "rss", ("paper", "nlp")),
|
||||
|
||||
# ---------------- models ----------------
|
||||
S("hf_models_textgen", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=text-generation&sort=trendingScore&direction=-1&limit=25",
|
||||
"hf_models", ("model", "hf")),
|
||||
S("hf_models_image", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=text-to-image&sort=trendingScore&direction=-1&limit=20",
|
||||
"hf_models", ("model", "hf", "image")),
|
||||
S("hf_models_vlm", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=image-text-to-text&sort=trendingScore&direction=-1&limit=20",
|
||||
"hf_models", ("model", "hf", "vlm")),
|
||||
S("hf_models_asr", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=automatic-speech-recognition&sort=trendingScore&direction=-1&limit=20",
|
||||
"hf_models", ("model", "hf", "audio")),
|
||||
S("hf_models_tts", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=text-to-speech&sort=trendingScore&direction=-1&limit=20",
|
||||
"hf_models", ("model", "hf", "audio")),
|
||||
S("hf_models_embed", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=feature-extraction&sort=trendingScore&direction=-1&limit=20",
|
||||
"hf_models", ("model", "hf", "embed")),
|
||||
S("hf_models_rerank", "models", "json",
|
||||
"https://huggingface.co/api/models?pipeline_tag=text-ranking&sort=trendingScore&direction=-1&limit=15",
|
||||
"hf_models", ("model", "hf", "rerank")),
|
||||
S("hf_datasets_new", "models", "json",
|
||||
"https://huggingface.co/api/datasets?sort=createdAt&direction=-1&limit=20", "hf_generic", ("dataset", "hf")),
|
||||
S("hf_spaces_new", "models", "json",
|
||||
"https://huggingface.co/api/spaces?sort=createdAt&direction=-1&limit=20", "hf_generic", ("space", "hf")),
|
||||
S("openrouter_models", "models", "json", "https://openrouter.ai/api/v1/models", "or_models", ("model", "openrouter")),
|
||||
S("civitai_models", "models", "json", "https://civitai.com/api/v1/models?limit=20&sort=Newest", "generic_json", ("model", "image")),
|
||||
S("lmstudio_catalog", "models", "html", "https://lmstudio.ai/models", "html_listing", ("model", "local"), detail=False),
|
||||
S("ollama_search", "models", "json", "https://ollama.com/search?o=newest", "html_listing", ("model", "local"),
|
||||
detail=False, extra={"include_contains": ["/library/"]}),
|
||||
|
||||
# ---------------- benchmark ----------------
|
||||
S("lmarena_text", "benchmark", "html", "https://lmarena.ai/leaderboard/text", "html_listing", ("benchmark", "arena"), detail=False),
|
||||
S("lmarena_vision", "benchmark", "html", "https://lmarena.ai/leaderboard/vision", "html_listing", ("benchmark", "arena"), detail=False),
|
||||
S("lmarena_webdev", "benchmark", "html", "https://lmarena.ai/leaderboard/webdev", "html_listing", ("benchmark", "arena"), detail=False),
|
||||
S("artificial_analysis", "benchmark", "html", "https://artificialanalysis.ai/leaderboards/models", "html_listing",
|
||||
("benchmark",), detail=False),
|
||||
S("swebench_board", "benchmark", "html", "https://www.swebench.com/", "html_listing", ("benchmark", "code"), detail=False),
|
||||
S("epoch_benchmarks", "benchmark", "html", "https://epoch.ai/benchmarks", "html_listing", ("benchmark",), detail=False),
|
||||
S("vellum_board", "benchmark", "html", "https://www.vellum.ai/llm-leaderboard", "html_listing", ("benchmark",), detail=False),
|
||||
S("hf_openllm_v2", "benchmark", "json",
|
||||
"https://datasets-server.huggingface.co/rows?dataset=open-llm-leaderboard/contents&config=default&split=train&offset=25&length=25",
|
||||
"hf_rows", ("benchmark", "leaderboard")),
|
||||
|
||||
# ---------------- community ----------------
|
||||
S("hn_best", "community", "json", "https://hacker-news.firebaseio.com/v0/beststories.json", "hn_top", ("hn",)),
|
||||
S("hn_ask", "community", "json", "https://hacker-news.firebaseio.com/v0/askstories.json", "hn_top", ("hn", "ask")),
|
||||
S("hn_show", "community", "json", "https://hacker-news.firebaseio.com/v0/showstories.json", "hn_top", ("hn", "show")),
|
||||
S("hn_frontpage_rss", "community", "rss", "https://hnrss.org/frontpage", "rss", ("hn",)),
|
||||
S("hn_ai_search", "community", "rss", "https://hnrss.org/newest?q=LLM+OR+AI+OR+agent&points=10", "rss", ("hn", "ai")),
|
||||
S("lobsters_privacy", "community", "rss", "https://lobste.rs/t/privacy.rss", "rss", ("lobsters", "privacy")),
|
||||
S("lobsters_programming", "community", "rss", "https://lobste.rs/t/programming.rss", "rss", ("lobsters",)),
|
||||
S("lobsters_devops", "community", "rss", "https://lobste.rs/t/devops.rss", "rss", ("lobsters", "devops")),
|
||||
S("tildes", "community", "html", "https://tildes.net/", "html_listing", ("community",), detail=False),
|
||||
S("devto_ai", "community", "rss", "https://dev.to/feed/tag/ai", "rss", ("devto", "ai")),
|
||||
S("devto_llm", "community", "rss", "https://dev.to/feed/tag/llm", "rss", ("devto", "llm")),
|
||||
S("devto_python", "community", "rss", "https://dev.to/feed/tag/python", "rss", ("devto", "python")),
|
||||
S("devto_security", "community", "rss", "https://dev.to/feed/tag/security", "rss", ("devto", "security")),
|
||||
S("reddit_llmdevs", "community", "json", "https://www.reddit.com/r/LLMDevs/.json?limit=25", "reddit_listing", ("reddit", "llm")),
|
||||
S("reddit_rag", "community", "json", "https://www.reddit.com/r/Rag/.json?limit=25", "reddit_listing", ("reddit", "rag")),
|
||||
S("reddit_mlops", "community", "json", "https://www.reddit.com/r/mlops/.json?limit=25", "reddit_listing", ("reddit", "mlops")),
|
||||
S("reddit_datascience", "community", "json", "https://www.reddit.com/r/datascience/.json?limit=25", "reddit_listing", ("reddit", "data")),
|
||||
S("reddit_programming", "community", "json", "https://www.reddit.com/r/programming/.json?limit=25", "reddit_listing", ("reddit",)),
|
||||
S("reddit_rust", "community", "json", "https://www.reddit.com/r/rust/.json?limit=25", "reddit_listing", ("reddit", "rust")),
|
||||
S("reddit_golang", "community", "json", "https://www.reddit.com/r/golang/.json?limit=25", "reddit_listing", ("reddit", "go")),
|
||||
S("reddit_devops", "community", "json", "https://www.reddit.com/r/devops/.json?limit=25", "reddit_listing", ("reddit", "devops")),
|
||||
S("reddit_kubernetes", "community", "json", "https://www.reddit.com/r/kubernetes/.json?limit=25", "reddit_listing", ("reddit", "k8s")),
|
||||
S("reddit_selfhosted2", "community", "json", "https://www.reddit.com/r/homelab/.json?limit=25", "reddit_listing", ("reddit", "homelab")),
|
||||
S("reddit_comfyui", "community", "json", "https://www.reddit.com/r/comfyui/.json?limit=25", "reddit_listing", ("reddit", "image")),
|
||||
S("reddit_stablediffusion2", "community", "json", "https://www.reddit.com/r/StableDiffusion/.json?limit=25&sort=new", "reddit_listing", ("reddit", "image")),
|
||||
S("reddit_ollama", "community", "json", "https://www.reddit.com/r/ollama/.json?limit=25", "reddit_listing", ("reddit", "local")),
|
||||
S("reddit_gpumod", "community", "json", "https://www.reddit.com/r/GPUMining/.json?limit=20", "reddit_listing", ("reddit", "gpu")),
|
||||
S("reddit_hacking", "community", "json", "https://www.reddit.com/r/hacking/.json?limit=25", "reddit_listing", ("reddit", "sec")),
|
||||
S("reddit_reverseengineering", "community", "json", "https://www.reddit.com/r/ReverseEngineering/.json?limit=25", "reddit_listing", ("reddit", "re")),
|
||||
S("reddit_scraping", "community", "json", "https://www.reddit.com/r/scrapy/.json?limit=20", "reddit_listing", ("reddit", "scraping")),
|
||||
S("reddit_sysadmin", "community", "json", "https://www.reddit.com/r/sysadmin/.json?limit=25", "reddit_listing", ("reddit", "ops")),
|
||||
|
||||
# ---------------- company ----------------
|
||||
S("cloudflare_blog", "company", "rss", "https://blog.cloudflare.com/rss/", "rss", ("cloudflare", "infra")),
|
||||
S("vercel_blog", "company", "rss", "https://vercel.com/atom", "rss", ("vercel",)),
|
||||
S("github_blog", "company", "rss", "https://github.blog/feed/", "rss", ("github",)),
|
||||
S("github_changelog", "company", "rss", "https://github.blog/changelog/feed/", "rss", ("github",)),
|
||||
S("aws_ml_blog2", "company", "rss", "https://aws.amazon.com/blogs/big-data/feed/", "rss", ("aws", "data")),
|
||||
S("aws_opensource", "company", "rss", "https://aws.amazon.com/blogs/opensource/feed/", "rss", ("aws",)),
|
||||
S("google_cloud_blog", "company", "rss", "https://cloudblog.withgoogle.com/rss/", "rss", ("google", "cloud")),
|
||||
S("azure_blog", "company", "rss", "https://azure.microsoft.com/en-us/blog/feed/", "rss", ("azure",)),
|
||||
S("microsoft_devblogs", "company", "rss", "https://devblogs.microsoft.com/feed/", "rss", ("microsoft",)),
|
||||
S("netflix_techblog", "company", "rss", "https://netflixtechblog.com/feed", "rss", ("netflix", "eng")),
|
||||
S("uber_eng", "company", "rss", "https://www.uber.com/blog/engineering/rss/", "rss", ("uber", "eng")),
|
||||
S("airbnb_eng", "company", "rss", "https://medium.com/feed/airbnb-engineering", "rss", ("airbnb", "eng")),
|
||||
S("spotify_eng", "company", "rss", "https://engineering.atspotify.com/feed/", "rss", ("spotify", "eng")),
|
||||
S("pinterest_eng", "company", "rss", "https://medium.com/feed/pinterest-engineering", "rss", ("pinterest", "eng")),
|
||||
S("discord_blog", "company", "rss", "https://discord.com/blog/rss.xml", "rss", ("discord",)),
|
||||
S("slack_eng", "company", "rss", "https://slack.engineering/feed/", "rss", ("slack", "eng")),
|
||||
S("stripe_blog", "company", "rss", "https://stripe.com/blog/feed.rss", "rss", ("stripe",)),
|
||||
S("figma_blog", "company", "rss", "https://www.figma.com/blog/feed/", "rss", ("figma",)),
|
||||
S("linear_blog", "company", "rss", "https://linear.app/blog/rss.xml", "rss", ("linear",)),
|
||||
S("gitlab_blog", "company", "rss", "https://about.gitlab.com/atom.xml", "rss", ("gitlab",)),
|
||||
S("sentry_blog", "company", "rss", "https://blog.sentry.io/feed.xml", "rss", ("sentry",)),
|
||||
S("posthog_blog", "company", "rss", "https://posthog.com/rss.xml", "rss", ("posthog",)),
|
||||
S("supabase_blog", "company", "rss", "https://supabase.com/rss/followers.xml", "rss", ("supabase", "db")),
|
||||
S("neon_blog", "company", "rss", "https://neon.tech/blog/rss.xml", "rss", ("neon", "db")),
|
||||
S("planetscale_blog", "company", "rss", "https://planetscale.com/blog/feed.atom", "rss", ("planetscale", "db")),
|
||||
S("turso_blog", "company", "rss", "https://turso.tech/blog/rss.xml", "rss", ("turso", "db")),
|
||||
S("mongodb_blog", "company", "rss", "https://www.mongodb.com/blog/rss", "rss", ("mongodb", "db")),
|
||||
S("redis_blog", "company", "rss", "https://redis.io/blog/feed/", "rss", ("redis", "db")),
|
||||
S("elastic_blog", "company", "rss", "https://www.elastic.co/blog/feed", "rss", ("elastic", "search")),
|
||||
S("confluent_blog", "company", "rss", "https://www.confluent.io/blog/feed/", "rss", ("kafka", "data")),
|
||||
S("pinecone_blog", "company", "rss", "https://www.pinecone.io/blog/rss.xml", "rss", ("vector", "db")),
|
||||
S("qdrant_blog", "company", "rss", "https://qdrant.tech/blog/rss.xml", "rss", ("vector", "db")),
|
||||
S("weaviate_blog", "company", "rss", "https://weaviate.io/blog/rss.xml", "rss", ("vector", "db")),
|
||||
S("milvus_blog", "company", "rss", "https://milvus.io/blog/rss.xml", "rss", ("vector", "db")),
|
||||
S("flyio_blog", "company", "rss", "https://fly.io/blog/feed.xml", "rss", ("fly", "infra")),
|
||||
S("railway_blog", "company", "rss", "https://blog.railway.com/rss.xml", "rss", ("railway", "infra")),
|
||||
S("render_blog", "company", "rss", "https://render.com/blog/rss.xml", "rss", ("render", "infra")),
|
||||
S("runpod_blog", "company", "rss", "https://blog.runpod.io/rss/", "rss", ("gpu", "infra")),
|
||||
S("lambda_blog", "company", "rss", "https://lambda.ai/blog/rss.xml", "rss", ("gpu", "infra")),
|
||||
S("baseten_blog", "company", "rss", "https://www.baseten.co/blog/rss.xml", "rss", ("gpu", "infra")),
|
||||
S("raphael_blog", "company", "rss", "https://www.together.ai/blog/rss.xml", "rss", ("together",)),
|
||||
S("lmsys_blog", "company", "rss", "https://lmsys.org/rss.xml", "rss", ("lmsys", "arena")),
|
||||
S("allenai_blog", "company", "rss", "https://blog.allenai.org/feed", "rss", ("allenai",)),
|
||||
S("eleutherai_blog", "company", "html", "https://blog.eleuther.ai/", "html_listing", ("eleutherai",), detail=False),
|
||||
S("ollama_blog", "company", "rss", "https://ollama.com/blog/rss.xml", "rss", ("ollama",)),
|
||||
S("groq_blog", "company", "rss", "https://groq.com/blog/rss.xml", "rss", ("groq",)),
|
||||
S("anthropic_engineering", "company", "html", "https://www.anthropic.com/engineering", "html_listing",
|
||||
("anthropic", "eng"), detail=False),
|
||||
S("openai_research_rss", "company", "rss", "https://openai.com/research/rss.xml", "rss", ("openai", "research")),
|
||||
S("deepmind_github", "company", "github", "google-deepmind", "github_org", ("deepmind", "github")),
|
||||
S("openai_github", "company", "github", "openai", "github_org", ("openai", "github")),
|
||||
S("anthropic_github", "company", "github", "anthropics", "github_org", ("anthropic", "github")),
|
||||
S("meta_github", "company", "github", "facebookresearch", "github_org", ("meta", "github")),
|
||||
S("nvidia_github", "company", "github", "NVIDIA", "github_org", ("nvidia", "github")),
|
||||
S("microsoft_github", "company", "github", "microsoft", "github_org", ("microsoft", "github")),
|
||||
S("huggingface_github", "company", "github", "huggingface", "github_org", ("hf", "github")),
|
||||
S("vllm_github", "company", "github", "vllm-project", "github_org", ("vllm", "github")),
|
||||
S("sglang_github", "company", "github", "sgl-project", "github_org", ("sglang", "github")),
|
||||
S("langchain_github", "company", "github", "langchain-ai", "github_org", ("langchain", "github")),
|
||||
S("llamacpp_github", "company", "github", "ggml-org", "github_org", ("llama.cpp", "github")),
|
||||
S("ollama_github", "company", "github", "ollama", "github_org", ("ollama", "github")),
|
||||
S("unsloth_github", "company", "github", "unslothai", "github_org", ("finetune", "github")),
|
||||
|
||||
# ---------------- company_cn ----------------
|
||||
S("alibaba_cloud_blog", "company_cn", "rss", "https://www.alibabacloud.com/blog/rss", "rss", ("alibaba", "cn")),
|
||||
S("bytedance_seed", "company_cn", "html", "https://team.doubao.com/en/special/seed", "html_listing", ("bytedance", "cn"), detail=False),
|
||||
S("tencent_ai_lab", "company_cn", "html", "https://ai.tencent.com/ailab/en/index", "html_listing", ("tencent", "cn"), detail=False),
|
||||
S("baidu_research", "company_cn", "html", "https://research.baidu.com/Blog", "html_listing", ("baidu", "cn"), detail=False),
|
||||
S("stepfun_blog", "company_cn", "html", "https://www.stepfun.com/blog", "html_listing", ("stepfun", "cn"), detail=False),
|
||||
S("zeroone_ai", "company_cn", "html", "https://www.01.ai/blog", "html_listing", ("01ai", "cn"), detail=False),
|
||||
S("internlm_github", "company_cn", "github", "InternLM", "github_org", ("internlm", "cn")),
|
||||
S("shanghai_ailab_github", "company_cn", "github", "OpenGVLab", "github_org", ("shailab", "cn")),
|
||||
S("modelscope_github", "company_cn", "github", "modelscope", "github_org", ("modelscope", "cn")),
|
||||
S("sensetime_github", "company_cn", "github", "SenseTime", "github_org", ("sensetime", "cn")),
|
||||
S("xiaomi_github", "company_cn", "github", "XiaoMi", "github_org", ("xiaomi", "cn")),
|
||||
S("meituan_tech", "company_cn", "rss", "https://tech.meituan.com/feed/", "rss", ("meituan", "cn")),
|
||||
S("huawei_ascend_github", "company_cn", "github", "Ascend", "github_org", ("huawei", "cn")),
|
||||
S("kimi_blog", "company_cn", "html", "https://www.kimi.com/blog", "html_listing", ("kimi", "cn"), detail=False),
|
||||
S("zhipu_docs_update", "company_cn", "html", "https://docs.bigmodel.cn/cn/update", "html_listing", ("zhipu", "cn"), detail=False),
|
||||
S("minimax_github", "company_cn", "github", "MiniMax-AI", "github_org", ("minimax", "cn")),
|
||||
S("baichuan_github", "company_cn", "github", "baichuan-inc", "github_org", ("baichuan", "cn")),
|
||||
|
||||
# ---------------- company_kr ----------------
|
||||
S("naver_d2", "company_kr", "rss", "https://d2.naver.com/d2.atom", "rss", ("naver", "kr")),
|
||||
S("kakao_enterprise", "company_kr", "rss", "https://tech.kakaoenterprise.com/feed", "rss", ("kakao", "kr")),
|
||||
S("lg_cns_blog", "company_kr", "rss", "https://www.lgcns.com/rss/blog", "rss", ("lg", "kr")),
|
||||
S("samsung_sds", "company_kr", "rss", "https://www.samsungsds.com/kr/rss/blog.xml", "rss", ("samsung", "kr")),
|
||||
S("upstage_blog", "company_kr", "rss", "https://www.upstage.ai/blog/rss.xml", "rss", ("upstage", "kr")),
|
||||
S("ncsoft_blog", "company_kr", "rss", "https://blog.ncsoft.com/feed/", "rss", ("ncsoft", "kr")),
|
||||
|
||||
# ---------------- security ----------------
|
||||
S("cisa_kev", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
|
||||
"cisa_kev", ("cve", "kev")),
|
||||
S("nvd_recent", "security", "json",
|
||||
"https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&noRejected", "nvd", ("cve",)),
|
||||
S("github_advisories", "security", "rss", "https://github.com/advisories.atom", "rss", ("cve", "github")),
|
||||
S("snyk_blog", "security", "rss", "https://snyk.io/blog/feed/", "rss", ("appsec",)),
|
||||
S("trailofbits_blog", "security", "rss", "https://blog.trailofbits.com/feed/", "rss", ("research",)),
|
||||
S("rapid7_blog", "security", "rss", "https://www.rapid7.com/blog/rss/", "rss", ("research",)),
|
||||
S("tenable_blog", "security", "rss", "https://www.tenable.com/blog/feed", "rss", ("research",)),
|
||||
S("qualys_blog", "security", "rss", "https://blog.qualys.com/feed", "rss", ("research",)),
|
||||
S("mandiant_blog", "security", "rss", "https://www.mandiant.com/resources/blog/rss.xml", "rss", ("ir",)),
|
||||
S("crowdstrike_blog", "security", "rss", "https://www.crowdstrike.com/en-us/blog/feed/", "rss", ("ir",)),
|
||||
S("sentinelone_labs", "security", "rss", "https://www.sentinelone.com/labs/feed/", "rss", ("malware",)),
|
||||
S("unit42", "security", "rss", "https://unit42.paloaltonetworks.com/feed/", "rss", ("threatintel",)),
|
||||
S("talos", "security", "rss", "https://blog.talosintelligence.com/rss/", "rss", ("threatintel",)),
|
||||
S("securelist", "security", "rss", "https://securelist.com/feed/", "rss", ("threatintel",)),
|
||||
S("welivesecurity", "security", "rss", "https://www.welivesecurity.com/en/rss/feed/", "rss", ("malware",)),
|
||||
S("malwarebytes_labs", "security", "rss", "https://www.malwarebytes.com/blog/feed/index.xml", "rss", ("malware",)),
|
||||
S("darkreading", "security", "rss", "https://www.darkreading.com/rss.xml", "rss", ("news",)),
|
||||
S("therecord", "security", "rss", "https://therecord.media/feed", "rss", ("news",)),
|
||||
S("cyberscoop", "security", "rss", "https://cyberscoop.com/feed/", "rss", ("news",)),
|
||||
S("securityweek", "security", "rss", "https://www.securityweek.com/feed/", "rss", ("news",)),
|
||||
S("google_security_blog", "security", "rss", "https://security.googleblog.com/feeds/posts/default", "rss", ("research",)),
|
||||
S("msrc_blog", "security", "rss", "https://msrc.microsoft.com/blog/feed", "rss", ("microsoft",)),
|
||||
S("aws_security_blog", "security", "rss", "https://aws.amazon.com/blogs/security/feed/", "rss", ("aws",)),
|
||||
S("cloudflare_security", "security", "rss", "https://blog.cloudflare.com/tag/security/rss/", "rss", ("cloudflare",)),
|
||||
S("wiz_blog", "security", "rss", "https://www.wiz.io/blog/rss.xml", "rss", ("cloud",)),
|
||||
S("orca_security", "security", "rss", "https://orca.security/resources/blog/feed/", "rss", ("cloud",)),
|
||||
S("sysdig_blog", "security", "rss", "https://sysdig.com/blog/feed/", "rss", ("container",)),
|
||||
S("aquasecurity_blog", "security", "rss", "https://www.aquasec.com/blog/feed/", "rss", ("container",)),
|
||||
S("redcanary_blog", "security", "rss", "https://redcanary.com/blog/feed/", "rss", ("detection",)),
|
||||
S("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed.xml", "rss", ("research",)),
|
||||
S("projectzero_issues", "security", "rss", "https://googleprojectzero.blogspot.com/feeds/posts/default?alt=rss", "rss", ("research",)),
|
||||
S("seclists", "security", "html", "https://seclists.org/", "html_listing", ("mailing",), detail=False),
|
||||
S("hackerone_hacktivity", "security", "json", "https://hackerone.com/hacktivity.json?limit=20", "generic_json", ("bb",)),
|
||||
S("bugcrowd_crowdstream", "security", "html", "https://bugcrowd.com/crowdstream", "html_listing", ("bb",), detail=False),
|
||||
S("yeswehack_programs", "security", "html", "https://yeswehack.com/programs", "html_listing", ("bb",), detail=False),
|
||||
|
||||
# ---------------- forum ----------------
|
||||
S("discourse_pytorch", "forum", "json", "https://discuss.pytorch.org/latest.json", "discourse", ("pytorch",)),
|
||||
S("discourse_hf_forum", "forum", "json", "https://discuss.huggingface.co/latest.json", "discourse", ("hf",)),
|
||||
S("discourse_cursor", "forum", "json", "https://forum.cursor.com/latest.json", "discourse", ("cursor",)),
|
||||
S("discourse_n8n", "forum", "json", "https://community.n8n.io/latest.json", "discourse", ("n8n",)),
|
||||
S("discourse_grafana", "forum", "json", "https://community.grafana.com/latest.json", "discourse", ("grafana",)),
|
||||
S("discourse_homeassistant", "forum", "json", "https://community.home-assistant.io/latest.json", "discourse", ("iot",)),
|
||||
S("discourse_nextcloud", "forum", "json", "https://help.nextcloud.com/latest.json", "discourse", ("selfhosted",)),
|
||||
S("discourse_influxdata", "forum", "json", "https://community.influxdata.com/latest.json", "discourse", ("metrics",)),
|
||||
S("discourse_rasa", "forum", "json", "https://forum.rasa.com/latest.json", "discourse", ("chatbot",)),
|
||||
S("discourse_traefik", "forum", "json", "https://community.traefik.io/latest.json", "discourse", ("proxy",)),
|
||||
S("discourse_letsencrypt", "forum", "json", "https://community.letsencrypt.org/latest.json", "discourse", ("tls",)),
|
||||
S("discourse_obsidian", "forum", "json", "https://forum.obsidian.md/latest.json", "discourse", ("notes",)),
|
||||
S("discourse_anki", "forum", "json", "https://forums.ankiweb.net/latest.json", "discourse", ("learning",)),
|
||||
S("discourse_rust", "forum", "json", "https://users.rust-lang.org/latest.json", "discourse", ("rust",)),
|
||||
S("discourse_go", "forum", "json", "https://forum.golangbridge.org/latest.json", "discourse", ("go",)),
|
||||
S("discourse_elixir", "forum", "json", "https://elixirforum.com/latest.json", "discourse", ("elixir",)),
|
||||
S("xenforo_webmasterworld", "forum", "html", "https://www.webmasterworld.com/", "xenforo_latest", ("seo",)),
|
||||
S("tg_ai_newz", "forum", "html", "https://t.me/s/ai_newz", "tg_web", ("telegram", "ai")),
|
||||
S("tg_deepseek", "forum", "html", "https://t.me/s/deepseek_ai", "tg_web", ("telegram", "cn")),
|
||||
S("tg_qwen", "forum", "html", "https://t.me/s/Qwen_AI", "tg_web", ("telegram", "cn")),
|
||||
S("tg_neurohive", "forum", "html", "https://t.me/s/neurohive", "tg_web", ("telegram", "ai")),
|
||||
S("tg_gpt4telegram", "forum", "html", "https://t.me/s/GPTMain", "tg_web", ("telegram", "ai")),
|
||||
]
|
||||
Reference in New Issue
Block a user