diff --git a/.gitignore b/.gitignore index 55f48d2..69734fa 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data/ __pycache__/ *.pyc .bot_token +*.session* +.local/ diff --git a/omni/abuse.py b/omni/abuse.py index 3cb7308..45aa275 100644 --- a/omni/abuse.py +++ b/omni/abuse.py @@ -141,6 +141,13 @@ def _qualifies(source, title, text, tags): sc -= 26 if not hits: return 0, [] + src0 = str(source or "") + # new-api панели: «N бесплатных моделей (цена 0)» — это готовая абуз-поверхность + if src0.startswith("pn_"): + m = re.search(r"(\d{1,4})\s*бесплатн\w*\s*модел", blob, re.I) + if m: + cnt = min(int(m.group(1)), 500) + return 40 + min(cnt // 10, 30), ["freeapi"] raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob)) src = str(source or "") known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_")) diff --git a/omni/dates.py b/omni/dates.py new file mode 100644 index 0000000..d0d390d --- /dev/null +++ b/omni/dates.py @@ -0,0 +1,33 @@ +"""Publication dates from RSS/Atom/APIs, normalized to aware UTC datetimes.""" +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +import re + + +def parse_published(value): + """Accept Unix seconds/milliseconds, ISO 8601 or RFC 2822; invalid -> None. + + Dates without a timezone are interpreted as UTC, not the host's local zone. + """ + if value is None or not str(value).strip(): + return None + text = str(value).strip() + try: + if re.fullmatch(r"\d{4}", text): + return None # A bare year is not a Unix timestamp. + if re.fullmatch(r"\d{8}", text): + return datetime.strptime(text, "%Y%m%d").replace(tzinfo=timezone.utc) + if re.fullmatch(r"-?\d+(?:\.\d+)?", text): + stamp = float(text) + if abs(stamp) >= 10**12: + stamp /= 1000 + return datetime.fromtimestamp(stamp, timezone.utc) + try: + dt = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + dt = parsedate_to_datetime(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + except (ValueError, TypeError, OverflowError, OSError): + return None diff --git a/omni/feeds.py b/omni/feeds.py index a9b52dd..ae83c4d 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET from bs4 import BeautifulSoup from . import config, net +from .dates import parse_published from .model import Item UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36", @@ -38,8 +39,9 @@ def _item(src, kind, url, title, text="", author="", published="", extra=None): m = re.search(r"CVE-\d{4}-\d{4,7}", body or "") if m: t = "%s · %s" % (m.group(0), t[:120]) + dt = parse_published(published) return Item(src.name, kind, url=str(url or ""), title=t[:300], text=body[:6000], - author=_txt(author)[:80], published=str(published or ""), source=src.name, + author=_txt(author)[:80], published=dt.isoformat(timespec="seconds") if dt else str(published or ""), source=src.name, extra=ex) @@ -100,17 +102,26 @@ def _rss(src, limit): root = ET.fromstring(r.text.encode("utf-8", "ignore")) except Exception: return _rss_regex(src, r.text, limit) - ns = {"a": "http://www.w3.org/2005/Atom"} + ns = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/elements/1.1/"} entries = root.findall(".//item") or root.findall(".//a:entry", ns) for e in entries[:limit]: - g = lambda tag: (e.findtext(tag) or e.findtext("a:" + tag, namespaces=ns) or "") + def g(tag): + direct = e.findtext(tag, namespaces=ns) or "" + if direct or ":" in tag: + return direct + return e.findtext("a:" + tag, namespaces=ns) or "" + link = g("link") if not link: - le = e.find("a:link", ns) or e.find("link") - link = le.get("href") if le is not None and le.get("href") else "" + links = list(e.findall("a:link", ns)) + list(e.findall("link")) + le = next((el for el in links if el.get("rel", "alternate") == "alternate" and el.get("href")), None) + link = le.get("href", "") if le is not None else "" + if link: + link = urllib.parse.urljoin(src.url, link) body = g("description") or g("summary") or g("content") + author = e.findtext("a:author/a:name", namespaces=ns) or g("author") or g("dc:creator") out.append(_item(src, "entry", link, g("title"), body, - g("author") or g("dc:creator"), g("pubDate") or g("published") or g("updated"))) + author, g("pubDate") or g("published") or g("updated"))) if not out and "arxiv.org" in src.url: return _arxiv_api(src, limit) return out @@ -910,18 +921,52 @@ ADAPTERS = { } +_CHALLENGE_TITLE = re.compile( + r"(?:just a moment|access denied|403 forbidden|404(?: not found)?|not found|" + r"verify (?:that )?you are human|checking your browser|" + r"attention required!?(?:\s*\|\s*cloudflare)?|please enable javascript)[.!…\s]*", + re.I, +) + + +def _valid_items(items): + """Drop broken records and obvious challenge pages, not short real posts.""" + out, seen = [], set() + for it in items: + if not isinstance(it, Item) or any(not isinstance(getattr(it, field, None), str) + for field in ("url", "title", "text")): + continue + try: + u = urllib.parse.urlsplit(it.url) + valid_url = u.scheme in ("http", "https") and bool(u.hostname) + except ValueError: + valid_url = False + if not valid_url or not (it.title.strip() or it.text.strip()): + continue + if _CHALLENGE_TITLE.fullmatch(it.title.strip()) and ( + len(it.text.strip()) < 160 or re.search( + r"cloudflare|captcha|verify you are human|checking your browser|just a moment|attention required", + it.text, re.I)): + continue + if it.url in seen: + continue + seen.add(it.url) + out.append(it) + return out + + def fetch(src, limit=None): fn = ADAPTERS.get(src.adapter) if not fn: return [] n = limit or src.limit try: - return fn(src, n) or [] + return _valid_items(fn(src, n) or []) except Exception: return [] -SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co") +SLOW_HOSTS = ("reddit.com", "pullpush.io", "api.github.com", "huggingface.co", "services.nvd.nist.gov") def fetch_many(sources, limit=None, workers=8): @@ -944,5 +989,6 @@ def fetch_many(sources, limit=None, workers=8): got = fetch(s, limit) report.append((s.name, len(got))) items.extend(got) - time.sleep(3.5 if "reddit.com" in s.url else 1.0) + delay = 6.5 if "services.nvd.nist.gov" in s.url else (3.5 if "reddit.com" in s.url else 1.0) + time.sleep(delay) return items, report \ No newline at end of file diff --git a/omni/panel_api.py b/omni/panel_api.py index 2aae377..d97b52b 100644 --- a/omni/panel_api.py +++ b/omni/panel_api.py @@ -13,8 +13,8 @@ import urllib.parse from . import abuse as abuse_mod from . import config, sources, source_state, store, taxonomy -BUCKET = ("CASE WHEN published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' " - "THEN substr(published,1,10) ELSE date(seen,'unixepoch') END") +BUCKET = ("CASE WHEN length(published)=10 AND published GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' " + "THEN published ELSE COALESCE(date(published,'localtime'),date(seen,'unixepoch','localtime')) END") def _clean(v, n=0): @@ -32,7 +32,7 @@ def _clean(v, n=0): # ---------- обзор ---------- -BUCKET_SEEN = "date(seen,'unixepoch')" +BUCKET_SEEN = "date(seen,'unixepoch','localtime')" def timeline(days=30, cat=None, mode="pub"): @@ -40,6 +40,8 @@ def timeline(days=30, cat=None, mode="pub"): mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет); mode="seen" — строго по времени импорта: «что комбайн принёс за день». + + last24 — скользящие 24 часа по выбранному mode; непарсируемая дата → seen. """ days = 30 if days is None else int(days or 0) days = max(1, min(365, days)) @@ -57,6 +59,11 @@ def timeline(days=30, cat=None, mode="pub"): for b, n, cve, sev in c.execute(sql, args): if b: got[b] = (int(n or 0), int(cve or 0), int(sev or 0)) + clock = "seen" if mode == "seen" else "COALESCE(CAST(strftime('%s',published) AS REAL),seen)" + recent_where = where + (" AND " if where else " WHERE ") + clock + " >= ? AND " + clock + " <= ?" + now = time.time() + last24 = c.execute("SELECT COUNT(*) FROM items" + recent_where, + args + [now - 86400, now]).fetchone()[0] today = _dt.date.today() out = [] for i in range(days - 1, -1, -1): @@ -65,7 +72,8 @@ def timeline(days=30, cat=None, mode="pub"): out.append({"day": d, "n": n, "cve": cve, "sev": sev}) return {"ok": True, "days": days, "mode": mode, "points": out, "total": sum(p["n"] for p in out), - "last24": (out[-1] or {}).get("n", 0), + "last24": int(last24), + "today": out[-1]["n"], "max": max([p["n"] for p in out] or [0])} diff --git a/omni/rank.py b/omni/rank.py index 3e08ae0..591b168 100644 --- a/omni/rank.py +++ b/omni/rank.py @@ -2,6 +2,8 @@ import re import time +from .dates import parse_published + WEIGHT = {"reddit": 1.15, "x": 1.10, "forums": 1.0, "tavily": 1.0, "exa": 1.0, "firecrawl": 0.9, "you": 0.9, "ddg": 0.75, "cards": 0.35, "treg": 0.8} @@ -24,20 +26,12 @@ KIND_BONUS = {"tweet": 0.15, "thread": 0.25, "tg_post": 0.2, "card": 0.05, "resu def _fresh(published): - """published: unix-строка или ISO. Возвращает 0..1.""" - if not published: - return 0.3 - try: - if published.isdigit(): - ts = int(published) - if ts > 10**12: - ts //= 1000 - else: - ts = time.mktime(time.strptime(published[:19], "%Y-%m-%dT%H:%M:%S")) - days = max(0.0, (time.time() - ts) / 86400) - return max(0.0, 1.0 - days / 30.0) - except Exception: + """Unix seconds/milliseconds, ISO or RFC 2822. Return freshness in 0..1.""" + dt = parse_published(published) + if dt is None: return 0.3 + days = max(0.0, (time.time() - dt.timestamp()) / 86400) + return max(0.0, 1.0 - days / 30.0) def score(it): diff --git a/omni/serve.py b/omni/serve.py index 3eb0414..74d418f 100644 --- a/omni/serve.py +++ b/omni/serve.py @@ -115,6 +115,33 @@ def _names_in_category(cat): return [s.name for s in sources.all_sources() if s.category == cat] +def _run_state(status): + """Класс статуса прогона: ok / empty / none / err. + + "ok", "ok 0/10", "ok-304" — успех (startswith); "пусто"/"empty" — источник ответил пусто, + это не ошибка; отсутствующий статус (None/"") — отдельный класс none; остальное — err. + """ + v = " ".join(str(status or "").split()).lower() + if not v: + return "none" + if v.startswith("ok"): + return "ok" + if v in ("пусто", "empty"): + return "empty" + return "err" + + +def _since_days(days): + """?days=N -> граница since (YYYY-MM-DD) для store.browse2; 0 — без фильтра. + + Единая логика для /api/items и экспортов: «сутки/неделя/месяц» значат одно и то же. + """ + if not days: + return None + import datetime as _dt + return (_dt.date.today() - _dt.timedelta(days=days)).isoformat() + + def _run_feed(job_id, category, limit, tags): picked = sources.pick(category, None, tags) _job(job_id, state="running", total=len(picked), done=0) @@ -211,6 +238,18 @@ class Handler(BaseHTTPRequestHandler): return self._send(404, "no ui", "text/plain") return self._send(200, f.read_bytes(), "text/html; charset=utf-8") + if path == "/api/panels": + import json as _json + fp = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "panels.json") + try: + with open(fp, "r", encoding="utf-8") as f: + raw = _json.load(f) + except Exception: + raw = [] + rows = raw if isinstance(raw, list) else (raw.get("panels") or raw.get("items") or []) + rows = sorted(rows, key=lambda x: -(x.get("free_count") or x.get("free") or 0)) + return self._json({"ok": True, "total": len(rows), "panels": rows[:200]}) + if path == "/api/stats": s = store.stats() per_source = store.count_by_source() @@ -264,6 +303,7 @@ class Handler(BaseHTTPRequestHandler): import time as _t con = store._con() if hasattr(store, "_con") else None rows = [] + by_status = {"ok": 0, "empty": 0, "err": 0, "none": 0} try: import sqlite3 as _s3 from .config import DB as _DB @@ -274,12 +314,22 @@ class Handler(BaseHTTPRequestHandler): d = dict(r) ts = d.get("ts") or 0 d["when"] = _t.strftime("%Y-%m-%d %H:%M", _t.localtime(ts)) if ts else "" + d["state"] = _run_state(d.get("status")) + d["ok"] = d["state"] == "ok" rows.append(d) total = db.execute("SELECT count(*) FROM runs").fetchone()[0] + # статусы считаем по всей таблице: total-ok из среза limit давал ложные «ошибки» + sql = ("SELECT COALESCE(SUM(CASE WHEN lower(trim(status)) LIKE 'ok%' THEN 1 ELSE 0 END),0)," + " COALESCE(SUM(CASE WHEN lower(trim(status)) IN ('пусто','empty') THEN 1 ELSE 0 END),0)," + " COALESCE(SUM(CASE WHEN status IS NULL OR trim(status)='' THEN 1 ELSE 0 END),0)" + " FROM runs") + n_ok, n_empty, n_none = db.execute(sql).fetchone() + by_status = {"ok": int(n_ok or 0), "empty": int(n_empty or 0), "none": int(n_none or 0)} + by_status["err"] = max(0, int(total) - by_status["ok"] - by_status["empty"] - by_status["none"]) db.close() except Exception as e: return self._json({"error": "runs: %s" % e}) - return self._json({"items": rows, "total": total, "shown": len(rows)}) + return self._json({"items": rows, "total": total, "shown": len(rows), "by_status": by_status}) if path == "/api/abuse": from . import abuse as _ab @@ -355,13 +405,13 @@ class Handler(BaseHTTPRequestHandler): typ = (q.get("type") or ["all"])[0] src = (q.get("src") or [""])[0].strip() or None days = _intv(q, "days", 0, 0, 3650) - since = None - if days: - import datetime as _dt - since = (_dt.date.today() - _dt.timedelta(days=days)).isoformat() + since = _since_days(days) rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort, priced=priced, typ=typ, since=since, src=src) + from . import abuse as _ab for _r in rows: + # find нужен itemRow/findChips: browse2 сам признаки абуза не считает + _r["find"] = _ab.extract(" ".join((_r.get("title") or "", _r.get("text") or ""))) _r["title"] = _clean_txt(_r.get("title"), 180) _r["text"] = _clean_txt(_r.get("text"), 260) _r["published"] = _fmt_date(_r.get("published")) @@ -374,8 +424,11 @@ class Handler(BaseHTTPRequestHandler): priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on") sort = (q.get("sort") or ["score"])[0] typ = (q.get("type") or ["all"])[0] - rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=5000, offset=0, - sort=sort, priced=priced, typ=typ) + since = _since_days(_intv(q, "days", 0, 0, 3650)) + limit = _intv(q, "limit", 1000, 1, 5000) + src = (q.get("src") or [""])[0].strip() or None + rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=0, + sort=sort, priced=priced, typ=typ, since=since, src=src) import io as _io buf = _io.StringIO() w = _csv.writer(buf, delimiter=";") @@ -445,12 +498,17 @@ class Handler(BaseHTTPRequestHandler): if path == "/api/export.json": cat = (q.get("category") or [None])[0] + priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on") + since = _since_days(_intv(q, "days", 0, 0, 3650)) + src = (q.get("src") or [""])[0].strip() or None rows, total = store.browse2(q=(q.get("q") or [""])[0].strip(), names=_names_in_category(cat), - limit=_intv(q, "limit", 500, 1, 5000), + limit=_intv(q, "limit", 1000, 1, 5000), sort=(q.get("sort") or ["score"])[0], - typ=(q.get("type") or ["all"])[0]) - data = json.dumps({"total": total, "items": rows}, ensure_ascii=False, indent=1).encode("utf-8") + typ=(q.get("type") or ["all"])[0], + priced=priced, since=since, src=src) + data = json.dumps({"total": total, "count": len(rows), "items": rows}, + ensure_ascii=False, indent=1).encode("utf-8") self.send_response(200) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Disposition", 'attachment; filename="omni.json"') diff --git a/omni/sources.py b/omni/sources.py index b213f42..be02da0 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -19,6 +19,7 @@ from .sources_tg_abuse import TG_ABUSE from .sources_ext4 import EXTRA5 from .sources_ext5 import EXTRA6 from .sources_fresh import fresh as _fresh_sources +from .sources_quality import quality_sources as _quality_sources from .sources_x import X_SOURCES @@ -298,7 +299,7 @@ def _gen_sources(): return out -def all_sources(): +def all_sources(include_fresh=True): """Реестр + пользовательские источники из панели, без дублей по url и по имени. Дедуп по двум ключам обязателен: @@ -306,8 +307,10 @@ def all_sources(): * name — имя источника является первичным ключом в source_flags/runs, поэтому зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут». - Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна → + Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая и quality-волны → пользовательские → abuse-radar → panels.json. + + include_fresh=False исключает текущий generated fresh-модуль при регенерации. """ from . import source_state extra = [] @@ -318,7 +321,8 @@ def all_sources(): ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")} extra.append(Source(**d)) seen_url, seen_name, out = set(), set(), [] - for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources(): + fresh = _fresh_sources(S) if include_fresh else [] + for s in SOURCES + _gen_sources() + fresh + _quality_sources(S) + extra + radar_sources() + panel_sources(): key = (s.url or "").rstrip("/").lower() name = (s.name or "").strip().lower() if (key and key in seen_url) or (name and name in seen_name): diff --git a/omni/sources_ext.py b/omni/sources_ext.py index 74fb404..959c28c 100644 --- a/omni/sources_ext.py +++ b/omni/sources_ext.py @@ -228,8 +228,9 @@ EXTRA = [ # ---------------- security ---------------- S("cisa_kev", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "cisa_kev", ("cve", "kev")), - S("nvd_recent", "security", "json", - "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&noRejected", "nvd", ("cve",)), + # nvd_recent намеренно НЕ здесь: живой nvd_recent с пересчитываемым окном + # lastMod отдаёт sources_fresh.py (tools/probe_fresh.py --emit). Статическая + # запись без окна здесь побеждала по first-wins и глушила свежую. S("github_advisories", "security", "rss", "https://github.com/advisories.atom", "rss", ("cve", "github")), S("snyk_blog", "security", "rss", "https://snyk.io/blog/feed/", "rss", ("appsec",)), S("trailofbits_blog", "security", "rss", "https://blog.trailofbits.com/feed/", "rss", ("research",)), diff --git a/omni/sources_fresh.py b/omni/sources_fresh.py index 69c101b..c0b15d7 100644 --- a/omni/sources_fresh.py +++ b/omni/sources_fresh.py @@ -21,58 +21,58 @@ def fresh(S): return [ S("darkwebinformer", "abuse", "rss", "https://darkwebinformer.com/feed", "rss", ("leak", "news")), S("databreaches_net", "abuse", "rss", "https://databreaches.net/feed/", "rss", ("breach", "news")), - S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach")), + S("hackread_breaches", "abuse", "rss", "https://hackread.com/category/data-breaches/feed/", "rss", ("breach",)), S("redpacketsecurity", "abuse", "rss", "https://www.redpacketsecurity.com/feed/", "rss", ("cve", "leak")), S("tg_vulns", "abuse", "html", "https://t.me/s/vulns", "tg_web", ("tg", "leak", "ru")), S("habr_ai", "community", "rss", "https://habr.com/ru/rss/hub/artificial_intelligence/", "rss", ("ru", "ai")), - S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup")), + S("infosec_writeups", "community", "rss", "https://infosecwriteups.com/feed", "rss", ("writeup",)), S("tg_ai_machinelearning", "community", "html", "https://t.me/s/ai_machinelearning_big_data", "tg_web", ("tg", "ai", "ru")), S("tg_denissexy", "community", "html", "https://t.me/s/denissexy", "tg_web", ("tg", "ai", "ru")), S("reddit_blueteamsec", "forum", "json", "https://www.reddit.com/r/blueteamsec/.json?limit=25", "reddit_listing", ("reddit", "blue-team")), S("reddit_exploitdev", "forum", "json", "https://www.reddit.com/r/ExploitDev/.json?limit=25", "reddit_listing", ("reddit", "exploit")), S("reddit_purpleteamsec", "forum", "json", "https://www.reddit.com/r/purpleteamsec/.json?limit=25", "reddit_listing", ("reddit", "purple-team")), S("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing", ("cn", "forum")), - S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec")), + S("acunetix_blog", "security", "rss", "https://www.acunetix.com/blog/feed/", "rss", ("appsec",)), S("anquanke_news", "security", "html", "https://www.anquanke.com/news", "html_listing", ("cn", "news")), S("arch_linux_asa", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", ("cve", "linux")), - S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti")), - S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news")), - S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research")), + S("arcticwolf_blog", "security", "rss", "https://arcticwolf.com/feed/", "rss", ("cti",)), + S("arstechnica_security", "security", "rss", "https://arstechnica.com/security/feed/", "rss", ("news",)), + S("bitdefender_labs", "security", "rss", "https://www.bitdefender.com/blog/api/rss/labs/", "rss", ("research",)), S("chrome_releases_security", "security", "rss", "https://chromereleases.googleblog.com/feeds/posts/default", "rss", ("cve", "browser")), S("cisco_security_blog", "security", "rss", "https://blogs.cisco.com/security/feed", "rss", ("cti", "research")), S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")), - S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")), - S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")), + S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news",)), + S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news",)), S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")), S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")), S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")), S("hackmag", "security", "rss", "https://hackmag.com/rss", "rss", ("ru", "research")), - S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news")), + S("helpnetsecurity", "security", "rss", "https://www.helpnetsecurity.com/feed/", "rss", ("news",)), S("industrialcyber", "security", "rss", "https://industrialcyber.co/feed/", "rss", ("ics", "news")), - S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news")), - S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research")), - S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")), + S("infosecurity_magazine", "security", "rss", "https://www.infosecurity-magazine.com/rss/news/", "rss", ("news",)), + S("malwaretech_blog", "security", "rss", "https://www.malwaretech.com/feed", "rss", ("research",)), + S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research",)), S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")), S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")), S("nvd_exploited_kw", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", "nvd", ("cve", "exploited")), S("nvd_recent", "security", "json", _nvd_week(), "nvd", ("cve", "nvd")), - S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")), + S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research",)), S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")), - S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")), + S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing",)), S("secrss", "security", "html", "https://www.secrss.com/", "html_listing", ("cn", "news")), - S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research")), - S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research")), + S("sekoia_blog", "security", "rss", "https://blog.sekoia.io/feed/", "rss", ("research",)), + S("sentinelone_blog", "security", "rss", "https://www.sentinelone.com/blog/feed/", "rss", ("research",)), S("sihaoyou_feed", "security", "rss", "https://www.4hou.com/feed", "rss", ("cn", "research")), - S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti")), - S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain")), - S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit")), + S("socradar_blog", "security", "rss", "https://socradar.io/feed/", "rss", ("cti",)), + S("sonatype_blog", "security", "rss", "https://www.sonatype.com/blog/rss.xml", "rss", ("supply-chain",)), + S("sploitus_rss", "security", "rss", "https://sploitus.com/rss", "rss", ("exploit",)), S("tg_bugbounty", "security", "html", "https://t.me/s/bugbounty", "tg_web", ("tg", "bugbounty")), S("tg_hackernews_ru", "security", "html", "https://t.me/s/hackernews_ru", "tg_web", ("tg", "ru")), S("tg_kalilinux", "security", "html", "https://t.me/s/kalilinux", "tg_web", ("tg", "tools")), - S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news")), + S("theregister_security", "security", "rss", "https://www.theregister.com/security/headlines.atom", "rss", ("news",)), S("ubuntu_usn", "security", "rss", "https://ubuntu.com/security/notices/rss.xml", "rss", ("cve", "linux")), - S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware")), - S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve")), + S("virustotal_blog", "security", "rss", "https://blog.virustotal.com/feeds/posts/default", "rss", ("malware",)), + S("vuldb_recent", "security", "rss", "https://vuldb.com/?rss.recent", "rss", ("cve",)), S("xakep_ru", "security", "rss", "https://xakep.ru/feed/", "rss", ("ru", "news")), ] diff --git a/omni/sources_quality.py b/omni/sources_quality.py new file mode 100644 index 0000000..7c6bea0 --- /dev/null +++ b/omni/sources_quality.py @@ -0,0 +1,174 @@ +"""Quality-волна источников: официальные release-фиды проектов и security-адвайзори вендоров. + +Зачем отдельный модуль: основной реестр (sources.py) и волны ext/fresh уже покрывают +топовые репозитории, но провал в (1) runtime/инференсе второго эшелона и (2) security- +продуктах, где релиз = факт CVE/патча. Здесь только официальные Atom/RSS-фиды +(github.com//releases.atom, linux-дистрибутивные и вендорные advisory-фиды), +каждая запись проверена живым прогоном. + +Модуль статический и НЕ импортирует .sources: sources.py уже импортирует волны, +обратный импорт дал бы цикл. Дедуп по url/name делает all_sources(). + +Категории — только существующие в реестре, без новой схемы: + models — релизы AI-runtime/инференса (diffusers, TensorRT-LLM, onnxruntime, ...); + community — SDK/агентные фреймворки (MCP python-sdk, openai-agents, langgraph); + security — продукты, где релиз = патч/CVE, контейнеры и advisory-фиды дистрибутивов; + company — официальные технические блоги вендоров (NVIDIA, HashiCorp). +Признак «это релиз» остаётся тегом `release` у всех 37 записей, kind=atom/rss. + +Проверка (tools/probe_quality_sources.py, отчёт data/quality_sources_probe.json): + * каждое имя и URL сверены с полным реестром all_sources() (≈3.5k источников) — ни одного + дубля по имени, нормализованному URL и зеркалу репозитория (github.com//); + * первые записи не пересекаются по URL/заголовкам с другими кандидатами (>=50%/60%); + * >= 2 записей с валидным http(s)-URL, без challenge/error-заголовков; + * штатный адаптер feeds.fetch (rss, Atom поддерживается) отдаёт 10-20 записей. + +Осознанно НЕ включено, хотя фиды живые (шум, не качество): pytorch/pytorch и +pydantic/pydantic-ai (в releases.atom CI-теги viable/strict, clai2-bleeding), +microsoft/DeepSpeed (nightly-last-green), keycloak отдаёт nightly-теги (оставлен — +версии рядом), portainer/immich/netbox/nextcloud/gitea/elastic/opensearch/minio +(низкий сигнал на релиз, не security), php.net (фид отдаёт 1 запись), а также +зеркала и медиа-перепечатки (Cisco Talos, MSRC blog, PortSwigger, Cloudflare, +Ubuntu USN, Rust/Go/Node/Python blogs) — они уже в реестре или дублируют его. +""" + +SPECS = [ + # ---------- AI runtime / inference (официальные релизы) ---------- + {"name": "gh_rel_diffusers", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/diffusers/releases.atom", + "tags": ("release", "github", "ai", "image"), "limit": 10}, + {"name": "gh_rel_peft", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/peft/releases.atom", + "tags": ("release", "github", "ai", "finetune"), "limit": 10}, + {"name": "gh_rel_accelerate", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/accelerate/releases.atom", + "tags": ("release", "github", "ai", "training"), "limit": 10}, + {"name": "gh_rel_textgenerationinference", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/text-generation-inference/releases.atom", + "tags": ("release", "github", "ai", "inference"), "limit": 10}, + {"name": "gh_rel_tokenizers", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/tokenizers/releases.atom", + "tags": ("release", "github", "ai", "nlp"), "limit": 10}, + {"name": "gh_rel_candle", "category": "models", "kind": "atom", + "url": "https://github.com/huggingface/candle/releases.atom", + "tags": ("release", "github", "ai", "rust"), "limit": 10}, + {"name": "gh_rel_tensorrtllm", "category": "models", "kind": "atom", + "url": "https://github.com/NVIDIA/TensorRT-LLM/releases.atom", + "tags": ("release", "github", "ai", "nvidia", "inference"), "limit": 10}, + {"name": "gh_rel_nvidiacontainertoolkit", "category": "models", "kind": "atom", + "url": "https://github.com/NVIDIA/nvidia-container-toolkit/releases.atom", + "tags": ("release", "github", "gpu", "container"), "limit": 10}, + {"name": "gh_rel_flashinfer", "category": "models", "kind": "atom", + "url": "https://github.com/flashinfer-ai/flashinfer/releases.atom", + "tags": ("release", "github", "ai", "inference"), "limit": 10}, + {"name": "gh_rel_onnxruntime", "category": "models", "kind": "atom", + "url": "https://github.com/microsoft/onnxruntime/releases.atom", + "tags": ("release", "github", "ai", "inference"), "limit": 10}, + {"name": "gh_rel_mlx", "category": "models", "kind": "atom", + "url": "https://github.com/ml-explore/mlx/releases.atom", + "tags": ("release", "github", "ai", "apple"), "limit": 10}, + {"name": "gh_rel_whispercpp", "category": "models", "kind": "atom", + "url": "https://github.com/ggml-org/whisper.cpp/releases.atom", + "tags": ("release", "github", "ai", "audio"), "limit": 10}, + {"name": "gh_rel_exo", "category": "models", "kind": "atom", + "url": "https://github.com/exo-explore/exo/releases.atom", + "tags": ("release", "github", "ai", "local"), "limit": 10}, + {"name": "gh_rel_vllmascend", "category": "models", "kind": "atom", + "url": "https://github.com/vllm-project/vllm-ascend/releases.atom", + "tags": ("release", "github", "ai", "inference"), "limit": 10}, + + # ---------- AI-агенты / SDK ---------- + {"name": "gh_rel_mcp_pythonsdk", "category": "community", "kind": "atom", + "url": "https://github.com/modelcontextprotocol/python-sdk/releases.atom", + "tags": ("release", "github", "ai", "mcp"), "limit": 10}, + {"name": "gh_rel_openaiagentspython", "category": "community", "kind": "atom", + "url": "https://github.com/openai/openai-agents-python/releases.atom", + "tags": ("release", "github", "ai", "agent"), "limit": 10}, + {"name": "gh_rel_langgraph", "category": "community", "kind": "atom", + "url": "https://github.com/langchain-ai/langgraph/releases.atom", + "tags": ("release", "github", "ai", "agent"), "limit": 10}, + + # ---------- security-продукты: релиз = патч/CVE ---------- + {"name": "gh_rel_caddy", "category": "security", "kind": "atom", + "url": "https://github.com/caddyserver/caddy/releases.atom", + "tags": ("security", "release", "github", "web"), "limit": 10}, + {"name": "gh_rel_keycloak", "category": "security", "kind": "atom", + "url": "https://github.com/keycloak/keycloak/releases.atom", + "tags": ("security", "release", "github", "iam"), "limit": 10}, + {"name": "gh_rel_grafana", "category": "security", "kind": "atom", + "url": "https://github.com/grafana/grafana/releases.atom", + "tags": ("security", "release", "github", "monitoring"), "limit": 10}, + {"name": "gh_rel_tailscale", "category": "security", "kind": "atom", + "url": "https://github.com/tailscale/tailscale/releases.atom", + "tags": ("security", "release", "github", "vpn"), "limit": 10}, + {"name": "gh_rel_envoy", "category": "security", "kind": "atom", + "url": "https://github.com/envoyproxy/envoy/releases.atom", + "tags": ("security", "release", "github", "proxy"), "limit": 10}, + {"name": "gh_rel_vault", "category": "security", "kind": "atom", + "url": "https://github.com/hashicorp/vault/releases.atom", + "tags": ("security", "release", "github", "secrets"), "limit": 10}, + {"name": "gh_rel_cosign", "category": "security", "kind": "atom", + "url": "https://github.com/sigstore/cosign/releases.atom", + "tags": ("security", "release", "github", "supply-chain"), "limit": 10}, + {"name": "gh_rel_trivy", "category": "security", "kind": "atom", + "url": "https://github.com/aquasecurity/trivy/releases.atom", + "tags": ("security", "release", "github", "scanner"), "limit": 10}, + {"name": "gh_rel_grype", "category": "security", "kind": "atom", + "url": "https://github.com/anchore/grype/releases.atom", + "tags": ("security", "release", "github", "scanner"), "limit": 10}, + {"name": "gh_rel_nuclei", "category": "security", "kind": "atom", + "url": "https://github.com/projectdiscovery/nuclei/releases.atom", + "tags": ("security", "release", "github", "scanner"), "limit": 10}, + {"name": "gh_rel_zaproxy", "category": "security", "kind": "atom", + "url": "https://github.com/zaproxy/zaproxy/releases.atom", + "tags": ("security", "release", "github", "web"), "limit": 10}, + {"name": "gh_rel_openssl", "category": "security", "kind": "atom", + "url": "https://github.com/openssl/openssl/releases.atom", + "tags": ("security", "release", "github", "crypto"), "limit": 10}, + {"name": "gh_rel_curl", "category": "security", "kind": "atom", + "url": "https://github.com/curl/curl/releases.atom", + "tags": ("security", "release", "github", "http"), "limit": 10}, + + # ---------- контейнеры/оркестрация: CVE-релевантные релизы ---------- + {"name": "gh_rel_runc", "category": "security", "kind": "atom", + "url": "https://github.com/opencontainers/runc/releases.atom", + "tags": ("security", "release", "github", "container"), "limit": 10}, + {"name": "gh_rel_containerd", "category": "security", "kind": "atom", + "url": "https://github.com/containerd/containerd/releases.atom", + "tags": ("security", "release", "github", "container"), "limit": 10}, + {"name": "gh_rel_kubernetes", "category": "security", "kind": "atom", + "url": "https://github.com/kubernetes/kubernetes/releases.atom", + "tags": ("security", "release", "github", "kubernetes"), "limit": 10}, + + # ---------- официальные advisory/technical фиды вендоров ---------- + {"name": "amazon_linux_alas", "category": "security", "kind": "rss", + "url": "https://alas.aws.amazon.com/alas.rss", + "tags": ("security", "release", "advisory", "linux", "aws"), "limit": 20}, + {"name": "aws_security_bulletins", "category": "security", "kind": "rss", + "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/", + "tags": ("security", "release", "advisory", "aws"), "limit": 20}, + {"name": "nvidia_dev_blog", "category": "company", "kind": "rss", + "url": "https://developer.nvidia.com/blog/feed/", + "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15}, + {"name": "hashicorp_blog", "category": "company", "kind": "rss", + "url": "https://www.hashicorp.com/blog/feed.xml", + "tags": ("release", "vendor", "hashicorp"), "limit": 15}, +] + +NAMES = tuple(d["name"] for d in SPECS) + + +def quality_sources(S, specs=None): + """Собрать Source-объекты quality-волны через переданный конструктор S. + + S передаётся снаружи (обычно sources.Source), чтобы модуль не импортировал .sources — + это дал бы циклический импорт. Вызов позиционный, как в sources_fresh.fresh: так + работает и настоящий Source, и лёгкая заглушка `class _Fake: __init__(self, *a, **k)`, + которой tools/run_fresh.py --wave quality собирает имена. + Теги всегда кортежи; adapter по умолчанию rss — он же разбирает Atom. + """ + out = [] + for d in (SPECS if specs is None else specs): + out.append(S(d["name"], d["category"], d["kind"], d["url"], + d.get("adapter", "rss"), tuple(d.get("tags") or ()), d.get("limit", 10))) + return out \ No newline at end of file diff --git a/omni/store.py b/omni/store.py index 803bcb0..8609e87 100644 --- a/omni/store.py +++ b/omni/store.py @@ -257,7 +257,7 @@ def stats(): by_kind = c.execute("SELECT kind, COUNT(*) FROM items GROUP BY kind ORDER BY 2 DESC").fetchall() priced = c.execute("SELECT COUNT(*) FROM items WHERE price IS NOT NULL").fetchone()[0] runs = c.execute("SELECT COUNT(*) FROM runs").fetchone()[0] - ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE status='ok'").fetchone()[0] + ok_runs = c.execute("SELECT COUNT(*) FROM runs WHERE lower(trim(status)) LIKE 'ok%'").fetchone()[0] return {"items": total, "priced": priced, "runs": runs, "runs_ok": ok_runs, "by_provider": by_prov, "by_kind": by_kind} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..15cfa70 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,35 @@ +"""All tests use disposable storage, including module-import side effects.""" +import os +import sys +import tempfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# This runs before test collection: abuse builds source metadata at import time. +_SESSION_DATA = tempfile.TemporaryDirectory(prefix="omni-pytest-") +os.environ["OMNI_DATA"] = _SESSION_DATA.name +os.environ["OMNI_DB"] = str(Path(_SESSION_DATA.name) / "collection.sqlite3") + +from omni import config + +config.DATA = Path(_SESSION_DATA.name) +config.DB = config.DATA / "collection.sqlite3" + + +@pytest.fixture(autouse=True) +def sandbox_data(tmp_path, monkeypatch): + """Give every test its own database, not only the new panel tests.""" + monkeypatch.setattr(config, "DATA", tmp_path) + monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3") + monkeypatch.setenv("OMNI_DATA", str(tmp_path)) + monkeypatch.setenv("OMNI_DB", str(config.DB)) + from omni import abuse, panel_api + monkeypatch.setattr(abuse, "DB", config.DB) + monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json") + + +def pytest_unconfigure(config): + _SESSION_DATA.cleanup() diff --git a/tests/test_ingest_quality.py b/tests/test_ingest_quality.py new file mode 100644 index 0000000..4299845 --- /dev/null +++ b/tests/test_ingest_quality.py @@ -0,0 +1,103 @@ +"""Offline regressions for feed metadata, date ranking and challenge filtering.""" +import sys +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from omni import feeds, rank +from omni.dates import parse_published +from omni.model import Item +from omni.sources import S + + +@pytest.mark.parametrize("value", [ + "2026-10-20T12:00:00Z", "2026-10-20T15:00:00+03:00", + "Tue, 20 Oct 2026 12:00:00 GMT", "2026-10-20T12:00:00", + "1792497600", "1792497600000", 1792497600, +]) +def test_publication_formats_have_equal_freshness(value, monkeypatch): + stamp = datetime(2026, 10, 20, 12, tzinfo=timezone.utc).timestamp() + parsed = parse_published(value) + assert parsed is not None + assert parsed.timestamp() == stamp + monkeypatch.setattr(rank.time, "time", lambda: stamp + 86400) + assert rank._fresh(value) == pytest.approx(1 - 1 / 30) + + +def test_date_only_and_invalid_dates(monkeypatch): + stamp = datetime(2026, 10, 20, tzinfo=timezone.utc).timestamp() + monkeypatch.setattr(rank.time, "time", lambda: stamp) + assert rank._fresh("2026-10-20") == 1 + for value in (None, "", "not a date", "99999999999999999999999999", "NaN"): + assert parse_published(value) is None + assert rank._fresh(value) == 0.3 + assert rank._fresh("2020-01-01") == 0 + assert parse_published("2026") is None + assert parse_published("20261020") == datetime(2026, 10, 20, tzinfo=timezone.utc) + + +def test_atom_alternate_links_author_and_date(monkeypatch): + src = S("atom_test", "models", "rss", "https://example.invalid/feed.atom") + xml = ''' + Runtime release 2.0 + + + Release team + 2026-10-20T15:00:00+03:00 + Faster inference and improved caching. + ''' + monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml)) + rows = feeds.fetch(src) + assert len(rows) == 1 + assert rows[0].url == "https://example.invalid/releases/2.0" + assert rows[0].author == "Release team" + assert rows[0].published == "2026-10-20T12:00:00+00:00" + + +def test_rss_namespaced_creator_and_no_author(monkeypatch): + src = S("rss_test", "security", "rss", "https://example.invalid/feed") + xml = ''' + Security advisoryhttps://example.invalid/advisory + Security teamTue, 20 Oct 2026 12:00:00 GMT + New patch availablehttps://example.invalid/patch + ''' + monkeypatch.setattr(feeds.net, "get", lambda *a, **kw: SimpleNamespace(status=200, text=xml)) + rows = feeds.fetch(src) + assert len(rows) == 2 + assert rows[0].author == "Security team" + assert rows[0].published == "2026-10-20T12:00:00+00:00" + assert rows[1].author == "" + + +def test_fetch_drops_invalid_pages_but_keeps_short_real_posts(monkeypatch): + src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter") + def item(path, title, text=""): + return Item("test", "entry", url=path, title=title, text=text) + good = item("https://example.invalid/release", "v1.2") + real = item("https://example.invalid/article", "Access denied", "Real analysis of authorization failure. " * 20) + rows = [ + item("", "A missing URL"), + item("javascript:alert(1)", "A dangerous URL"), + item("https://example.invalid/blank", ""), + item("https://example.invalid/challenge", "Just a moment...", "Checking your browser"), + item("https://example.invalid/denied", "Attention Required! | Cloudflare", "captcha"), + item("https://example.invalid/long-challenge", "Just a moment...", + "Checking your browser before accessing this site. " * 20), + {"title": "Malformed adapter result"}, None, "not an Item", + good, item(good.url, "Duplicate record"), real, + ] + monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: rows) + assert feeds.fetch(src) == [good, real] + + +def test_fetch_bad_record_does_not_discard_valid_siblings(monkeypatch): + src = S("test", "security", "rss", "https://example.invalid/feed", "test_adapter") + good = Item("test", "entry", url="https://example.invalid/release", title="Real release") + bad = Item("test", "entry", url="https://example.invalid/bad", title="Malformed") + bad.text = None + monkeypatch.setitem(feeds.ADAPTERS, "test_adapter", lambda source, n: [bad, good]) + assert feeds.fetch(src) == [good] diff --git a/tests/test_panel_api.py b/tests/test_panel_api.py index f38de1d..8888943 100644 --- a/tests/test_panel_api.py +++ b/tests/test_panel_api.py @@ -5,11 +5,38 @@ import sys import time from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) os.environ.setdefault("PYTHONIOENCODING", "utf-8") +@pytest.fixture(autouse=True) +def isolated_panel_data(tmp_path, monkeypatch): + """Never touch the operator's database, watchlist, digest or TG session.""" + from omni import abuse, config, panel_api, store, tgsess + from omni.model import Item + + monkeypatch.setattr(config, "DATA", tmp_path) + monkeypatch.setattr(config, "DB", tmp_path / "test.sqlite3") + monkeypatch.setattr(abuse, "DB", config.DB) + monkeypatch.setattr(panel_api, "WATCH", tmp_path / "watch.json") + monkeypatch.setattr(config, "tg_cfg", lambda: {"session": "", "limit": 100}) + monkeypatch.setattr(tgsess, "enabled", lambda: False) + today = time.strftime("%Y-%m-%d") + store.add([ + Item(provider="pytest", kind="post", source="pytest-panel", + title="Firecrawl security dataset leak announcement", + text="Public incident report for regression coverage.", + url="https://example.invalid/seed", published=today), + Item(provider="pytest", kind="post", source="pytest-other", + title="Guide to observability and performance metrics", + text="A completely different article about tracing.", + url="https://example.invalid/other", published=today), + ]) + + def _tmp_item(tag=None): from omni import store from omni.model import Item @@ -124,14 +151,19 @@ def test_health_counts(): # ---------- tg ---------- -def test_tg_status_shape_and_flag(): - from omni import panel_api +@pytest.mark.parametrize("enabled", [False, True]) +def test_tg_status_shape_and_flag(monkeypatch, enabled): + from omni import config, panel_api, tgsess + monkeypatch.setattr(tgsess, "enabled", lambda: enabled) + monkeypatch.setattr(tgsess, "me", lambda: {"id": 123, "first_name": "Test"}) + monkeypatch.setattr(config, "tg_cfg", + lambda: {"session": "test-account", "limit": 100}) d = panel_api.tg_status() - assert {"enabled", "me", "error", "limit"} <= set(d) - assert isinstance(d["enabled"], bool) - if d["enabled"]: - assert d["session"] + assert {"enabled", "me", "error", "limit", "session"} <= set(d) + assert d["enabled"] is enabled + assert d["me"] == ({"id": 123, "first_name": "Test"} if enabled else None) + assert d["error"] is None # ---------- digest ---------- @@ -205,3 +237,170 @@ def test_browse2_filters_since_and_src(): # src + since вместе rows5, total5 = store.browse2(q="", limit=5, src=name, since="2099-01-01") assert total5 == 0 and rows5 == [] + + +def test_registry_dedupes_url_and_name(monkeypatch): + from omni import sources, source_state + + first = sources.S("first", "security", "rss", "https://example.invalid/feed") + same_url = sources.S("alias", "security", "rss", first.url + "/") + same_name = sources.S("FIRST", "security", "rss", "https://example.invalid/mirror") + other = sources.S("other", "security", "rss", "https://example.invalid/other") + monkeypatch.setattr(sources, "SOURCES", [first, same_url, same_name, other]) + monkeypatch.setattr(sources, "_gen_sources", lambda: []) + monkeypatch.setattr(sources, "_fresh_sources", lambda factory: []) + monkeypatch.setattr(sources, "_quality_sources", lambda factory: []) + monkeypatch.setattr(sources, "radar_sources", lambda: []) + monkeypatch.setattr(sources, "panel_sources", lambda: []) + monkeypatch.setattr(source_state, "custom", lambda: []) + assert sources.all_sources() == [first, other] + + +def test_fresh_sources_have_valid_metadata(): + from omni import feeds, sources + from omni.sources_fresh import fresh + + rows = fresh(sources.S) + assert rows + assert all(isinstance(s.tags, tuple) for s in rows) + assert all(s.adapter in feeds.ADAPTERS for s in rows) + assert len({s.name for s in rows}) == len(rows) + + +def test_quality_wave_is_integrated_without_collisions(): + from omni import feeds, sources + from omni.sources_quality import quality_sources + + wave = quality_sources(sources.S) + registry = sources.all_sources() + assert len(wave) >= 30 + assert all(isinstance(s.tags, tuple) and s.adapter in feeds.ADAPTERS for s in wave) + assert {s.name for s in wave} <= {s.name for s in registry} + assert len(registry) == len({s.name.strip().lower() for s in registry}) + assert len(registry) == len({s.url.rstrip("/").lower() for s in registry}) + + +@pytest.mark.parametrize("tags", [(), ("news",), ("news", "cve")]) +def test_fresh_generator_preserves_tag_tuples(tmp_path, tags): + import runpy + + from omni import sources + + emit = runpy.run_path(str(ROOT / "tools" / "probe_fresh.py"))["emit"] + row = {"name": "test_feed", "category": "security", "kind": "rss", + "adapter": "rss", "url": "https://example.invalid/feed", "tags": tags} + path = tmp_path / "generated_sources.py" + assert emit([row], [], path, dedupe=False) == 1 + generated = runpy.run_path(str(path))["fresh"](sources.S) + assert generated[0].tags == tags + assert isinstance(generated[0].tags, tuple) + + +def test_timeline_local_day_and_rolling_24_hours(monkeypatch): + import datetime + from omni import panel_api, store + + now = datetime.datetime(2026, 10, 20, 1, 0).timestamp() + class FrozenDate(datetime.date): + @classmethod + def today(cls): + return cls(2026, 10, 20) + + monkeypatch.setattr(panel_api._dt, "date", FrozenDate) + monkeypatch.setattr(panel_api.time, "time", lambda: now) + with store.db() as c: + c.execute("UPDATE items SET seen=? WHERE source='pytest-panel'", (now - 1800,)) + c.execute("UPDATE items SET seen=? WHERE source='pytest-other'", (now - 23 * 3600,)) + data = panel_api.timeline(days=2, mode="seen") + assert data["last24"] == 2, "rolling 24 hours must include yesterday evening" + assert data["today"] == 1, "local 00:30 import belongs to today's bar" + assert [p["n"] for p in data["points"]] == [1, 1] + + +def test_run_stats_count_partial_successes(): + from omni import store + + for status in ("ok", "ok 5/10", "пусто", "Exception: timeout"): + store.log_run("pytest", "test", "", 0, 0, status) + stats = store.stats() + assert stats["runs"] == 4 + assert stats["runs_ok"] == 2 + + +def test_timeline_last24_respects_publication_mode(monkeypatch): + import datetime + from omni import panel_api, store + + now = time.time() + monkeypatch.setattr(panel_api.time, "time", lambda: now) + old = datetime.datetime.fromtimestamp(now - 48 * 3600, datetime.timezone.utc).isoformat() + future = datetime.datetime.fromtimestamp(now + 48 * 3600, datetime.timezone.utc).isoformat() + with store.db() as c: + c.execute("UPDATE items SET seen=?", (now - 60,)) + c.execute("UPDATE items SET published=? WHERE source='pytest-panel'", (old,)) + c.execute("UPDATE items SET published=? WHERE source='pytest-other'", (future,)) + assert panel_api.timeline(mode="seen")["last24"] == 2 + assert panel_api.timeline(mode="pub")["last24"] == 0 + + +def test_http_exports_match_feed_filters_and_item_find(monkeypatch): + import csv + import datetime + import io + import json + import threading + import urllib.request + from http.server import ThreadingHTTPServer + from omni import serve, store + from omni.model import Item + + today = datetime.date.today().isoformat() + old = (datetime.date.today() - datetime.timedelta(days=10)).isoformat() + rows = [ + Item("pytest", "post", source="quality-http", title="promo code: QUAL2026 $20 30 days", + text="Official promotional offer for testing.", url="https://example.invalid/quality-new", published=today), + Item("pytest", "post", source="quality-http", title="Legacy license costs $99", + text="An old pricing page.", url="https://example.invalid/quality-old", published=old), + Item("pytest", "post", source="quality-http", title="Latest debugging guide", + text="A free guide to tracing request latency.", url="https://example.invalid/quality-free", published=today), + ] + store.add(rows) + for status in (" OK ", "ok 2/5", "пусто", "Exception: timeout", None): + store.log_run("pytest", "quality-http", "", 0, 0, status) + monkeypatch.setattr(serve, "UI_TOKEN", "") + server = ThreadingHTTPServer(("127.0.0.1", 0), serve.Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = "http://127.0.0.1:%s" % server.server_address[1] + query = "?src=quality-http&days=1&priced=1&limit=1000" + try: + def get(path): + with urllib.request.urlopen(base + path, timeout=10) as res: + assert res.status == 200 + return res.read().decode("utf-8-sig") + + feed = json.loads(get("/api/items" + query)) + export = json.loads(get("/api/export.json" + query)) + csv_rows = list(csv.DictReader(io.StringIO(get("/api/export.csv" + query)), delimiter=";")) + assert feed["total"] == export["total"] == 1 + assert [r["url"] for r in feed["items"]] == [rows[0].url] + assert [r["url"] for r in export["items"]] == [rows[0].url] + assert [r["url"] for r in csv_rows] == [rows[0].url] + assert "QUAL2026" in feed["items"][0]["find"]["codes"] + runs = json.loads(get("/api/runs?limit=2")) + assert runs["total"] == 5 and runs["shown"] == 2 + assert runs["by_status"] == {"ok": 2, "empty": 1, "err": 1, "none": 1} + states = {r["state"] for r in json.loads(get("/api/runs?limit=10"))["items"]} + assert states == {"ok", "empty", "err", "none"} + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + +def test_web_has_free_panel_block(): + """Вкладка абузов обязана показывать free-панели new-api.""" + html = open(os.path.join(ROOT, "web", "index.html"), encoding="utf-8").read() + assert "loadPanels" in html + assert "/api/panels" in html + assert "pn-grid" in html + assert ".pn-grid{" in html diff --git a/tools/fresh_candidates.py b/tools/fresh_candidates.py index 2ebf45f..f7cc660 100644 --- a/tools/fresh_candidates.py +++ b/tools/fresh_candidates.py @@ -4,7 +4,6 @@ переезжают в omni/sources_fresh.py (генерируется `--emit`). Формат dict совпадает с Source(**d); адаптеры — только из feeds.ADAPTERS. """ -import datetime def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None, **kw): @@ -14,10 +13,6 @@ def d(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, return row -_NOW = datetime.datetime.utcnow() -_NVD_SINCE = (_NOW - datetime.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000") -_NVD_UNTIL = _NOW.strftime("%Y-%m-%dT%H:%M:%S.000") - CANDIDATES = [ # ---------------- 1. security news / research ---------------- d("hackread", "security", "rss", "https://hackread.com/feed/", "rss", ("news",)), diff --git a/tools/probe_fresh.py b/tools/probe_fresh.py index 5500b28..9957525 100644 --- a/tools/probe_fresh.py +++ b/tools/probe_fresh.py @@ -7,8 +7,19 @@ Критерий живого: feeds.fetch(src, limit) вернул >= 1 запись. Мёртвым добираем диагностику прямым net.get (код/блок/размер), чтобы причина в отчёте была фактической, а не «0 items». + +Публичные хелперы (для тестов/внешних вызовов): + candidate_source(c) -> (Source, marker) — dict кандидата -> Source, url + динамического источника пересчитан ДО сети; маркер лежит в extra["_dynamic"]. + _registry_sources() -> [Source] — all_sources(include_fresh=False) + _registry_keys() -> (set url, set name) — нормализованные ключи реестра без + текущей sources_fresh (иначе перегенерация считает свою волну дублем). + _registry_urls() -> set url — совместимость. + emit(rows, dead, path, dedupe=True) — только известный dynamic-маркер; + неизвестный маркер = ValueError, а не замороженный протухший url. """ import argparse +import datetime as _dt import json import sys import time @@ -60,34 +71,130 @@ def probe_one(src, limit): def _norm(u): - u = u.strip().lower().rstrip("/") + u = str(u or "").strip().lower().rstrip("/") for pre in ("https://", "http://"): if u.startswith(pre): u = u[len(pre):] return u[4:] if u.startswith("www.") else u -def _registry_urls(): +def _norm_name(n): + """Имя источника — первичный ключ source_flags/runs: сравниваем без регистра, + пробелов и разделителей, иначе nvd-recent и nvd_recent разъезжаются.""" + return "".join(ch for ch in str(n or "").strip().lower() if ch.isalnum()) + + +def _registry_sources(): + """Реестр без текущего sources_fresh: перегенерация не должна считать + собственную прошлую волну «уже существующей» и стирать свои же находки.""" from omni.sources import all_sources - return {_norm(s.url) for s in all_sources()} + try: + return all_sources(include_fresh=False) + except TypeError: # реестр без флага — совместимость + return all_sources() + + +def _registry_keys(): + """(нормализованные url, нормализованные имена) реестра без sources_fresh.""" + rows = _registry_sources() + return ({_norm(s.url) for s in rows if s.url}, + {_norm_name(s.name) for s in rows if s.name}) + + +def _registry_urls(): + """Только url-ключи реестра (совместимость с внешними вызовами).""" + return _registry_keys()[0] + + +NVD_BASE = "https://services.nvd.nist.gov/rest/json/cves/2.0" +KNOWN_DYNAMIC = ("NVD_LAST_WEEK",) + + +def _nvd_week_url(): + """Фактический url NVD на момент прогона: окно lastMod <=120 дней, база + без окна отдаёт самые старые записи страницы и как «свежая» бесполезна.""" + now = _dt.datetime.utcnow() + since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000") + return ("%s?resultsPerPage=30&lastModStartDate=%s&lastModEndDate=%s" + % (NVD_BASE, since, now.strftime("%Y-%m-%dT%H:%M:%S.000"))) + + +def candidate_source(c): + """dict кандидата -> (Source, dynamic-маркер), url пересчитан до сети. + + Маркер сохраняется в extra["_dynamic"], чтобы отчёт/emit знали, что url + нельзя замораживать. Неизвестный маркер — ValueError: молча подставить + статический url значит опубликовать заведомо протухший источник. + """ + row = dict(c) + marker = str(row.pop("dynamic", "") or "") + extra = dict(row.get("extra") or {}) + if marker: + if marker not in KNOWN_DYNAMIC: + raise ValueError("unknown dynamic marker %r" % marker) + if marker == "NVD_LAST_WEEK": + row["url"] = _nvd_week_url() + extra["_dynamic"] = marker + row["extra"] = extra + return Source(**row), marker # контент-дубли уже имеющихся в реестре фидов (url отличается, лента та же) CONTENT_DUP = {"elastic_security_labs", "securityweek_alt", "reddit_netsec"} +def _extra_lit(extra): + """extra-литерал без служебных ключей: _dynamic — метаданные прогона, не источника.""" + clean = {k: v for k, v in sorted((extra or {}).items()) if k != "_dynamic"} + return "" if not clean else ", extra=%r" % (clean,) + + +def _frozen_window(rows): + """Строки, где в url уже стоит вычисленное окно (lastMod), но маркера нет: + такой отчёт старее фикса dynamic — при emit окно застынет на дате прогона.""" + return [str(r.get("name")) for r in rows + if not r.get("dynamic") and "lastmodstartdate=" in str(r.get("url") or "").lower()] + + def emit(rows, dead, path, dedupe=True): - """omni/sources_fresh.py: только живые, без импортов из omni.sources.""" - def q(s): - return '"%s"' % str(s).replace('"', '\\"') + """omni/sources_fresh.py: только живые, без импортов из omni.sources. + + Литералы пишем через repr: json.dumps даёт true/false/null — это не Python. + Дедуп сверяет и url, и имя (нормализованные) и добавляет принятые строки в + виденное, поэтому партия уникальна и сама по себе. Динамический url + поддерживается только для известного маркера: неизвестный — явная ошибка, + а не замороженный url, который молча протухнет. + """ + rows = list(rows) + unknown = sorted({str(r.get("dynamic")) for r in rows + if r.get("dynamic") and r.get("dynamic") not in KNOWN_DYNAMIC}) + if unknown: + raise ValueError("emit: неизвестный dynamic-маркер %s — замораживать url нельзя" + % ", ".join(map(repr, unknown))) + frozen = _frozen_window(rows) + if frozen: + print("emit: ВНИМАНИЕ, окно в url без dynamic-маркера (%s): отчёт устарел, " + "перепрогони probe_fresh" % ", ".join(frozen)) if dedupe: - have = _registry_urls() - kept, skipped = [], [] - for r in [x for x in rows if x["name"] not in CONTENT_DUP]: - n = _norm(r["url"]) - (skipped if n in have else kept).append(r) + have_url, have_name = _registry_keys() + seen_url, seen_name = set(have_url), set(have_name) + kept, skipped, dupes = [], [], [] + for r in rows: + n, nm = _norm(r.get("url")), _norm_name(r.get("name")) + in_registry = (bool(n) and n in have_url) or (bool(nm) and nm in have_name) + if r.get("name") in CONTENT_DUP: + in_registry = True + if in_registry or (bool(n) and n in seen_url) or (bool(nm) and nm in seen_name): + (skipped if in_registry else dupes).append(r) + continue + if n: + seen_url.add(n) + if nm: + seen_name.add(nm) + kept.append(r) print("emit: уже есть в реестре -> %d: %s" % (len(skipped), ", ".join(r["name"] for r in skipped))) + print("emit: дубли внутри партии -> %d: %s" % (len(dupes), ", ".join(r["name"] for r in dupes))) rows = kept lines = [ @@ -114,23 +221,23 @@ def emit(rows, dead, path, dedupe=True): " return [", ] for r in rows: - extra = "" - if r.get("extra"): - extra = ", extra=%s" % json.dumps(r["extra"], ensure_ascii=False, sort_keys=True) - limit = r.get("limit") or 15 - lim = "" if limit == 15 else ", limit=%d" % limit + marker = str(r.get("dynamic") or "") + tags = repr(tuple(r.get("tags") or ())) + limit = r.get("limit") + lim = ", limit=%d" % limit if isinstance(limit, int) and limit != 20 else "" det = ", detail=True" if r.get("detail") else "" - tags = "(%s)" % ", ".join(q(t) for t in r.get("tags") or ()) - if r.get("dynamic") == "NVD_LAST_WEEK" or r.get("name") == "nvd_recent": - lines.append(' S(%s, %s, %s, _nvd_week(), %s, %s%s%s),' % ( - q(r["name"]), q(r["category"]), q(r["kind"]), q(r["adapter"]), tags, lim, det)) + if marker == "NVD_LAST_WEEK": + lines.append(" S(%r, %r, %r, _nvd_week(), %r, %s%s%s%s)," % ( + r["name"], r.get("category", ""), r.get("kind", ""), r.get("adapter", "rss"), + tags, lim, det, _extra_lit(r.get("extra")))) continue - lines.append(' S(%s, %s, %s, %s, %s, %s%s%s%s),' % ( - q(r["name"]), q(r["category"]), q(r["kind"]), q(r["url"]), q(r["adapter"]), - tags, lim, det, extra)) + lines.append(" S(%r, %r, %r, %r, %r, %s%s%s%s)," % ( + r["name"], r.get("category", ""), r.get("kind", ""), r.get("url", ""), r.get("adapter", "rss"), + tags, lim, det, _extra_lit(r.get("extra")))) lines += [" ]", "", "", "# --- отклонено прогоном (не отдают записи без ключа/логина) ---"] for r in dead: - lines.append("# %-26s %-44s %s" % (r["name"], r["url"][:44], (r["err"] or "0 items")[:90])) + lines.append("# %-26s %-44s %s" % (r.get("name", "?"), str(r.get("url") or "")[:44], + (r.get("err") or "0 items")[:90])) text = "\n".join(lines) + "\n" path.write_text(text, encoding="utf-8") return len(rows) @@ -153,6 +260,12 @@ def main(): rows = rep["results"] alive = [r for r in rows if r["ok"]] dead = [r for r in rows if not r["ok"]] + frozen = _frozen_window(alive) + if frozen: + print("emit-only: отчёт без dynamic-маркера для %s — окно в url застынет на дате прогона; " + "перепрогони probe_fresh (сеть) или правь источники вручную" + % ", ".join(frozen)) + return 2 n = emit(alive, dead, ROOT / "omni" / "sources_fresh.py", dedupe=not a.no_dedupe) print("omni/sources_fresh.py: %d источников" % n) return 0 @@ -161,11 +274,11 @@ def main(): srcs = [] for c in cands: try: - cc = dict(c) - cc.pop("dynamic", None) - srcs.append(Source(**cc)) + src, _marker = candidate_source(c) except Exception as e: # noqa: BLE001 print("skip %s: %s" % (c.get("name"), e)) + continue + srcs.append(src) t0 = time.time() with ThreadPoolExecutor(max_workers=a.workers) as ex: diff --git a/tools/probe_quality_sources.py b/tools/probe_quality_sources.py new file mode 100644 index 0000000..ca24673 --- /dev/null +++ b/tools/probe_quality_sources.py @@ -0,0 +1,439 @@ +"""Живой прогон пула quality-кандидатов (официальные release/security фиды). + + python tools/probe_quality_sources.py # широкий пул -> data/quality_sources_pool.json + python tools/probe_quality_sources.py --only gh_rel_ # фильтр по имени + python tools/probe_quality_sources.py --confirm # сверить SPECS модуля -> data/quality_sources_probe.json + +Что проверяется (acceptance): + 1. прямой запрос net.get с жёстким timeout, XML парсится локально (RSS и Atom ); + 2. >= MIN_ITEMS записей с непустым http(s)-URL; + 3. ни одна из первых записей не является challenge/error-страницей (заголовок-заглушка); + 4. нет дублей с основным реестром omni.sources: имя, нормализованный URL, зеркало репозитория + (в --confirm свои 37 SPECS из этого множества вычитаются: после интеграции они уже в + all_sources(), иначе прогон «отклонил» бы сам себя; остальной реестр сверяется как есть); + 5. нет дублей между кандидатами: пересечение URL записей и заголовков (>= 60%). +Плюс фиксируется, что вернул штатный адаптер feeds.fetch: Atom-link/relative-link/DC-creator +в feeds.py починены, поэтому у принятых источников ожидается n >= MIN_ITEMS и blank_url == 0; +ненулевой blank_url — сигнал регрессии адаптера, а не норма. + +Отчёт: data/quality_sources_probe.json (data/ в .gitignore). В прод-БД ничего не пишется. +""" +import argparse +import json +import re +import sys +import time +import urllib.parse +import xml.etree.ElementTree as ET +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from omni import net # noqa: E402 +from omni.sources import Source # noqa: E402 + +OUT = ROOT / "data" / "quality_sources_probe.json" +OUT_POOL = ROOT / "data" / "quality_sources_pool.json" +MIN_ITEMS = 2 +TITLE_OVERLAP_REJECT = 0.6 +ENTRY_OVERLAP_REJECT = 0.5 +TIMEOUT = 12 + +GH = """huggingface/diffusers +huggingface/peft +huggingface/accelerate +huggingface/text-generation-inference +huggingface/tokenizers +huggingface/candle +pytorch/pytorch +pytorch/torchao +NVIDIA/TensorRT-LLM +NVIDIA/nvidia-container-toolkit +flashinfer-ai/flashinfer +microsoft/onnxruntime +microsoft/DeepSpeed +ml-explore/mlx +ggml-org/whisper.cpp +openai/whisper +AUTOMATIC1111/stable-diffusion-webui +exo-explore/exo +vllm-project/vllm-ascend +caddyserver/caddy +keycloak/keycloak +grafana/grafana +tailscale/tailscale +envoyproxy/envoy +hashicorp/vault +hashicorp/terraform +hashicorp/consul +sigstore/cosign +aquasecurity/trivy +anchore/grype +projectdiscovery/nuclei +zaproxy/zaproxy +moby/moby +containerd/containerd +opencontainers/runc +kubernetes/kubernetes +istio/istio +argoproj/argo-cd +cert-manager/cert-manager +traefik/traefik +prometheus/prometheus +opensearch-project/OpenSearch +elastic/elasticsearch +minio/minio +go-gitea/gitea +nextcloud/server +portainer/portainer +wazuh/wazuh +OISF/suricata +zeek/zeek +fail2ban/fail2ban +openssl/openssl +openssh/openssh-portable +curl/curl +spring-projects/spring-boot +quarkusio/quarkus +jenkinsci/jenkins +immich-app/immich +netbox-community/netbox +open-telemetry/opentelemetry-collector +modelcontextprotocol/python-sdk +openai/openai-agents-python +pydantic/pydantic-ai +langchain-ai/langgraph""" + +# имя = gh_rel_ без дефисов/точек (как в реестре: gh_rel_llamacpp, gh_rel_transformers) +def _gh_name(repo): + return "gh_rel_" + repo.split("/")[1].lower().replace("-", "").replace(".", "").replace("_", "") + + +_AI_TAG = ("release", "github", "ai") +_SEC_TAG = ("security", "release", "github") + +POOL = [ + {"name": _gh_name(r), "category": "release", "kind": "atom", + "url": "https://github.com/%s/releases.atom" % r, "tags": _AI_TAG, "limit": 10} + for r in GH.split() +] + +POOL += [ + # ---------- официальные security-фиды вендоров/проектов ---------- + {"name": "chrome_releases", "category": "security", "kind": "atom", + "url": "https://chromereleases.googleblog.com/feeds/posts/default", + "tags": ("security", "vendor", "chrome"), "limit": 15}, + {"name": "project_zero", "category": "security", "kind": "atom", + "url": "https://googleprojectzero.blogspot.com/feeds/posts/default", + "tags": ("security", "research", "google"), "limit": 10}, + {"name": "ubuntu_usn", "category": "security", "kind": "rss", + "url": "https://ubuntu.com/security/notices/rss.xml", + "tags": ("security", "vendor", "ubuntu"), "limit": 20}, + {"name": "amazon_linux_alas", "category": "security", "kind": "rss", + "url": "https://alas.aws.amazon.com/alas.rss", + "tags": ("security", "vendor", "aws"), "limit": 20}, + {"name": "msrc_blog", "category": "security", "kind": "rss", + "url": "https://msrc.microsoft.com/blog/feed", + "tags": ("security", "vendor", "microsoft"), "limit": 15}, + {"name": "ms_security_blog", "category": "security", "kind": "rss", + "url": "https://www.microsoft.com/en-us/security/blog/feed/", + "tags": ("security", "vendor", "microsoft"), "limit": 15}, + {"name": "cisco_talos", "category": "security", "kind": "rss", + "url": "https://blog.talosintelligence.com/rss/", + "tags": ("security", "research", "cisco"), "limit": 15}, + {"name": "portswigger_research", "category": "security", "kind": "rss", + "url": "https://portswigger.net/research/rss", + "tags": ("security", "research", "web"), "limit": 15}, + {"name": "trail_of_bits", "category": "security", "kind": "rss", + "url": "https://blog.trailofbits.com/feed/", + "tags": ("security", "research"), "limit": 15}, + {"name": "cisa_advisories", "category": "security", "kind": "rss", + "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml", + "tags": ("security", "vendor", "cisa"), "limit": 20}, + {"name": "sans_isc", "category": "security", "kind": "rss", + "url": "https://isc.sans.edu/rssfeed.xml", + "tags": ("security", "research", "sans"), "limit": 15}, + {"name": "nodejs_blog", "category": "release", "kind": "rss", + "url": "https://nodejs.org/en/feed/blog.xml", + "tags": ("release", "vendor", "nodejs"), "limit": 15}, + {"name": "rust_blog", "category": "release", "kind": "atom", + "url": "https://blog.rust-lang.org/feed.xml", + "tags": ("release", "vendor", "rust"), "limit": 15}, + {"name": "go_blog", "category": "release", "kind": "atom", + "url": "https://go.dev/blog/feed.atom", + "tags": ("release", "vendor", "go"), "limit": 15}, + {"name": "php_news", "category": "release", "kind": "atom", + "url": "https://www.php.net/feed.atom", + "tags": ("release", "vendor", "php"), "limit": 15}, + {"name": "python_insider", "category": "release", "kind": "atom", + "url": "https://pythoninsider.blogspot.com/feeds/posts/default", + "tags": ("release", "vendor", "python"), "limit": 15}, + {"name": "gitlab_releases", "category": "release", "kind": "atom", + "url": "https://about.gitlab.com/atom.xml", + "tags": ("release", "vendor", "gitlab"), "limit": 15}, + {"name": "cloudflare_security", "category": "security", "kind": "rss", + "url": "https://blog.cloudflare.com/tag/security/rss/", + "tags": ("security", "vendor", "cloudflare"), "limit": 15}, + {"name": "aws_security_bulletins", "category": "security", "kind": "rss", + "url": "https://aws.amazon.com/security/security-bulletins/rss/feed/", + "tags": ("security", "vendor", "aws"), "limit": 20}, + {"name": "hashicorp_blog", "category": "release", "kind": "rss", + "url": "https://www.hashicorp.com/blog/feed.xml", + "tags": ("release", "vendor", "hashicorp"), "limit": 15}, + {"name": "kubernetes_blog", "category": "release", "kind": "rss", + "url": "https://kubernetes.io/feed.xml", + "tags": ("release", "vendor", "kubernetes"), "limit": 15}, + {"name": "docker_blog", "category": "release", "kind": "rss", + "url": "https://www.docker.com/feed/", + "tags": ("release", "vendor", "docker"), "limit": 15}, + # ---------- технические/AI release-блоги вендоров ---------- + {"name": "nvidia_dev_blog", "category": "release", "kind": "rss", + "url": "https://developer.nvidia.com/blog/feed/", + "tags": ("release", "vendor", "nvidia", "ai"), "limit": 15}, + {"name": "pytorch_blog", "category": "release", "kind": "rss", + "url": "https://pytorch.org/feed/", + "tags": ("release", "vendor", "pytorch", "ai"), "limit": 15}, + {"name": "vllm_blog", "category": "release", "kind": "rss", + "url": "https://blog.vllm.ai/feed.xml", + "tags": ("release", "vendor", "vllm", "ai"), "limit": 15}, + {"name": "ray_blog", "category": "release", "kind": "rss", + "url": "https://www.anyscale.com/blog/rss.xml", + "tags": ("release", "vendor", "ray", "ai"), "limit": 15}, + {"name": "langchain_blog", "category": "release", "kind": "rss", + "url": "https://blog.langchain.dev/rss/", + "tags": ("release", "vendor", "langchain", "ai"), "limit": 15}, + {"name": "modal_blog", "category": "release", "kind": "rss", + "url": "https://modal.com/blog/feed.xml", + "tags": ("release", "vendor", "modal", "ai"), "limit": 15}, + {"name": "ollama_blog", "category": "release", "kind": "rss", + "url": "https://ollama.com/blog/rss.xml", + "tags": ("release", "vendor", "ollama", "ai"), "limit": 15}, +] + +BAD_TITLE_MARKS = ("just a moment", "attention required", "access denied", "403 forbidden", + "404 not found", "not found", "rate limit", "too many requests", + "checking your browser", "ddos-guard", "domain not found", + "privacy error", "service unavailable", "bad gateway", "error") + + +def _norm_url(u): + u = (u or "").strip().lower().rstrip("/") + for pre in ("https://", "http://"): + if u.startswith(pre): + u = u[len(pre):] + return u[4:] if u.startswith("www.") else u + + +def _valid_url(u): + try: + p = urllib.parse.urlsplit(u or "") + except Exception: + return False + return p.scheme in ("http", "https") and bool(p.netloc) + + +def _bad_title(t): + low = (t or "").strip().lower() + if len(low) < 3: + return True + return any(m in low for m in BAD_TITLE_MARKS) and len(low) < 60 + + +def _parse(text): + """Локальный парсер RSS/Atom -> [(title, link, published)].""" + out = [] + try: + root = ET.fromstring(text.encode("utf-8", "ignore")) + except Exception: + return out + ns = {"a": "http://www.w3.org/2005/Atom"} + items = root.findall(".//item") + if items: + for e in items: + link = (e.findtext("link") or "").strip() + if not link: + ge = e.find("{http://www.w3.org/2005/Atom}link") + if ge is not None: + link = (ge.get("href") or "").strip() + out.append(((e.findtext("title") or "").strip(), link, + (e.findtext("pubDate") or e.findtext("published") or "").strip())) + return out + for e in root.findall(".//a:entry", ns): + link = "" + for le in e.findall("a:link", ns): + if le.get("rel") in (None, "alternate") and le.get("href"): + link = le.get("href").strip() + if le.get("rel") == "alternate": + break + out.append(((e.findtext("a:title", namespaces=ns) or "").strip(), link, + (e.findtext("a:published", namespaces=ns) or e.findtext("a:updated", namespaces=ns) or "").strip())) + return out + + +def _registry(): + from omni import sources as reg + rows = reg.all_sources() + names, urls, repos = set(), set(), set() + for s in rows: + names.add((s.name or "").strip().lower()) + urls.add(_norm_url(s.url)) + m = re.search(r"github\.com/([^/]+)/([^/?#]+)", (s.url or ""), re.I) + if m: + repos.add((m.group(1).lower(), m.group(2).lower().removesuffix(".git"))) + return rows, names, urls, repos + + +def _repo_of(url): + m = re.search(r"github\.com/([^/]+)/([^/?#]+)", url or "", re.I) + return (m.group(1).lower(), m.group(2).lower().removesuffix(".git")) if m else None + + +def _probe(spec): + t0 = time.time() + r = net.get(spec["url"], timeout=TIMEOUT) + status = {"name": spec["name"], "category": spec["category"], "kind": spec["kind"], + "url": spec["url"], "tags": list(spec["tags"]), "adapter": spec.get("adapter", "rss"), + "limit": spec.get("limit", 10), "http": getattr(r, "status", 0), + "mode": getattr(r, "mode", ""), "bytes": len(getattr(r, "text", "") or ""), + "ms": int((time.time() - t0) * 1000), "err": ""} + if getattr(r, "error", ""): + status["err"] = str(r.error)[:160] + if getattr(r, "status", 0) != 200 or not (r.text or "").strip(): + status["entries"] = 0 + status["reject"] = "http_%s" % getattr(r, "status", 0) + return status + parsed = _parse(r.text) + good = [(t, u, p) for (t, u, p) in parsed if _valid_url(u) and not _bad_title(t)] + status["entries"] = len(parsed) + status["valid"] = len(good) + status["samples"] = [{"title": t[:140], "url": u[:220], "published": p[:32]} for (t, u, p) in good[:3]] + if len(good) < MIN_ITEMS: + status["reject"] = "only_%d_valid" % len(good) + else: + status["reject"] = "" + return status + + +def _fetch_check(spec): + """Что вернул штатный адаптер (документируем эффект бага Atom-link).""" + from omni import feeds + src = Source(spec["name"], spec["category"], spec["kind"], spec["url"], + spec.get("adapter", "rss"), tuple(spec["tags"]), spec.get("limit", 10)) + t0 = time.time() + try: + items = feeds.fetch(src, spec.get("limit", 10)) or [] + err = "" + except Exception as e: # noqa: BLE001 + items, err = [], "raise:%s" % str(e)[:120] + blank = sum(1 for i in items if not _valid_url(i.url)) + return {"n": len(items), "blank_url": blank, "ms": int((time.time() - t0) * 1000), + "err": err, "samples": [{"title": (i.title or "")[:120], "url": (i.url or "")[:200]} + for i in items[:3]]} + + +def _dup_reasons(status, names, urls, repos, accepted): + why = [] + name = (status["name"] or "").lower() + if name in names: + why.append("name_in_registry") + if _norm_url(status["url"]) in urls: + why.append("url_in_registry") + repo = _repo_of(status["url"]) + if repo and repo in repos: + why.append("repo_mirror:%s/%s" % repo) + entry_urls = {_norm_url(s["url"]) for s in status.get("samples", [])} + titles = {re.sub(r"\W+", "", s["title"].lower()) for s in status.get("samples", [])} + for prev in accepted: + p_urls = {_norm_url(s["url"]) for s in prev.get("samples", [])} + p_titles = {re.sub(r"\W+", "", s["title"].lower()) for s in prev.get("samples", [])} + if entry_urls and p_urls and len(entry_urls & p_urls) / len(entry_urls) >= ENTRY_OVERLAP_REJECT: + why.append("entries_overlap:%s" % prev["name"]) + break + if titles and p_titles and len(titles & p_titles) / len(titles) >= TITLE_OVERLAP_REJECT: + why.append("titles_overlap:%s" % prev["name"]) + break + return why + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--only", default="") + ap.add_argument("--workers", type=int, default=8) + ap.add_argument("--limit", type=int, default=10) + ap.add_argument("--confirm", action="store_true", help="прогон SPECS из omni.sources_quality") + args = ap.parse_args() + + if args.confirm: + from omni.sources_quality import SPECS as pool + else: + pool = POOL + if args.only: + pool = [p for p in pool if args.only in p["name"]] + + rows, names, urls, repos = _registry() + # после интеграции quality_sources в all_sources() эти же записи есть в реестре: + # себя из множества дублей вычитаем, иначе confirm-прогон «отклонит» все свои же источники + names -= {(p["name"] or "").lower() for p in pool} + urls -= {_norm_url(p["url"]) for p in pool} + repos -= {r for r in (_repo_of(p["url"]) for p in pool) if r} + t0 = time.time() + with ThreadPoolExecutor(max_workers=args.workers) as ex: + probed = list(ex.map(_probe, pool)) + elapsed = round(time.time() - t0, 1) + + accepted, rejected = [], [] + for st in probed: + if st["reject"]: + st["dup"] = [] + rejected.append(st) + continue + why = _dup_reasons(st, names, urls, repos, accepted) + st["dup"] = why + (rejected if why else accepted).append(st) + + with ThreadPoolExecutor(max_workers=4) as ex: + for st, chk in zip(accepted, ex.map(_fetch_check, accepted)): + st["feeds_fetch"] = chk + + report = { + "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "mode": "confirm" if args.confirm else "pool", + "pool": len(pool), "raw_pool_size": len(POOL), "registry_sources": len(rows), + "module": "omni/sources_quality.py" if args.confirm else None, + "elapsed_s": elapsed, "min_items": MIN_ITEMS, + "acceptance": { + "min_valid_entries": MIN_ITEMS, + "valid_url_required": "http(s) с непустым netloc", + "bad_title_marks": list(BAD_TITLE_MARKS), + "dup_keys": ["name", "normalized_url", "github_repo_mirror", + "entries_overlap>=%.1f" % ENTRY_OVERLAP_REJECT, + "titles_overlap>=%.1f" % TITLE_OVERLAP_REJECT], + }, + "accepted_n": len(accepted), "rejected_n": len(rejected), + "accepted_by_category": {}, + "accepted": accepted, "rejected": rejected, + } + for st in accepted: + report["accepted_by_category"][st["category"]] = report["accepted_by_category"].get(st["category"], 0) + 1 + + target = OUT if args.confirm else OUT_POOL + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") + + print("pool=%d accepted=%d rejected=%d elapsed=%.1fs" % (len(pool), len(accepted), len(rejected), elapsed)) + for st in accepted: + s = st["samples"][0] if st["samples"] else {"title": "", "url": ""} + ff = st.get("feeds_fetch") or {} + print("OK %-30s %2d/%2d %5dms fetch=%d(blank %d) | %s | %s" + % (st["name"], st["valid"], st["entries"], st["ms"], ff.get("n", 0), + ff.get("blank_url", 0), s["title"][:60], s["url"][:80])) + for st in rejected: + print("NO %-30s %s %s" % (st["name"], st.get("reject") or ",".join(st.get("dup") or []), + (st.get("samples") or [{}])[0].get("title", "")[:50])) + print("report:", target) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/run_fresh.py b/tools/run_fresh.py index 6f05790..abac14f 100644 --- a/tools/run_fresh.py +++ b/tools/run_fresh.py @@ -23,10 +23,16 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=10) ap.add_argument("--workers", type=int, default=8) - ap.add_argument("--out", default="data/fresh_run.json") + ap.add_argument("--wave", choices=("fresh", "quality"), default="fresh") + ap.add_argument("--out", default="") a = ap.parse_args() - names = {f.name for f in _fresh(_Fake)} + if a.wave == "quality": + from omni.sources_quality import quality_sources + names = {f.name for f in quality_sources(_Fake)} + else: + names = {f.name for f in _fresh(_Fake)} + output = a.out or "data/%s_run.json" % a.wave picked = [s for s in sources.all_sources() if s.name in names] print("источников свежей волны:", len(picked), "| лимит", a.limit) t0 = time.time() @@ -45,7 +51,8 @@ def main(): out = {"when": int(t0), "secs": round(time.time() - t0, 1), "picked": len(picked), "alive": len(ok), "empty": len(bad), "fetched": len(items), "new": new, "dup": dup, "by_source": dict(sorted(by_src.items(), key=lambda x: -x[1])), "empty_sources": bad} - json.dump(out, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1) + with open(output, "w", encoding="utf-8") as f: + json.dump(out, f, ensure_ascii=False, indent=1) print("секунд:", out["secs"], "| отдали:", len(ok), "| пустых:", len(bad)) print("получено записей:", len(items), "| новых в базе:", new, "| дублей:", dup) print("топ:", list(out["by_source"].items())[:12]) diff --git a/tools/shot_abuse.py b/tools/shot_abuse.py index ce1d048..4a169f7 100644 --- a/tools/shot_abuse.py +++ b/tools/shot_abuse.py @@ -9,7 +9,7 @@ with Camoufox(headless=True, geoip=False, humanize=True) as b: p = b.new_page(viewport={"width": 1400, "height": 1000}) p.goto(URL, wait_until="domcontentloaded", timeout=60000) p.wait_for_selector("#view", timeout=60000) - p.wait_for_timeout(7000) + p.wait_for_timeout(14000) txt = p.inner_text("#view")[:900] p.screenshot(path=OUT, full_page=False) print(txt) diff --git a/web/index.html b/web/index.html index 258594a..442a2d6 100644 --- a/web/index.html +++ b/web/index.html @@ -210,6 +210,11 @@ label.chk{display:inline-flex;align-items:center;gap:6px;color:var(--muted);font .top{padding:8px 10px} .crumb{display:none} } +.pn-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(190px,1fr));gap:8px;margin-top:8px} +.pn{display:flex;flex-direction:column;gap:2px;padding:8px 10px;border:1px solid var(--line);border-radius:10px;text-decoration:none;color:inherit;font-size:12px} +.pn:hover{border-color:var(--acc)} +.pn b{font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.pn .muted{opacity:.6} @@ -261,6 +266,11 @@ const dt=s=>{if(!s)return"";s=String(s);const m=s.match(/^(\d{4})-(\d{2})-(\d{2} const ago=s=>{if(!s)return"";const t=Date.parse(!isNaN(s)?+s:s);if(!t)return dt(s); const d=(Date.now()-t)/1000;if(d<3600)return Math.max(1,Math.round(d/60))+" мин";if(d<86400)return Math.round(d/3600)+" ч";return Math.round(d/86400)+" дн"}; const debounce=(f,ms=220)=>{let t;return(...a)=>{clearTimeout(t);t=setTimeout(()=>f(...a),ms)}}; +/* статус прогона: "ok*" — успех; "пусто"/"empty" — пустой ответ (не ошибка); "" — нет статуса */ +const isOkSt=s=>String(s==null?"":s).trim().toLowerCase().startsWith("ok"); +const isEmpSt=s=>["пусто","empty"].includes(String(s==null?"":s).trim().toLowerCase()); +const stKey=s=>{const v=String(s==null?"":s).trim();return !v?"none":(isOkSt(v)?"ok":(isEmpSt(v)?"empty":"err"))}; +const stPill=s=>{const k=stKey(s);return k==="ok"?"ok":(k==="err"?"no":"unk")}; let TOAST_T; function toast(msg,bad=false){const el=$("#toast");el.textContent=msg;el.classList.toggle("bad",!!bad);el.classList.add("on");clearTimeout(TOAST_T);TOAST_T=setTimeout(()=>el.classList.remove("on"),2600);} @@ -303,9 +313,9 @@ const NAV=[ {id:"digest", label:"Дайджест", icon:"M6 3h9l4 4v14H6zM15 3v4h4"}, {id:"watch", label:"Watchlist", icon:"M2 12s3.5-6 10-6 10 6 10 6-3.5 6-10 6S2 12 2 12zM12 9a3 3 0 1 0 0 6 3 3 0 0 0 0-6z"} ]; -const S={view:"overview",q:"",cat:"",type:"all",sort:"score",days:0,priced:false, +const S={view:"overview",q:"",cat:"",type:"all",sort:"score",days:0,priced:false,src:"", offset:0,items:[],total:0,auto:localStorage.getItem("omni.auto")==="1", - types:[],cats:[]}; + runQ:"",runF:"all",types:[],cats:[]}; function route(){ const h=location.hash.replace(/^#\/?/,""); @@ -314,6 +324,7 @@ function route(){ S.view=NAV.some(n=>n.id===v)?v:"overview"; S.q=p.get("q")||""; S.cat=p.get("cat")||""; S.type=p.get("type")||"all"; S.sort=p.get("sort")||"score"; S.days=+(p.get("days")||0); S.priced=p.get("priced")==="1"; + S.src=p.get("src")||""; S.offset=0; $("#q").value=S.q; $(".crumb").innerHTML=""+esc(NAV.find(n=>n.id===S.view).label)+""+(S.q?` · «${esc(S.q)}»`:""); @@ -328,6 +339,7 @@ function go(view,params={},replace=false){ const sort=params.sort??S.sort; if(sort!=="score")p.set("sort",sort); const days=params.days??S.days; if(days)p.set("days",days); if((params.priced??S.priced))p.set("priced","1"); + const src=params.src!==undefined?params.src:S.src; if(src)p.set("src",src); const url="#/"+view+(p.toString()?"?"+p:""); if(replace)history.replaceState(null,"",url); else location.hash=url; if(replace)route(); @@ -376,10 +388,12 @@ V.overview=async()=>{ api("/api/watch").catch(()=>({rules:[]})), api("/api/abuse?limit=8").catch(()=>({items:[]})) ]); S.stats=st; - const day=tl.last24||tl.points[tl.points.length-1]?.n||0, prev=tl.points[tl.points.length-2]?.n||0; - const dPct=prev?Math.round((day-prev)/prev*100):null; + // last24 — честное скользящее окно от бэкенда; «сегодня» неполный день, процент не считаем + const day=tl.last24??0; const live=(st.sources?(st.alive/st.sources*100):0).toFixed(0); - const okRate=st.runs?(st.runs_ok/st.runs*100).toFixed(0):0; + const bs=runs.by_status||{}; + const runAll=(bs.ok||0)+(bs.empty||0)+(bs.err||0)+(bs.none||0); + const okRate=runAll?Math.round((bs.ok||0)/runAll*100):(st.runs?Math.round(st.runs_ok/st.runs*100):0); const kpi=(label,val,sub,accent)=>`
${label}
${val}
${sub}
`; const hits=(watch.rules||[]).reduce((a,r)=>a+(r.count||0),0); @@ -390,7 +404,7 @@ V.overview=async()=>{
- ${kpi("Записей всего",num(st.items),`+${num(day)} за сутки импорта${dPct===null?"":` ${dPct>=0?"▲":"▼"} ${Math.abs(dPct)}%`}`)} + ${kpi("Записей всего",num(st.items),`за последние 24 часа: +${num(day)}`)} ${kpi("Источников живых",`${num(st.alive)}/ ${num(st.sources)}`,`${live}% реестра отдают записи`)} ${kpi("Ключей готово",`${num(st.keys?.ready||0)}/ ${num(st.keys?.keys||0)}`,`${num(st.keys?.services||0)} сервисов · ротация и остывание`)} ${kpi("Watchlist",num((watch.rules||[]).length),`${num(hits)} совпадений по правилам`)} @@ -413,10 +427,10 @@ V.overview=async()=>{

Свежее вся лента →

${skel(4)}
-

Последние прогоны все →

+

Последние прогоны ok ${num(okRate)}% · все →

${(runs.items||[]).map(r=>` - `).join("")}
${esc(r.when||"")} ${esc(r.source||"—")}${r.got??""}/${r.kept??""}${esc(r.status||"")}
+ ${esc(r.status||"—")}`).join("")}
@@ -441,8 +455,8 @@ V.feed=async()=>{ const box=$("#view"); box.innerHTML=`

Лента

Все записи реестра: фильтры по типу, периоду, сортировке; клик — детали и соседи.

- CSV - JSON + CSV + JSON
Типы: @@ -456,12 +470,15 @@ V.feed=async()=>{
${skel(6)}
`; await loadItems(true); }; -const exQS=()=>new URLSearchParams({q:S.q,category:S.cat,type:S.type,sort:S.sort,...(S.priced?{priced:"1"}:{})}).toString(); +const exQS=()=>new URLSearchParams({q:S.q,category:S.cat,type:S.type,sort:S.sort, + ...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{}),...(S.src?{src:S.src}:{}), + limit:"1000"}).toString(); async function loadItems(reset){ if(reset){S.offset=0;S.items=[]} const p=new URLSearchParams({limit:"40",offset:String(S.offset),q:S.q,sort:S.sort,type:S.type, - ...(S.cat?{category:S.cat}:{}),...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{})}); + ...(S.cat?{category:S.cat}:{}),...(S.days?{days:String(S.days)}:{}),...(S.priced?{priced:"1"}:{}), + ...(S.src?{src:S.src}:{})}); const d=await api("/api/items?"+p); S.total=d.total; S.items=S.items.concat(d.items); S.offset+=d.items.length; const list=$("#list"); if(!list)return; @@ -475,8 +492,8 @@ function findChips(f){ if(!f) return ""; const p=[]; (f.codes||[]).slice(0,2).forEach(c=>p.push(`code: ${esc(c)}`)); - if(f.amount) p.push(`${esc(f.amount)}`); - if(f.term) p.push(`${esc(f.term)}`); + if(f.amount) p.push(`${esc(f.amount)}`); + if(f.term) p.push(`${esc(f.term)}`); return p.join(""); } function itemRow(it,compact){ @@ -536,6 +553,7 @@ async function openItem(id){ function closeDrawer(){$("#drawer").classList.remove("on");$("#backdrop").classList.remove("on");} /* ---------- abuse ---------- */ + V.abuse=async()=>{ const d=await api("/api/abuse?limit=120"); const tags=Object.entries(d.tags||{}).slice(0,14); @@ -547,9 +565,24 @@ V.abuse=async()=>{ ${tags.map(([t,n])=>`${esc(d.labels?.[t]||t)} ${num(n)}`).join("")}
${(d.items||[]).map(it=>itemRow(it)).join("")||`
пока пусто
`}
+
`; S.abuse=(d.items||[]); + loadPanels(); }; +async function loadPanels(){ + const box=$("#pn-box"); if(!box) return; + try{ + const d=await api("/api/panels"); + const all=(d.panels||[]).filter(p=>(p.free_count||p.free||0)>0); + const rows=all.slice(0,40); + if(!rows.length){ box.innerHTML=""; return; } + box.innerHTML=`

Free-панели new-api всего ${num(d.total)} · с бесплатными ${num(all.length)}${all.length>rows.length?` · показано ${num(rows.length)}`:""}

+
${rows.map(p=>` + ${esc(p.host||p.url||"?")}${num(p.free_count||p.free||0)} free + ${num(p.models||p.model_count||0)} моделей`).join("")}
`; + }catch(e){ box.innerHTML=""; } +} async function abuseByTag(tag){ const d=await api("/api/abuse?limit=60"+(tag?"&tag="+encodeURIComponent(tag):"")); const box=$("#ab-list"); if(box) box.innerHTML=(d.items||[]).map(it=>itemRow(it)).join("")||`
пусто
`; @@ -644,31 +677,36 @@ async function srcDetail(name){ /* ---------- runs ---------- */ V.runs=async()=>{ const d=await api("/api/runs?limit=200"); - const ok=(d.items||[]).filter(r=>r.status==="ok").length; + const bs=d.by_status||{}; + const items=d.items||[]; + const ok=bs.ok??items.filter(r=>isOkSt(r.status)).length; // "ok", "ok-304" — успех + const empty=bs.empty??items.filter(r=>stKey(r.status)==="empty").length; + const err=bs.err??items.filter(r=>stKey(r.status)==="err").length; + const none=bs.none??items.filter(r=>stKey(r.status)==="none").length; + const rf=(k,l,n)=>`${l} ${num(n)}`; $("#view").innerHTML=`

Прогоны

Журнал сборов: что запросили, сколько получили и сколько нового легло в базу.

- все ${num(d.total)} - ok ${ok} - ошибки ${d.total-ok}
+ ${rf("all","все",d.total)}${rf("ok","ok",ok)}${empty?rf("empty","пусто",empty):""}${rf("err","ошибки",err)}${none?rf("none","без статуса",none):""}
- - всего ${num(d.total)}
+ + всего ${num(d.total)} · показано ${num(items.length)}
- ${(d.items||[]).map(r=>` + ${items.map(r=>` - `).join("")}
когдаисточникзапросgotkeptбайтстатус
${esc(r.when||"")}${esc(r.source||"—")}
${esc(r.when||"")}${esc(r.source||"—")} ${esc(r.query||"")} ${r.got??""}${r.kept??""}${r.bytes?num(r.bytes):""}${esc(r.status||"")}
`; - S.runQ=""; S.runF="all"; + ${esc(r.status||"—")}`).join("")}`; + // S.runQ/S.runF не сбрасываем: автообновление перерисовывает раздел каждые 60 с const applyRuns=()=>{const v=(S.runQ||"").toLowerCase(); $$("#r-tbl tbody tr").forEach(tr=>{ const okT=!v||tr.textContent.toLowerCase().includes(v); - const okS=S.runF==="all"||(S.runF==="ok"?tr.dataset.st==="ok":tr.dataset.st!=="ok"); + const okS=S.runF==="all"||S.runF===tr.dataset.st; tr.style.display=(okT&&okS)?"":"none";});}; $("#r-q").addEventListener("input",debounce(e=>{S.runQ=e.target.value;applyRuns()},120)); $$("#view [data-rf]").forEach(c=>c.addEventListener("click",()=>{ $$("#view [data-rf]").forEach(x=>x.classList.remove("on"));c.classList.add("on");S.runF=c.dataset.rf;applyRuns();})); + applyRuns(); }; /* ---------- telegram ---------- */