reform/omni-parse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: типизация записей (14 типов, до 3 на запись), чипы и бейджи типов в панели, CSV с типами, /api/types, doctor --only по категориям, волны источников 5-6 (635 всего)

Browse Source
This commit is contained in:
Vlad Gothbreach
2026-10-26 19:01:45 +03:00
parent 4196d66b4e
commit 7b3d708402
10 changed files with 652 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
omni/doctor.py
View File

@@ -86,7 +86,11 @@ def fix(src):
def run(only=None, do_fix=False, workers=8, limit=3):
from . import source_state
picked = sources.pick(None, only, None)
only = (only or "").strip() or None
if only and only in sources.CATEGORIES:
picked = sources.pick(category=only, respect_enabled=False)
else:
picked = sources.pick(only=only, respect_enabled=False)
cache = load_cache()
results = []
with ThreadPoolExecutor(max_workers=workers) as ex:

37
omni/serve.py
View File

@@ -204,6 +204,12 @@ class Handler(BaseHTTPRequestHandler):
"categories": cats,
})
if path == "/api/types":
from . import taxonomy
cnt = store.type_counts()
items = [{"key": k, "label": taxonomy.LABELS.get(k, k), "n": v} for k, v in cnt.items()]
return self._json({"ok": True, "types": items, "total": sum(cnt.values())})
if path == "/api/runs":
try:
limit = max(1, min(500, int((q.get("limit") or ["80"])[0])))
@@ -299,13 +305,42 @@ class Handler(BaseHTTPRequestHandler):
limit = min(int((q.get("limit") or ["40"])[0]), 200)
offset = int((q.get("offset") or ["0"])[0])
sort = (q.get("sort") or ["score"])[0]
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort)
priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
typ = (q.get("type") or ["all"])[0]
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
sort=sort, priced=priced, typ=typ)
for _r in rows:
_r["title"] = _clean_txt(_r.get("title"), 180)
_r["text"] = _clean_txt(_r.get("text"), 260)
_r["published"] = _fmt_date(_r.get("published"))
return self._json({"count": len(rows), "total": total, "offset": offset, "items": rows})
if path == "/api/export.csv":
import csv as _csv
cat = (q.get("category") or [None])[0]
query = (q.get("q") or [""])[0].strip()
priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
sort = (q.get("sort") or ["score"])[0]
typ = (q.get("type") or ["all"])[0]
rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=5000, offset=0,
sort=sort, priced=priced, typ=typ)
import io as _io
buf = _io.StringIO()
w = _csv.writer(buf, delimiter=";")
w.writerow(["source", "kind", "types", "title", "author", "published", "score", "price", "currency", "url"])
for r in rows:
w.writerow([r.get("source"), r.get("kind"), r.get("types"), (r.get("title") or "").replace(chr(10), " "),
r.get("author"), r.get("published"), r.get("score"), r.get("price"),
r.get("currency"), r.get("url")])
data = buf.getvalue().encode("utf-8-sig")
self.send_response(200)
self.send_header("Content-Type", "text/csv; charset=utf-8")
self.send_header("Content-Disposition", 'attachment; filename="omni.csv"')
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
if path.startswith("/api/job/"):
jid = path.rsplit("/", 1)[-1]
return self._json(JOBS.get(jid) or {"state": "unknown"})

4
omni/sources.py
View File

@@ -14,6 +14,8 @@ from .sources_ext import EXTRA
from .sources_ext2 import EXTRA2
from .sources_ext3 import EXTRA3
from .sources_abuse import ABUSE_LIVE
from .sources_ext4 import EXTRA5
from .sources_ext5 import EXTRA6
@dataclass
@@ -197,7 +199,7 @@ def _dedup(rows):
return out
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4])
SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6])
BY_NAME = {s.name: s for s in SOURCES}
CATEGORIES = sorted({s.category for s in SOURCES})

47
omni/sources_ext4.py Normal file
View File

@@ -0,0 +1,47 @@
"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):
security/AI/форумы/reddit/tg/релизы/CN. Дата прогона 2026-10-26, живых 37."""
def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
EXTRA5 = [
S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed.xml", "rss", ('security', 'research'), limit=15),
S("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ('security', 'research'), limit=15),
S("hackread", "security", "rss", "https://www.hackread.com/feed/", "rss", ('security', 'research'), limit=15),
S("thecyberwire", "security", "rss", "https://thecyberwire.com/feeds/rss.xml", "rss", ('security', 'research'), limit=15),
S("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ('security', 'research'), limit=15),
S("wiz_blog", "security", "rss", "https://www.wiz.io/feed/rss.xml", "rss", ('security', 'research'), limit=15),
S("aquasec", "security", "rss", "https://blog.aquasec.com/rss.xml", "rss", ('security', 'research'), limit=15),
S("crowdstrike", "security", "rss", "https://www.crowdstrike.com/blog/feed/", "rss", ('security', 'research'), limit=15),
S("yeswehack", "security", "rss", "https://www.yeswehack.com/feed", "rss", ('security', 'research'), limit=15),
S("deepmind_blog", "company", "rss", "https://deepmind.google/blog/rss.xml", "rss", ('ai', 'blog'), limit=15),
S("the_decoder", "company", "rss", "https://the-decoder.com/feed/", "rss", ('ai', 'blog'), limit=15),
S("bair_blog", "company", "rss", "https://bair.berkeley.edu/blog/feed.xml", "rss", ('ai', 'blog'), limit=15),
S("nvidia_blog", "company", "rss", "https://blogs.nvidia.com/feed/", "rss", ('ai', 'blog'), limit=15),
S("karpathy", "company", "rss", "https://karpathy.github.io/feed.xml", "rss", ('ai', 'blog'), limit=15),
S("alignment_forum", "company", "rss", "https://www.alignmentforum.org/feed.xml", "rss", ('ai', 'blog'), limit=15),
S("lesswrong", "company", "rss", "https://www.lesswrong.com/feed.xml", "rss", ('ai', 'blog'), limit=15),
S("arxiv_cs_ai", "papers", "rss", "https://export.arxiv.org/rss/cs.AI", "rss", ('papers', 'arxiv'), limit=15),
S("arxiv_cs_cr", "papers", "rss", "https://export.arxiv.org/rss/cs.CR", "rss", ('papers', 'arxiv'), limit=15),
S("arxiv_cs_cl", "papers", "rss", "https://export.arxiv.org/rss/cs.CL", "rss", ('papers', 'arxiv'), limit=15),
S("arxiv_cs_lg", "papers", "rss", "https://export.arxiv.org/rss/cs.LG", "rss", ('papers', 'arxiv'), limit=15),
S("dc_openwrt", "forum", "json", "https://forum.openwrt.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("tg_seeallochnaya", "community", "html", "https://t.me/s/seeallochnaya", "tg_web", ('telegram',), limit=15),
S("tg_data_secrets", "community", "html", "https://t.me/s/data_secrets", "tg_web", ('telegram',), limit=15),
S("vercel_blog", "company", "rss", "https://vercel.com/blog/rss.xml", "rss", ('infra', 'blog'), limit=15),
S("docker_blog", "company", "rss", "https://www.docker.com/blog/feed/", "rss", ('infra', 'blog'), limit=15),
S("supabase_blog", "company", "rss", "https://supabase.com/blog/rss.xml", "rss", ('infra', 'blog'), limit=15),
S("arxiv_cs_se", "company", "rss", "https://export.arxiv.org/rss/cs.SE", "rss", ('infra', 'blog'), limit=15),
S("arxiv_cs_ir", "company", "rss", "https://export.arxiv.org/rss/cs.IR", "rss", ('infra', 'blog'), limit=15),
S("arxiv_stat_ml", "company", "rss", "https://export.arxiv.org/rss/stat.ML", "rss", ('infra', 'blog'), limit=15),
S("dc_rust_internals", "forum", "json", "https://internals.rust-lang.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("dc_nix", "forum", "json", "https://discourse.nixos.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("dc_godot", "forum", "json", "https://forum.godotengine.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("dc_swift", "forum", "json", "https://forums.swift.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("dc_tryghost", "forum", "json", "https://forum.ghost.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("dc_blender", "forum", "json", "https://devtalk.blender.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
S("tg_gpt_main", "community", "html", "https://t.me/s/gpt_main", "tg_web", ('telegram',), limit=15),
S("tg_neuralshit", "community", "html", "https://t.me/s/neuralshit", "tg_web", ('telegram',), limit=15),
]

18
omni/sources_ext5.py Normal file
View File

@@ -0,0 +1,18 @@
"""Волна 6: CVE-фиды, абуз/SEO-блоги, инфра-блоги. Каждая строка проверена живым fetch (items >= 2)."""
def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
"tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
EXTRA6 = [
S("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",), limit=15),
S("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"), limit=15),
S("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",), limit=15),
S("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"), limit=15),
S("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"), limit=15),
S("tensorflow_blog", "company", "rss", "https://blog.tensorflow.org/feeds/posts/default", "rss", ("infra", "blog"), limit=15),
S("neo4j_blog", "company", "rss", "https://neo4j.com/blog/feed/", "rss", ("infra", "blog"), limit=15),
S("postgres_news", "company", "rss", "https://www.postgresql.org/news.rss", "rss", ("infra", "blog"), limit=15),
]

55
omni/store.py
View File

@@ -4,7 +4,7 @@ import sqlite3
import time
from pathlib import Path
from . import config
from . import config, taxonomy
SCHEMA = """
CREATE TABLE IF NOT EXISTS items (
@@ -43,6 +43,10 @@ def conn():
c.execute("PRAGMA journal_mode=WAL")
c.execute("PRAGMA synchronous=NORMAL")
c.executescript(SCHEMA)
cols = {r[1] for r in c.execute("PRAGMA table_info(items)")}
if "types" not in cols:
c.execute("ALTER TABLE items ADD COLUMN types TEXT")
c.commit()
return c
@@ -85,9 +89,10 @@ def add(items):
try:
cur = c.execute(
"INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
"tags,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
"tags,types,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(it.key(), it.provider, it.kind, it.source, it.url, it.title, it.text, it.author,
it.published, it.lang, ",".join(it.tags), it.price, it.currency, it.score,
it.published, it.lang, ",".join(it.tags), ",".join(taxonomy.classify(it.title, it.text)),
it.price, it.currency, it.score,
_signed(h), json.dumps(it.extra, ensure_ascii=False)[:2000], ts))
except sqlite3.IntegrityError:
dup += 1
@@ -146,11 +151,11 @@ def browse(category=None, q="", limit=60, offset=0):
args += [limit, offset]
with conn() as c:
rows = c.execute(sql, args).fetchall()
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
return [dict(zip(keys, r)) for r in rows]
def browse2(q="", names=None, limit=60, offset=0, sort="score"):
def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None):
"""Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
where = "WHERE 1=1 "
args = []
@@ -160,15 +165,49 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score"):
if q:
where += "AND (title LIKE ? OR text LIKE ?) "
args += [f"%{q}%", f"%{q}%"]
if priced:
where += "AND price IS NOT NULL AND price != '' "
if typ and typ != "all":
where += "AND (',' || COALESCE(types,'') || ',') LIKE ? "
args += [f"%,{typ},%"]
order = {"fresh": "ORDER BY published DESC, id DESC",
"price": "ORDER BY CAST(price AS REAL) DESC, score DESC",
"old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
with conn() as c:
total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url "
"FROM items " + where + ("ORDER BY published DESC, id DESC LIMIT ? OFFSET ?" if sort == "fresh" else "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"),
rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
"FROM items " + where + order + " LIMIT ? OFFSET ?",
args + [limit, offset]).fetchall()
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
return [dict(zip(keys, r)) for r in rows], total
def type_counts():
"""Сколько записей на каждый тип."""
out = {}
with conn() as c:
for row in c.execute("SELECT COALESCE(types,'') FROM items"):
for t in (row[0] or "").split(","):
t = t.strip()
if t:
out[t] = out.get(t, 0) + 1
return dict(sorted(out.items(), key=lambda kv: -kv[1]))
def retype(limit=None):
"""Пересчитать типы у уже сохранённых записей."""
n = 0
with conn() as c:
q = "SELECT id,title,text FROM items"
if limit:
q += " LIMIT %d" % int(limit)
for iid, title, text in c.execute(q).fetchall():
c.execute("UPDATE items SET types=? WHERE id=?",
(",".join(taxonomy.classify(title, text)), iid))
n += 1
return n
def count_by_source():
with conn() as c:
return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall())

92
omni/taxonomy.py Normal file
View File

@@ -0,0 +1,92 @@
# -*- coding: utf-8 -*-
"""Типизация записей: не «категория источника», а тип самого поста.
Категория отвечает «откуда», тип — «что именно»: слив, триал, софт, промо,
перепродажа, CVE, релиз, гайд, датасет, модель, вакансия, цена, вопрос, новость.
Одна запись может иметь несколько типов, порядок — по весу совпадений.
"""
import re
# (key, label, weight, [regex, ...])
RULES = [
("leak", "слив", 10, [
r"\bслив\w*", r"утечк\w+", r"\bleak(ed|s)?\b", r"\bdump(ed)?\b", r"база\s+(данных|слит)",
r"\bcombo(list)?\b", r"stealer\s*log", r"\bdb\s+dump\b", r"пасс\w*\s+от\s+",
]),
("trial", "триал", 9, [
r"\btrial\b", r"триал\w*", r"пробн\w+\s+период", r"free\s+(credits?|tier|plan)",
r"бесплатн\w+\s+(кредит|токен|доступ|период|тариф)", r"\bfreebies?\b", r"щедр\w+\s+лимит",
r"\$\d+\s*(free|бесплат)", r"no\s+credit\s+card",
]),
("tool", "софт", 8, [
r"\bсофт\w*", r"\btool(s|kit)?\b", r"скрипт\w*", r"\bparser\b", r"\bчекер\b", r"\bбот\w*",
r"\bcrawler\b", r"\bscraper\b", r"\bautomation\b", r"автоматизац\w+", r"\bcli\b",
r"\bsdk\b", r"open[\s-]?source\s+(tool|project)",
]),
("promo", "промо", 7, [
r"промокод\w*", r"\bpromo\w*", r"\bcoupon\b", r"скидк\w+", r"\bdiscount\b", r"giveaway",
r"раздач\w+", r"\bairdrop\b", r"бонус\w*\s+(код|за\s+регистрац)", r"-?\d{1,2}%\s*off",
]),
("resale", "перепродажа", 7, [
r"\bпродам\b", r"\bпродаж\w+", r"\bresell\w*", r"\bв\s+аренду\b", r"\bкуплю\b",
r"\bsell(ing)?\s+(account|key|access)", r"перепрод\w+", r"\bоптом\b", r"\bв\s+наличии\b.*\$",
]),
("cve", "уязвимость", 10, [
r"\bCVE-\d{4}-\d{3,7}\b", r"\bvulnerab\w+", r"уязвим\w+", r"\bRCE\b", r"\bXSS\b",
r"\bSQLi?\b", r"\bexploit\w*", r"\b0day\b", r"\bpatch\w*\s+tuesday", r"security\s+advisory",
]),
("release", "релиз", 6, [
r"\brelease[sd]?\b", r"\bрелиз\w*", r"\bchangelog\b", r"\bv\d+\.\d+(\.\d+)?\b",
r"вышл\w+\s+версия", r"\bnow\s+available\b", r"\bGA\b", r"\bpreview\s+build\b",
]),
("guide", "гайд", 5, [
r"\bгайд\w*", r"\btutorial\b", r"\bhow[\s-]to\b", r"инструкц\w+", r"\bруководств\w+",
r"\bкак\s+(сделать|настроить|поднять|получить)", r"\bwalkthrough\b", r"\bcheat\s*sheet\b",
]),
("dataset", "датасет", 6, [
r"\bdataset\w*", r"датасет\w*", r"\bcorpus\b", r"\bbenchmark\s+suite\b", r"\bannotated\b",
]),
("model", "модель", 5, [
r"\bweights\b", r"\bcheckpoint\w*", r"\.gguf\b", r"\.safetensors\b", r"\bllm\b",
r"\bмодел\w+\s+(вышла|обучен| weights)", r"\bquantiz\w+", r"\bfine-?tun\w+",
]),
("job", "вакансия", 6, [
r"\bваканси\w+", r"\bhiring\b", r"\bjob\s+(offer|post)", r"\bremote\b.*\$",
r"\bsalary\b", r"\$\d+\s*/\s*(hr|hour)", r"\bищу\s+работу\b", r"\bfor\s+hire\b",
]),
("price", "цена", 4, [
r"\bцен[аы]\b", r"\bpricing\b", r"\bтариф\w*", r"\$\d", r"\d+\s*₽", r"\d+\s*€",
r"\bper\s+(month|year|1m)\b", r"/\s*1m\s+tokens",
]),
("question", "вопрос", 3, [
r"\?\s*$", r"^\s*(как|почему|что|где|кто|можно\s+ли)\b", r"\bhelp\b", r"\bподскажите\b",
]),
]
COMPILED = [(k, l, w, [re.compile(p, re.I | re.U) for p in pats]) for k, l, w, pats in RULES]
LABELS = {k: l for k, l, _, _ in RULES}
LABELS["news"] = "новость"
KEYS = [k for k, _, _, _ in RULES]
def classify(title, text, limit=3):
"""Список типов записи по убыванию веса. Если ничего не сработало — ['news']."""
t = (title or "") + "\n" + (text or "")[:2000]
if not t.strip():
return ["news"]
hits = []
for key, label, w, pats in COMPILED:
n = 0
for p in pats:
if p.search(t):
n += 1
if n:
hits.append((w + n, key))
if not hits:
return ["news"]
hits.sort(reverse=True)
return [k for _, k in hits[:limit]]
def labels(keys):
return [LABELS.get(k, k) for k in keys or []]

241
tools/wave5.py Normal file
View File

@@ -0,0 +1,241 @@
# -*- coding: utf-8 -*-
"""Волна 5: кандидаты источников -> живая проверка реальным адаптером -> omni/sources_ext4.py.
Пишем только те, что реально отдали >= 2 записи. Никаких "на глаз".
"""
import sys, os, json, concurrent.futures as cf
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
os.chdir(r"C:/Users/User/tmp/omni-parse")
from omni import discover, sources, config
config.load()
C = [] # (name, category, kind, url, adapter, tags)
def s(name, cat, kind, url, ad="rss", tags=()):
C.append((name, cat, kind, url, ad, tuple(tags)))
# ---------- security ----------
for n, u in [
("krebs", "https://krebsonsecurity.com/feed/"),
("thn", "https://feeds.feedburner.com/TheHackersNews"),
("securityweek", "https://www.securityweek.com/feed/"),
("darkreading", "https://www.darkreading.com/rss.xml"),
("portswigger_research", "https://portswigger.net/research/rss"),
("projectzero", "https://googleprojectzero.blogspot.com/feeds/posts/default"),
("talos", "https://blog.talosintelligence.com/rss/"),
("sans_isc", "https://isc.sans.edu/rssfeed.xml"),
("exploitdb", "https://www.exploit-db.com/rss.xml"),
("packetstorm", "https://packetstormsecurity.com/feeds"),
("unit42", "https://unit42.paloaltonetworks.com/feed/"),
("checkpoint_research", "https://research.checkpoint.com/feed/"),
("securelist", "https://securelist.com/feed/"),
("welivesecurity", "https://www.welivesecurity.com/en/rss/feed/"),
("malwarebytes", "https://www.malwarebytes.com/blog/feed/index.xml"),
("sentinelone_labs", "https://www.sentinelone.com/labs/feed/"),
("hackread", "https://www.hackread.com/feed/"),
("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
("gbhackers", "https://gbhackers.com/feed/"),
("cybersecuritynews", "https://cybersecuritynews.com/feed/"),
("securityaffairs", "https://securityaffairs.com/feed"),
("therecord", "https://therecord.media/feed"),
("riskybusiness", "https://risky.biz/feeds/risky-business.xml"),
("trailofbits", "https://blog.trailofbits.com/feed/"),
("snyk_blog", "https://snyk.io/blog/feed/"),
("wiz_blog", "https://www.wiz.io/feed/rss.xml"),
("aquasec", "https://blog.aquasec.com/rss.xml"),
("sysdig", "https://sysdig.com/blog/feed/"),
("crowdstrike", "https://www.crowdstrike.com/blog/feed/"),
("mandiant", "https://www.mandiant.com/resources/blog/rss.xml"),
("recordedfuture", "https://www.recordedfuture.com/feed"),
("intigriti", "https://blog.intigriti.com/feed/"),
("yeswehack", "https://www.yeswehack.com/feed"),
("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
("hackerone_blog", "https://www.hackerone.com/blog.rss"),
]:
s(n, "security", "rss", u, "rss", ("security", "research"))
# ---------- ai / company ----------
for n, u in [
("openai_news", "https://openai.com/news/rss.xml"),
("deepmind_blog", "https://deepmind.google/blog/rss.xml"),
("google_ai_blog", "https://blog.google/technology/ai/rss/"),
("hf_blog", "https://huggingface.co/blog/feed.xml"),
("the_decoder", "https://the-decoder.com/feed/"),
("simonwillison", "https://simonwillison.net/atom/everything/"),
("latent_space", "https://www.latent.space/feed"),
("importai", "https://importai.substack.com/feed"),
("oneusefulthing", "https://www.oneusefulthing.org/feed"),
("bair_blog", "https://bair.berkeley.edu/blog/feed.xml"),
("lilianweng", "https://lilianweng.github.io/feed.xml"),
("jalammar", "https://jalammar.github.io/feed.xml"),
("msr_blog", "https://www.microsoft.com/en-us/research/feed/"),
("aws_ml_blog", "https://aws.amazon.com/blogs/machine-learning/feed/"),
("nvidia_blog", "https://blogs.nvidia.com/feed/"),
("meta_ai_blog", "https://ai.meta.com/blog/rss/"),
("stability_news", "https://stability.ai/news/rss.xml"),
("cohere_blog", "https://cohere.com/blog/rss.xml"),
("mistral_news", "https://mistral.ai/news/rss.xml"),
("together_blog", "https://www.together.ai/blog/rss.xml"),
("eleuther_blog", "https://blog.eleuther.ai/feed.xml"),
("karpathy", "https://karpathy.github.io/feed.xml"),
("raschka", "https://magazine.sebastianraschka.com/feed"),
("interconnects", "https://www.interconnects.ai/feed"),
("alignment_forum", "https://www.alignmentforum.org/feed.xml"),
("lesswrong", "https://www.lesswrong.com/feed.xml"),
]:
s(n, "company", "rss", u, "rss", ("ai", "blog"))
# ---------- papers ----------
for n, u in [
("arxiv_cs_ai", "https://export.arxiv.org/rss/cs.AI"),
("arxiv_cs_cr", "https://export.arxiv.org/rss/cs.CR"),
("arxiv_cs_cl", "https://export.arxiv.org/rss/cs.CL"),
("arxiv_cs_lg", "https://export.arxiv.org/rss/cs.LG"),
]:
s(n, "papers", "rss", u, "rss", ("papers", "arxiv"))
# ---------- community / forum (discourse latest.json) ----------
for n, u in [
("dc_pytorch", "https://discuss.pytorch.org/latest.json"),
("dc_huggingface", "https://discuss.huggingface.co/latest.json"),
("dc_elastic", "https://discuss.elastic.co/latest.json"),
("dc_gitlab", "https://forum.gitlab.com/latest.json"),
("dc_homeassistant", "https://community.home-assistant.io/latest.json"),
("dc_manjaro", "https://forum.manjaro.org/latest.json"),
("dc_grafana", "https://community.grafana.com/latest.json"),
("dc_vue", "https://forum.vuejs.org/latest.json"),
("dc_kotlin", "https://discuss.kotlinlang.org/latest.json"),
("dc_bitwarden", "https://community.bitwarden.com/latest.json"),
("dc_openwrt", "https://forum.openwrt.org/latest.json"),
("dc_python", "https://discuss.python.org/latest.json"),
("dc_jupyter", "https://discuss.jupyter.org/latest.json"),
("dc_fly", "https://community.fly.io/latest.json"),
("dc_meta", "https://meta.discourse.org/latest.json"),
]:
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
# ---------- reddit ----------
for n, sub in [("r_netsec", "netsec"), ("r_bugbounty", "bugbounty"), ("r_llmdevs", "LLMDevs"),
("r_cybersecurity", "cybersecurity"), ("r/sideproject", "SideProject"),
("r/SaaS", "SaaS"), ("r/selfhosted", "selfhosted")]:
s(n, "community", "html", "https://www.reddit.com/r/%s/new/" % sub, "reddit_listing", ("reddit",))
# ---------- telegram ----------
for n, ch in [("tg_seeallochnaya", "seeallochnaya"), ("tg_data_secrets", "data_secrets"),
("tg_ai_newz", "ai_newz"), ("tg_addmetoai", "addmetoai")]:
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
# ---------- marketing / traffic forums ----------
s("bhw_index", "forum", "rss", "https://www.blackhatworld.com/forums/-/index.rss", "rss", ("traffic", "seo"))
s("so_llm", "forum", "rss", "https://stackoverflow.com/feeds/tag?tagnames=llm&sort=newest", "rss", ("qa",))
# ---------- releases ----------
for n, repo in [("gh_rel_aider", "Aider-AI/aider"), ("gh_rel_cline", "cline/cline"),
("gh_rel_continue", "continuedev/continue"), ("gh_rel_openhands", "All-Hands-AI/OpenHands"),
("gh_rel_unsloth", "unslothai/unsloth"), ("gh_rel_axolotl", "axolotl-ai-cloud/axolotl"),
("gh_rel_sglang", "sgl-project/sglang"), ("gh_rel_dspy", "stanfordnlp/dspy"),
("gh_rel_crewai", "crewAIInc/crewAI"), ("gh_rel_mem0", "mem0ai/mem0"),
("gh_rel_graphrag", "microsoft/graphrag"), ("gh_rel_llamaindex", "run-llama/llama_index"),
("gh_rel_semgrep", "semgrep/semgrep"), ("gh_rel_nuclei", "projectdiscovery/nuclei")]:
s(n, "release", "atom", "https://github.com/%s/releases.atom" % repo, "rss", ("release", "github"))
# ---------- cn ----------
for n, u in [("qbitai", "https://www.qbitai.com/feed"), ("jiqizhixin", "https://www.jiqizhixin.com/rss"),
("infoq_cn", "https://www.infoq.cn/feed"), ("36kr", "https://36kr.com/feed")]:
s(n, "company_cn", "rss", u, "rss", ("cn",))
# ---------- волна 5b: инфра/дев-блоги, AI-провайдеры, ещё форумы ----------
for n, u in [
("cloudflare_blog", "https://blog.cloudflare.com/rss/"),
("vercel_blog", "https://vercel.com/blog/rss.xml"),
("github_blog", "https://github.blog/feed/"),
("gitlab_blog", "https://about.gitlab.com/atom.xml"),
("docker_blog", "https://www.docker.com/blog/feed/"),
("supabase_blog", "https://supabase.com/blog/rss.xml"),
("neon_blog", "https://neon.tech/blog/rss.xml"),
("fly_blog", "https://fly.io/blog/feed.xml"),
("stripe_blog", "https://stripe.com/blog/feed.rss"),
("elastic_blog", "https://www.elastic.co/blog/feed"),
("anthropic_news", "https://www.anthropic.com/news/rss.xml"),
("openrouter_blog", "https://openrouter.ai/blog/rss.xml"),
("groq_blog", "https://groq.com/blog/rss.xml"),
("cerebras_blog", "https://www.cerebras.ai/blog/rss.xml"),
("ai2_blog", "https://blog.allenai.org/feed"),
("databricks_blog", "https://www.databricks.com/feed"),
("wandb_blog", "https://wandb.ai/fully-connected/rss.xml"),
("modal_blog", "https://modal.com/blog/feed.xml"),
("replicate_blog", "https://replicate.com/blog/rss.xml"),
("baseten_blog", "https://www.baseten.co/blog/rss.xml"),
("scale_blog", "https://scale.com/blog/rss.xml"),
("ooba_blog", "https://blog.ollama.com/rss.xml"),
("lmarena_blog", "https://lmarena.ai/blog/rss.xml"),
("arxiv_cs_se", "https://export.arxiv.org/rss/cs.SE"),
("arxiv_cs_ir", "https://export.arxiv.org/rss/cs.IR"),
("arxiv_stat_ml", "https://export.arxiv.org/rss/stat.ML"),
("thegradient", "https://thegradient.pub/rss/"),
("paperswithcode_blog", "https://paperswithcode.com/blog/rss"),
("mlsafe", "https://www.mlsafety.org/rss.xml"),
]:
s(n, "company", "rss", u, "rss", ("infra", "blog"))
for n, u in [
("dc_rust_internals", "https://internals.rust-lang.org/latest.json"),
("dc_racket", "https://discourse.racket-lang.org/latest.json"),
("dc_nix", "https://discourse.nixos.org/latest.json"),
("dc_godot", "https://forum.godotengine.org/latest.json"),
("dc_swift", "https://forums.swift.org/latest.json"),
("dc_go", "https://forum.golangbridge.org/latest.json"),
("dc_raspberry", "https://forums.raspberrypi.com/latest.json"),
("dc_tryghost", "https://forum.ghost.org/latest.json"),
("dc_blender", "https://devtalk.blender.org/latest.json"),
("dc_obsidian", "https://forum.obsidian.md/latest.json"),
]:
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
for n, ch in [("tg_ai_machinelearning", "ai_machinelearning"), ("tg_gpt_main", "gpt_main"),
("tg_neuralshit", "neuralshit"), ("tg_llm_underhood", "llm_underhood"),
("tg_abuz_ai_soft2", "abuz_ai_soft"), ("tg_cybersecurity_ch", "cybersecurity_channel")]:
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
def probe_one(row):
name, cat, kind, url, ad, tags = row
try:
ok, n, used = discover.probe(url, cat, limit=6)
return (name, cat, kind, url, ad, tags, ok, n, used)
except Exception as e:
return (name, cat, kind, url, ad, tags, False, 0, str(e)[:40])
if __name__ == "__main__":
have = {s_.url.rstrip("/") for s_ in sources.all_sources()}
cand = [r for r in C if r[3].rstrip("/") not in have]
print("кандидатов:", len(C), "| новых к проверке:", len(cand))
res = []
with cf.ThreadPoolExecutor(16) as ex:
for r in ex.map(probe_one, cand):
res.append(r)
live = [r for r in res if r[6] and r[7] >= 2]
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
for r in sorted(res, key=lambda x: -x[7])[:12]:
print(" %-22s %s %s" % (r[0], r[8], r[7]))
print("--- skip")
print(", ".join(x[0] for x in res if x not in live))
# write module
head = ('"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):\n'
'security/AI/форумы/reddit/tg/релизы/CN. Дата прогона %s, живых %d."""\n\n'
% (__import__("datetime").date.today(), len(live)))
body = ["def S(name, category, kind, url, adapter=\"rss\", tags=(), limit=15, detail=False, extra=None):",
" return {\"name\": name, \"category\": category, \"kind\": kind, \"url\": url, \"adapter\": adapter,",
" \"tags\": tuple(tags), \"limit\": limit, \"detail\": detail, \"extra\": extra or {}}",
"", "", "EXTRA5 = ["]
for row in live:
name, cat, kind, url, ad, tags = row[:6]
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15),' % (name, cat, kind, url, ad, tags))
body.append("]")
open("omni/sources_ext4.py", "w", encoding="utf-8", newline="\n").write(head + "\n".join(body) + "\n")
print("written omni/sources_ext4.py")

106
tools/wave6.py Normal file
View File

@@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
"""Волна 6: CVE-фиды, абуз-борды, трафик/SEO, ещё блоги. Только живые (>=2 записи)."""
import sys, os, json, datetime, concurrent.futures as cf
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
os.chdir(r"C:/Users/User/tmp/omni-parse")
from omni import discover, sources, config
config.load()
C = []
def s(name, cat, kind, url, ad="rss", tags=(), extra=None):
C.append({"name": name, "category": cat, "kind": kind, "url": url,
"adapter": ad, "tags": list(tags), "extra": extra or {}})
# --- CVE / уязвимости ---
s("cvefeed_critical", "security", "rss", "https://cvefeed.io/rssfeed/severity/critical", "rss", ("cve", "security"))
s("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",))
s("cisa_kev_json", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "cisa_kev", ("cve", "kev"))
s("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=20", "nvd", ("cve", "nvd"))
s("gh_advisories", "security", "atom", "https://github.com/advisories.atom", "rss", ("cve", "github"))
s("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"))
s("cve_org_blog", "security", "rss", "https://www.cve.org/RSS/News", "rss", ("cve",))
s("msrc_blog", "security", "rss", "https://msrc.microsoft.com/blog/feed", "rss", ("cve", "microsoft"))
s("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",))
s("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"))
# --- абуз-борды и трафик ---
s("ab_seochao_arbitrage", "abuse", "rss", "https://seochao.ru/forums/arbitrazh-trafika.85/index.rss", "rss", ("traffic", "abuse"))
s("ab_seochao_accounts", "abuse", "rss", "https://seochao.ru/forums/akkaunty.60/index.rss", "rss", ("accounts", "abuse"))
s("ab_blackhatworld", "abuse", "rss", "https://www.blackhatworld.com/forums/misc.18/index.rss", "rss", ("traffic", "abuse"))
s("ab_bhw_marketplace", "abuse", "rss", "https://www.blackhatworld.com/forums/marketplace.141/index.rss", "rss", ("market", "abuse"))
s("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"))
s("ab_zennoposter_blog", "abuse", "rss", "https://zennolab.com/discussion/forums/novosti.30/index.rss", "rss", ("automation", "abuse"))
# --- AI / инфра ---
for n, u in [
("swebench_blog", "https://www.swebench.com/blog/rss.xml"),
("evalplus", "https://github.com/evalplus/evalplus/releases.atom"),
("openai_python_rel", "https://github.com/openai/openai-python/releases.atom"),
("vllm_blog", "https://blog.vllm.ai/feed.xml"),
("pytorch_blog", "https://pytorch.org/feed.xml"),
("tensorflow_blog", "https://blog.tensorflow.org/feeds/posts/default"),
("ray_blog", "https://www.anyscale.com/blog/rss.xml"),
("modal_changelog", "https://modal.com/blog/rss.xml"),
("langchain_blog", "https://blog.langchain.dev/rss/"),
("llamaindex_blog", "https://www.llamaindex.ai/blog/rss.xml"),
("qdrant_blog", "https://qdrant.tech/blog/rss.xml"),
("weaviate_blog", "https://weaviate.io/blog/rss.xml"),
("chroma_blog", "https://www.trychroma.com/blog/rss.xml"),
("pinecone_blog", "https://www.pinecone.io/blog/rss.xml"),
("neo4j_blog", "https://neo4j.com/blog/feed/"),
("redis_blog", "https://redis.io/blog/feed/"),
("clickhouse_blog", "https://clickhouse.com/blog/rss.xml"),
("duckdb_news", "https://duckdb.org/news/feed.xml"),
("sqlite_news", "https://sqlite.org/news.rss"),
("postgres_news", "https://www.postgresql.org/news.rss"),
]:
s(n, "company", "rss", u, "rss", ("infra", "blog"))
# --- форумы/комьюнити ---
for n, u in [
("dc_duckdb", "https://discuss.duckdb.org/latest.json"),
("dc_supabase", "https://github.com/supabase/supabase/discussions.atom"),
("dc_cloudflare", "https://community.cloudflare.com/latest.json"),
("dc_vercel", "https://github.com/vercel/next.js/discussions.atom"),
]:
s(n, "forum", "json" if u.endswith(".json") else "atom", u,
"discourse" if u.endswith(".json") else "rss", ("forum",))
def probe_one(row):
try:
ok, n, ad = discover.probe(row["url"], row["category"], limit=6)
return row, ok, n, ad
except Exception as e:
return row, False, 0, str(e)[:40]
if __name__ == "__main__":
have = {x.url.rstrip("/") for x in sources.all_sources()}
cand = [r for r in C if r["url"].rstrip("/") not in have]
print("кандидатов:", len(C), "| новых:", len(cand))
res = []
with cf.ThreadPoolExecutor(14) as ex:
for row, ok, n, ad in ex.map(probe_one, cand):
res.append((row, ok, n, ad))
live = [(r, n, ad) for r, ok, n, ad in res if ok and n >= 2]
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
for r, n, ad in live:
print(" %-24s %-16s %s %s" % (r["name"], ad, r["category"], n))
print("--- мёртвые:", ", ".join(r["name"] for r, ok, n, ad in res if not (ok and n >= 2)))
head = ('"""Волна 6 (авто-проверка, только items >= 2): CVE, абуз-борды, инфра-блоги, форумы.\n'
'Дата %s, живых %d."""\n\n' % (datetime.date.today(), len(live)))
body = ['def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):',
' return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,',
' "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}',
"", "", "EXTRA6 = ["]
for r, n, ad in live:
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15, extra=%r),\n' %
(r["name"], r["category"], r["kind"], r["url"], r["adapter"], tuple(r["tags"]), r["extra"]))
body.append("]")
open("omni/sources_ext5.py", "w", encoding="utf-8", newline="\n").write(head + "".join(body).replace("),\n\n", "),\n") + "\n")
print("written omni/sources_ext5.py")

70
web/index.html
View File

@@ -269,6 +269,11 @@ select:focus,input:focus{border-color:rgba(124,108,244,.6)}
.runs-time{font:11.5px var(--mono);color:var(--dim2)}
.chip b{font:10.5px var(--mono);color:var(--dim2);font-weight:500;margin-left:3px}
.chip.on b{color:#cbc4ff}
.tps{display:flex;flex-wrap:wrap;gap:6px;margin:7px 0 3px}
.tp{font-size:11px;line-height:1;padding:4px 8px;border-radius:999px;background:rgba(120,180,255,.12);
color:#9dc4ff;border:1px solid rgba(120,180,255,.24);letter-spacing:.02em}
#type-tools{display:flex;flex-wrap:wrap;gap:6px;align-items:center;margin:-2px 0 14px}
</style>
</head>
<body>
@@ -324,6 +329,7 @@ select:focus,input:focus{border-color:rgba(124,108,244,.6)}
<section id="view-feed">
<div class="kpis" id="kpis"></div>
<div class="toolbar" id="feed-tools"></div>
<div class="filters" id="type-tools"></div>
<div class="cards" id="cards"></div>
<div class="empty" id="cards-more" style="display:none"><button class="btn" id="btn-more">Показать ещё</button></div>
</section>
@@ -445,7 +451,11 @@ const api = async (p, opt) => {
};
const post = (p, b) => api(p, {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(b || {})});
const S = {view: 'feed', cat: 'all', q: '', state: '', srcQ: '', offset: 0, limit: 36, sort: 'score', stats: null, sources: [], job: null};
const TYPE_LABEL = {news: 'новость', tool: 'софт', release: 'релиз', cve: 'уязвимость', model: 'модель',
question: 'вопрос', price: 'цена', guide: 'гайд', dataset: 'датасет', leak: 'слив', trial: 'триал',
promo: 'промо', resale: 'перепродажа', job: 'вакансия'};
const S = {view: 'feed', cat: 'all', q: '', state: '', srcQ: '', offset: 0, limit: 36, sort: 'score', priced: false, type: 'all', types: [], stats: null, sources: [], job: null};
function toast(msg, cls) {
const d = el('div', cls || '', esc(msg));
@@ -476,6 +486,7 @@ function syncChips() {
const k = b.dataset.k || 'cat';
if (k === 'cat') b.classList.toggle('on', b.dataset.cat === S.cat);
if (k === 'sort') b.classList.toggle('on', b.dataset.sort === S.sort);
if (k === 'priced') b.classList.toggle('on', !!S.priced);
});
}
@@ -494,6 +505,25 @@ function setCat(name) {
}
/* ---------- stats / KPI ---------- */
function buildTypes(list) {
const box = $('#type-tools');
if (!box) return;
S.types = list || S.types;
const arr = S.types || [];
box.innerHTML = '<span class="kv" style="margin-right:6px">тип:</span>' +
`<button class="chip ${S.type === 'all' ? 'on' : ''}" data-t="all">все</button>` +
arr.filter(x => x.n > 0).map(x =>
`<button class="chip ${S.type === x.key ? 'on' : ''}" data-t="${esc(x.key)}">${esc(x.label)} <b>${fmt(x.n)}</b></button>`).join('');
box.querySelectorAll('.chip').forEach(b => {
b.onclick = () => { S.type = b.dataset.t; S.offset = 0; buildTypes(S.types); loadItems(true); };
});
}
async function loadTypes() {
const r = await api('/api/types');
if (r.ok) buildTypes(r.types);
}
async function loadStats() {
const s = await api('/api/stats');
if (s.error) return toast(s.error, 'err');
@@ -508,6 +538,7 @@ async function loadStats() {
`<b>${fmt(s.runs)}</b> прогонов · <b>${fmt(s.priced)}</b> с ценой`;
renderCats(s.categories);
buildChips(s.categories);
loadTypes();
const ks = [
['записей', s.items, 'в sqlite + fts5', ''],
['источников', s.sources, `${s.sources_enabled} включено`, ''],
@@ -533,6 +564,7 @@ function card(it) {
<span class="score">${it.score != null ? (+it.score).toFixed(1) : ''}</span></div>
<h3>${esc(it.title || '(без заголовка)')}</h3>
${text ? `<p>${esc(text)}</p>` : ''}
${it.types ? `<div class="tps">${String(it.types).split(',').filter(Boolean).map(t => `<span class="tp">${esc(TYPE_LABEL[t] || t)}</span>`).join('')}</div>` : ''}
<div class="meta">${it.author ? `<span>${esc(String(it.author).slice(0, 22))}</span>` : ''}
${pub ? `<span>${esc(pub)}</span>` : ''}${price}</div>`;
if (it.url) c.onclick = () => window.open(it.url, '_blank', 'noopener');
@@ -542,7 +574,7 @@ function card(it) {
async function loadItems(reset) {
if (reset) { S.offset = 0; $('#cards').innerHTML = '<div class="skel"></div><div class="skel"></div><div class="skel"></div>'; }
const u = `/api/items?category=${encodeURIComponent(S.cat)}&q=${encodeURIComponent(S.q)}&limit=${S.limit}&offset=${S.offset}&sort=${S.sort}`;
const u = `/api/items?category=${encodeURIComponent(S.cat)}&q=${encodeURIComponent(S.q)}&limit=${S.limit}&offset=${S.offset}&sort=${S.sort}&priced=${S.priced ? 1 : 0}&type=${encodeURIComponent(S.type)}`;
const r = await api(u);
if (r.error) return toast(r.error, 'err');
if (reset) $('#cards').innerHTML = '';
@@ -792,6 +824,7 @@ function setView(v) {
u.searchParams.set('view', v);
S.cat && S.cat !== 'all' ? u.searchParams.set('cat', S.cat) : u.searchParams.delete('cat');
S.q ? u.searchParams.set('q', S.q) : u.searchParams.delete('q');
S.priced ? u.searchParams.set('priced', '1') : u.searchParams.delete('priced');
history.replaceState(null, '', u.toString());
} catch (e) {}
if (v === 'feed') loadItems(true);
@@ -866,17 +899,27 @@ function buildChips(cats) {
const list = cats || (S.stats && S.stats.categories) || [];
const tot = {}; list.forEach(c => { tot[c.name] = c.total; });
box.innerHTML =
`<button class="chip" data-k="cat" data-cat="all">всё <b>${fmt(S.stats ? S.stats.sources : 0)}</b></button>` +
list.map(c => `<button class="chip" data-k="cat" data-cat="${esc(c.name)}">${esc(c.name)} <b>${fmt(c.total)}</b></button>`).join('') +
'<span style="flex:1"></span>' +
'<button class="chip" data-k="sort" data-sort="score">по score</button>' +
'<button class="chip" data-k="sort" data-sort="fresh">по свежести</button>';
box.querySelectorAll('.chip').forEach(b => {
b.onclick = () => {
if (b.dataset.k === 'sort') { S.sort = b.dataset.sort; syncChips(); loadItems(true); }
else setCat(b.dataset.cat);
};
});
`<button class="chip" data-k="cat" data-cat="all">всё <b>${fmt(S.stats ? S.stats.sources : 0)}</b></button>` +
list.map(c => `<button class="chip" data-k="cat" data-cat="${esc(c.name)}">${esc(c.name)} <b>${fmt(c.total)}</b></button>`).join('') +
(S.q ? `<button class="chip on" data-k="clear">поиск: ${esc(S.q.slice(0, 18))} ✕</button>` : '') +
'<span style="flex:1"></span>' +
'<button class="chip" data-k="priced">только с ценой</button>' +
'<button class="chip" data-k="sort" data-sort="score">по score</button>' +
'<button class="chip" data-k="sort" data-sort="fresh">по свежести</button>' +
'<button class="chip" data-k="sort" data-sort="price">по цене</button>' +
'<button class="chip" data-k="csv">CSV</button>';
box.querySelectorAll('.chip').forEach(b => {
b.onclick = () => {
if (b.dataset.k === 'sort') { S.sort = b.dataset.sort; syncChips(); loadItems(true); }
else if (b.dataset.k === 'priced') { S.priced = !S.priced; syncChips(); loadItems(true); }
else if (b.dataset.k === 'clear') { S.q = ''; $('#q').value = ''; syncChips(); loadItems(true); }
else if (b.dataset.k === 'csv') {
const u = `/api/export.csv?category=${encodeURIComponent(S.cat)}&q=${encodeURIComponent(S.q)}&sort=${S.sort}&priced=${S.priced ? 1 : 0}&type=${encodeURIComponent(S.type)}`;
window.open(u, '_blank'); toast('CSV выгружается', 'ok');
}
else setCat(b.dataset.cat);
};
});
syncChips();
}
@@ -954,6 +997,7 @@ try { if (localStorage.getItem('omni.cats') === '0') { $('#cats').classList.add(
const p = new URLSearchParams(location.search);
if (p.get('cat')) S.cat = p.get('cat');
if (p.get('q')) { S.q = p.get('q'); $('#q').value = S.q; }
if (p.get('priced') === '1') S.priced = true;
await loadStats();
const v = p.get('view');
if (v === 'abuse') { setView('abuse'); setTimeout(() => loadAbuse().catch(e => console.error('abuse', e)), 300); }