feat: типизация записей (14 типов, до 3 на запись), чипы и бейджи типов в панели, CSV с типами, /api/types, doctor --only по категориям, волны источников 5-6 (635 всего)

2026-10-26 19:01:45 +03:00
parent 4196d66b4e
commit 7b3d708402
10 changed files with 652 additions and 24 deletions
--- a/omni/doctor.py
+++ b/omni/doctor.py
@@ -86,7 +86,11 @@ def fix(src):

 def run(only=None, do_fix=False, workers=8, limit=3):
    from . import source_state
-    picked = sources.pick(None, only, None)
+    only = (only or "").strip() or None
+    if only and only in sources.CATEGORIES:
+        picked = sources.pick(category=only, respect_enabled=False)
+    else:
+        picked = sources.pick(only=only, respect_enabled=False)
    cache = load_cache()
    results = []
    with ThreadPoolExecutor(max_workers=workers) as ex:
--- a/omni/serve.py
+++ b/omni/serve.py
@@ -204,6 +204,12 @@ class Handler(BaseHTTPRequestHandler):
                "categories": cats,
            })

+        if path == "/api/types":
+            from . import taxonomy
+            cnt = store.type_counts()
+            items = [{"key": k, "label": taxonomy.LABELS.get(k, k), "n": v} for k, v in cnt.items()]
+            return self._json({"ok": True, "types": items, "total": sum(cnt.values())})
+
        if path == "/api/runs":
            try:
                limit = max(1, min(500, int((q.get("limit") or ["80"])[0])))
@@ -299,13 +305,42 @@ class Handler(BaseHTTPRequestHandler):
            limit = min(int((q.get("limit") or ["40"])[0]), 200)
            offset = int((q.get("offset") or ["0"])[0])
            sort = (q.get("sort") or ["score"])[0]
-            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset, sort=sort)
+            priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
+            typ = (q.get("type") or ["all"])[0]
+            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=limit, offset=offset,
+                                        sort=sort, priced=priced, typ=typ)
            for _r in rows:
                _r["title"] = _clean_txt(_r.get("title"), 180)
                _r["text"] = _clean_txt(_r.get("text"), 260)
                _r["published"] = _fmt_date(_r.get("published"))
            return self._json({"count": len(rows), "total": total, "offset": offset, "items": rows})

+        if path == "/api/export.csv":
+            import csv as _csv
+            cat = (q.get("category") or [None])[0]
+            query = (q.get("q") or [""])[0].strip()
+            priced = (q.get("priced") or ["0"])[0] in ("1", "true", "on")
+            sort = (q.get("sort") or ["score"])[0]
+            typ = (q.get("type") or ["all"])[0]
+            rows, total = store.browse2(q=query, names=_names_in_category(cat), limit=5000, offset=0,
+                                        sort=sort, priced=priced, typ=typ)
+            import io as _io
+            buf = _io.StringIO()
+            w = _csv.writer(buf, delimiter=";")
+            w.writerow(["source", "kind", "types", "title", "author", "published", "score", "price", "currency", "url"])
+            for r in rows:
+                w.writerow([r.get("source"), r.get("kind"), r.get("types"), (r.get("title") or "").replace(chr(10), " "),
+                            r.get("author"), r.get("published"), r.get("score"), r.get("price"),
+                            r.get("currency"), r.get("url")])
+            data = buf.getvalue().encode("utf-8-sig")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/csv; charset=utf-8")
+            self.send_header("Content-Disposition", 'attachment; filename="omni.csv"')
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+
        if path.startswith("/api/job/"):
            jid = path.rsplit("/", 1)[-1]
            return self._json(JOBS.get(jid) or {"state": "unknown"})
--- a/omni/sources.py
+++ b/omni/sources.py
@@ -14,6 +14,8 @@ from .sources_ext import EXTRA
 from .sources_ext2 import EXTRA2
 from .sources_ext3 import EXTRA3
 from .sources_abuse import ABUSE_LIVE
+from .sources_ext4 import EXTRA5
+from .sources_ext5 import EXTRA6


@dataclass
@@ -197,7 +199,7 @@ def _dedup(rows):
    return out


-SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4])
+SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6])

 BY_NAME = {s.name: s for s in SOURCES}
 CATEGORIES = sorted({s.category for s in SOURCES})
--- a/omni/sources_ext4.py
+++ b/omni/sources_ext4.py
@@ -0,0 +1,47 @@
+"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):
+security/AI/форумы/reddit/tg/релизы/CN. Дата прогона 2026-10-26, живых 37."""
+
+def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
+    return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
+            "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
+
+
+EXTRA5 = [
+    S("sans_isc", "security", "rss", "https://isc.sans.edu/rssfeed.xml", "rss", ('security', 'research'), limit=15),
+    S("checkpoint_research", "security", "rss", "https://research.checkpoint.com/feed/", "rss", ('security', 'research'), limit=15),
+    S("hackread", "security", "rss", "https://www.hackread.com/feed/", "rss", ('security', 'research'), limit=15),
+    S("thecyberwire", "security", "rss", "https://thecyberwire.com/feeds/rss.xml", "rss", ('security', 'research'), limit=15),
+    S("securityaffairs", "security", "rss", "https://securityaffairs.com/feed", "rss", ('security', 'research'), limit=15),
+    S("wiz_blog", "security", "rss", "https://www.wiz.io/feed/rss.xml", "rss", ('security', 'research'), limit=15),
+    S("aquasec", "security", "rss", "https://blog.aquasec.com/rss.xml", "rss", ('security', 'research'), limit=15),
+    S("crowdstrike", "security", "rss", "https://www.crowdstrike.com/blog/feed/", "rss", ('security', 'research'), limit=15),
+    S("yeswehack", "security", "rss", "https://www.yeswehack.com/feed", "rss", ('security', 'research'), limit=15),
+    S("deepmind_blog", "company", "rss", "https://deepmind.google/blog/rss.xml", "rss", ('ai', 'blog'), limit=15),
+    S("the_decoder", "company", "rss", "https://the-decoder.com/feed/", "rss", ('ai', 'blog'), limit=15),
+    S("bair_blog", "company", "rss", "https://bair.berkeley.edu/blog/feed.xml", "rss", ('ai', 'blog'), limit=15),
+    S("nvidia_blog", "company", "rss", "https://blogs.nvidia.com/feed/", "rss", ('ai', 'blog'), limit=15),
+    S("karpathy", "company", "rss", "https://karpathy.github.io/feed.xml", "rss", ('ai', 'blog'), limit=15),
+    S("alignment_forum", "company", "rss", "https://www.alignmentforum.org/feed.xml", "rss", ('ai', 'blog'), limit=15),
+    S("lesswrong", "company", "rss", "https://www.lesswrong.com/feed.xml", "rss", ('ai', 'blog'), limit=15),
+    S("arxiv_cs_ai", "papers", "rss", "https://export.arxiv.org/rss/cs.AI", "rss", ('papers', 'arxiv'), limit=15),
+    S("arxiv_cs_cr", "papers", "rss", "https://export.arxiv.org/rss/cs.CR", "rss", ('papers', 'arxiv'), limit=15),
+    S("arxiv_cs_cl", "papers", "rss", "https://export.arxiv.org/rss/cs.CL", "rss", ('papers', 'arxiv'), limit=15),
+    S("arxiv_cs_lg", "papers", "rss", "https://export.arxiv.org/rss/cs.LG", "rss", ('papers', 'arxiv'), limit=15),
+    S("dc_openwrt", "forum", "json", "https://forum.openwrt.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("tg_seeallochnaya", "community", "html", "https://t.me/s/seeallochnaya", "tg_web", ('telegram',), limit=15),
+    S("tg_data_secrets", "community", "html", "https://t.me/s/data_secrets", "tg_web", ('telegram',), limit=15),
+    S("vercel_blog", "company", "rss", "https://vercel.com/blog/rss.xml", "rss", ('infra', 'blog'), limit=15),
+    S("docker_blog", "company", "rss", "https://www.docker.com/blog/feed/", "rss", ('infra', 'blog'), limit=15),
+    S("supabase_blog", "company", "rss", "https://supabase.com/blog/rss.xml", "rss", ('infra', 'blog'), limit=15),
+    S("arxiv_cs_se", "company", "rss", "https://export.arxiv.org/rss/cs.SE", "rss", ('infra', 'blog'), limit=15),
+    S("arxiv_cs_ir", "company", "rss", "https://export.arxiv.org/rss/cs.IR", "rss", ('infra', 'blog'), limit=15),
+    S("arxiv_stat_ml", "company", "rss", "https://export.arxiv.org/rss/stat.ML", "rss", ('infra', 'blog'), limit=15),
+    S("dc_rust_internals", "forum", "json", "https://internals.rust-lang.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("dc_nix", "forum", "json", "https://discourse.nixos.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("dc_godot", "forum", "json", "https://forum.godotengine.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("dc_swift", "forum", "json", "https://forums.swift.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("dc_tryghost", "forum", "json", "https://forum.ghost.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("dc_blender", "forum", "json", "https://devtalk.blender.org/latest.json", "discourse", ('forum', 'discourse'), limit=15),
+    S("tg_gpt_main", "community", "html", "https://t.me/s/gpt_main", "tg_web", ('telegram',), limit=15),
+    S("tg_neuralshit", "community", "html", "https://t.me/s/neuralshit", "tg_web", ('telegram',), limit=15),
+]
--- a/omni/sources_ext5.py
+++ b/omni/sources_ext5.py
@@ -0,0 +1,18 @@
+"""Волна 6: CVE-фиды, абуз/SEO-блоги, инфра-блоги. Каждая строка проверена живым fetch (items >= 2)."""
+
+
+def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):
+    return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,
+            "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}
+
+
+EXTRA6 = [
+    S("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",), limit=15),
+    S("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"), limit=15),
+    S("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",), limit=15),
+    S("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"), limit=15),
+    S("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"), limit=15),
+    S("tensorflow_blog", "company", "rss", "https://blog.tensorflow.org/feeds/posts/default", "rss", ("infra", "blog"), limit=15),
+    S("neo4j_blog", "company", "rss", "https://neo4j.com/blog/feed/", "rss", ("infra", "blog"), limit=15),
+    S("postgres_news", "company", "rss", "https://www.postgresql.org/news.rss", "rss", ("infra", "blog"), limit=15),
+]
--- a/omni/store.py
+++ b/omni/store.py
@@ -4,7 +4,7 @@ import sqlite3
 import time
 from pathlib import Path

-from . import config
+from . import config, taxonomy

 SCHEMA = """
 CREATE TABLE IF NOT EXISTS items (
@@ -43,6 +43,10 @@ def conn():
    c.execute("PRAGMA journal_mode=WAL")
    c.execute("PRAGMA synchronous=NORMAL")
    c.executescript(SCHEMA)
+    cols = {r[1] for r in c.execute("PRAGMA table_info(items)")}
+    if "types" not in cols:
+        c.execute("ALTER TABLE items ADD COLUMN types TEXT")
+        c.commit()
    return c


@@ -85,9 +89,10 @@ def add(items):
            try:
                cur = c.execute(
                    "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
-                    "tags,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                    "tags,types,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                    (it.key(), it.provider, it.kind, it.source, it.url, it.title, it.text, it.author,
-                     it.published, it.lang, ",".join(it.tags), it.price, it.currency, it.score,
+                     it.published, it.lang, ",".join(it.tags), ",".join(taxonomy.classify(it.title, it.text)),
+                     it.price, it.currency, it.score,
                     _signed(h), json.dumps(it.extra, ensure_ascii=False)[:2000], ts))
            except sqlite3.IntegrityError:
                dup += 1
@@ -146,11 +151,11 @@ def browse(category=None, q="", limit=60, offset=0):
    args += [limit, offset]
    with conn() as c:
        rows = c.execute(sql, args).fetchall()
-    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
+    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
    return [dict(zip(keys, r)) for r in rows]


-def browse2(q="", names=None, limit=60, offset=0, sort="score"):
+def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None):
    """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
    where = "WHERE 1=1 "
    args = []
@@ -160,15 +165,49 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score"):
    if q:
        where += "AND (title LIKE ? OR text LIKE ?) "
        args += [f"%{q}%", f"%{q}%"]
+    if priced:
+        where += "AND price IS NOT NULL AND price != '' "
+    if typ and typ != "all":
+        where += "AND (',' || COALESCE(types,'') || ',') LIKE ? "
+        args += [f"%,{typ},%"]
+    order = {"fresh": "ORDER BY published DESC, id DESC",
+             "price": "ORDER BY CAST(price AS REAL) DESC, score DESC",
+             "old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
    with conn() as c:
        total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
-        rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url "
-                         "FROM items " + where + ("ORDER BY published DESC, id DESC LIMIT ? OFFSET ?" if sort == "fresh" else "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"),
+        rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
+                         "FROM items " + where + order + " LIMIT ? OFFSET ?",
                         args + [limit, offset]).fetchall()
-    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
+    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
    return [dict(zip(keys, r)) for r in rows], total


+def type_counts():
+    """Сколько записей на каждый тип."""
+    out = {}
+    with conn() as c:
+        for row in c.execute("SELECT COALESCE(types,'') FROM items"):
+            for t in (row[0] or "").split(","):
+                t = t.strip()
+                if t:
+                    out[t] = out.get(t, 0) + 1
+    return dict(sorted(out.items(), key=lambda kv: -kv[1]))
+
+
+def retype(limit=None):
+    """Пересчитать типы у уже сохранённых записей."""
+    n = 0
+    with conn() as c:
+        q = "SELECT id,title,text FROM items"
+        if limit:
+            q += " LIMIT %d" % int(limit)
+        for iid, title, text in c.execute(q).fetchall():
+            c.execute("UPDATE items SET types=? WHERE id=?",
+                      (",".join(taxonomy.classify(title, text)), iid))
+            n += 1
+    return n
+
+
 def count_by_source():
    with conn() as c:
        return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall())
--- a/omni/taxonomy.py
+++ b/omni/taxonomy.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+"""Типизация записей: не «категория источника», а тип самого поста.
+
+Категория отвечает «откуда», тип — «что именно»: слив, триал, софт, промо,
+перепродажа, CVE, релиз, гайд, датасет, модель, вакансия, цена, вопрос, новость.
+Одна запись может иметь несколько типов, порядок — по весу совпадений.
+"""
+import re
+
+# (key, label, weight, [regex, ...])
+RULES = [
+    ("leak", "слив", 10, [
+        r"\bслив\w*", r"утечк\w+", r"\bleak(ed|s)?\b", r"\bdump(ed)?\b", r"база\s+(данных|слит)",
+        r"\bcombo(list)?\b", r"stealer\s*log", r"\bdb\s+dump\b", r"пасс\w*\s+от\s+",
+    ]),
+    ("trial", "триал", 9, [
+        r"\btrial\b", r"триал\w*", r"пробн\w+\s+период", r"free\s+(credits?|tier|plan)",
+        r"бесплатн\w+\s+(кредит|токен|доступ|период|тариф)", r"\bfreebies?\b", r"щедр\w+\s+лимит",
+        r"\$\d+\s*(free|бесплат)", r"no\s+credit\s+card",
+    ]),
+    ("tool", "софт", 8, [
+        r"\bсофт\w*", r"\btool(s|kit)?\b", r"скрипт\w*", r"\bparser\b", r"\bчекер\b", r"\bбот\w*",
+        r"\bcrawler\b", r"\bscraper\b", r"\bautomation\b", r"автоматизац\w+", r"\bcli\b",
+        r"\bsdk\b", r"open[\s-]?source\s+(tool|project)",
+    ]),
+    ("promo", "промо", 7, [
+        r"промокод\w*", r"\bpromo\w*", r"\bcoupon\b", r"скидк\w+", r"\bdiscount\b", r"giveaway",
+        r"раздач\w+", r"\bairdrop\b", r"бонус\w*\s+(код|за\s+регистрац)", r"-?\d{1,2}%\s*off",
+    ]),
+    ("resale", "перепродажа", 7, [
+        r"\bпродам\b", r"\bпродаж\w+", r"\bresell\w*", r"\bв\s+аренду\b", r"\bкуплю\b",
+        r"\bsell(ing)?\s+(account|key|access)", r"перепрод\w+", r"\bоптом\b", r"\bв\s+наличии\b.*\$",
+    ]),
+    ("cve", "уязвимость", 10, [
+        r"\bCVE-\d{4}-\d{3,7}\b", r"\bvulnerab\w+", r"уязвим\w+", r"\bRCE\b", r"\bXSS\b",
+        r"\bSQLi?\b", r"\bexploit\w*", r"\b0day\b", r"\bpatch\w*\s+tuesday", r"security\s+advisory",
+    ]),
+    ("release", "релиз", 6, [
+        r"\brelease[sd]?\b", r"\bрелиз\w*", r"\bchangelog\b", r"\bv\d+\.\d+(\.\d+)?\b",
+        r"вышл\w+\s+версия", r"\bnow\s+available\b", r"\bGA\b", r"\bpreview\s+build\b",
+    ]),
+    ("guide", "гайд", 5, [
+        r"\bгайд\w*", r"\btutorial\b", r"\bhow[\s-]to\b", r"инструкц\w+", r"\bруководств\w+",
+        r"\bкак\s+(сделать|настроить|поднять|получить)", r"\bwalkthrough\b", r"\bcheat\s*sheet\b",
+    ]),
+    ("dataset", "датасет", 6, [
+        r"\bdataset\w*", r"датасет\w*", r"\bcorpus\b", r"\bbenchmark\s+suite\b", r"\bannotated\b",
+    ]),
+    ("model", "модель", 5, [
+        r"\bweights\b", r"\bcheckpoint\w*", r"\.gguf\b", r"\.safetensors\b", r"\bllm\b",
+        r"\bмодел\w+\s+(вышла|обучен| weights)", r"\bquantiz\w+", r"\bfine-?tun\w+",
+    ]),
+    ("job", "вакансия", 6, [
+        r"\bваканси\w+", r"\bhiring\b", r"\bjob\s+(offer|post)", r"\bremote\b.*\$",
+        r"\bsalary\b", r"\$\d+\s*/\s*(hr|hour)", r"\bищу\s+работу\b", r"\bfor\s+hire\b",
+    ]),
+    ("price", "цена", 4, [
+        r"\bцен[аы]\b", r"\bpricing\b", r"\bтариф\w*", r"\$\d", r"\d+\s*₽", r"\d+\s*€",
+        r"\bper\s+(month|year|1m)\b", r"/\s*1m\s+tokens",
+    ]),
+    ("question", "вопрос", 3, [
+        r"\?\s*$", r"^\s*(как|почему|что|где|кто|можно\s+ли)\b", r"\bhelp\b", r"\bподскажите\b",
+    ]),
+]
+
+COMPILED = [(k, l, w, [re.compile(p, re.I | re.U) for p in pats]) for k, l, w, pats in RULES]
+LABELS = {k: l for k, l, _, _ in RULES}
+LABELS["news"] = "новость"
+KEYS = [k for k, _, _, _ in RULES]
+
+
+def classify(title, text, limit=3):
+    """Список типов записи по убыванию веса. Если ничего не сработало — ['news']."""
+    t = (title or "") + "\n" + (text or "")[:2000]
+    if not t.strip():
+        return ["news"]
+    hits = []
+    for key, label, w, pats in COMPILED:
+        n = 0
+        for p in pats:
+            if p.search(t):
+                n += 1
+        if n:
+            hits.append((w + n, key))
+    if not hits:
+        return ["news"]
+    hits.sort(reverse=True)
+    return [k for _, k in hits[:limit]]
+
+
+def labels(keys):
+    return [LABELS.get(k, k) for k in keys or []]