feat: типизация записей (14 типов, до 3 на запись), чипы и бейджи типов в панели, CSV с типами, /api/types, doctor --only по категориям, волны источников 5-6 (635 всего)

omni/store.py

@@ -4,7 +4,7 @@ import sqlite3
 import time
 from pathlib import Path
 
-from . import config
+from . import config, taxonomy
 
 SCHEMA = """
 CREATE TABLE IF NOT EXISTS items (
@@ -43,6 +43,10 @@ def conn():
     c.execute("PRAGMA journal_mode=WAL")
     c.execute("PRAGMA synchronous=NORMAL")
     c.executescript(SCHEMA)
+    cols = {r[1] for r in c.execute("PRAGMA table_info(items)")}
+    if "types" not in cols:
+        c.execute("ALTER TABLE items ADD COLUMN types TEXT")
+        c.commit()
     return c
 
 
@@ -85,9 +89,10 @@ def add(items):
             try:
                 cur = c.execute(
                     "INSERT INTO items(k,provider,kind,source,url,title,text,author,published,lang,"
-                    "tags,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                    "tags,types,price,currency,score,h,extra,seen) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                     (it.key(), it.provider, it.kind, it.source, it.url, it.title, it.text, it.author,
-                     it.published, it.lang, ",".join(it.tags), it.price, it.currency, it.score,
+                     it.published, it.lang, ",".join(it.tags), ",".join(taxonomy.classify(it.title, it.text)),
+                     it.price, it.currency, it.score,
                      _signed(h), json.dumps(it.extra, ensure_ascii=False)[:2000], ts))
             except sqlite3.IntegrityError:
                 dup += 1
@@ -146,11 +151,11 @@ def browse(category=None, q="", limit=60, offset=0):
     args += [limit, offset]
     with conn() as c:
         rows = c.execute(sql, args).fetchall()
-    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
+    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
     return [dict(zip(keys, r)) for r in rows]
 
 
-def browse2(q="", names=None, limit=60, offset=0, sort="score"):
+def browse2(q="", names=None, limit=60, offset=0, sort="score", priced=False, typ=None):
     """Записи для панели: фильтр по списку источников (категория), подстрока, пагинация + total."""
     where = "WHERE 1=1 "
     args = []
@@ -160,15 +165,49 @@ def browse2(q="", names=None, limit=60, offset=0, sort="score"):
     if q:
         where += "AND (title LIKE ? OR text LIKE ?) "
         args += [f"%{q}%", f"%{q}%"]
+    if priced:
+        where += "AND price IS NOT NULL AND price != '' "
+    if typ and typ != "all":
+        where += "AND (',' || COALESCE(types,'') || ',') LIKE ? "
+        args += [f"%,{typ},%"]
+    order = {"fresh": "ORDER BY published DESC, id DESC",
+             "price": "ORDER BY CAST(price AS REAL) DESC, score DESC",
+             "old": "ORDER BY published ASC, id ASC"}.get(sort, "ORDER BY score DESC, id DESC")
     with conn() as c:
         total = c.execute("SELECT COUNT(*) FROM items " + where, args).fetchone()[0]
-        rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url "
-                         "FROM items " + where + ("ORDER BY published DESC, id DESC LIMIT ? OFFSET ?" if sort == "fresh" else "ORDER BY score DESC, id DESC LIMIT ? OFFSET ?"),
+        rows = c.execute("SELECT provider,kind,source,title,text,author,published,score,price,currency,url,types "
+                         "FROM items " + where + order + " LIMIT ? OFFSET ?",
                          args + [limit, offset]).fetchall()
-    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url")
+    keys = ("provider", "kind", "source", "title", "text", "author", "published", "score", "price", "currency", "url", "types")
     return [dict(zip(keys, r)) for r in rows], total
 
 
+def type_counts():
+    """Сколько записей на каждый тип."""
+    out = {}
+    with conn() as c:
+        for row in c.execute("SELECT COALESCE(types,'') FROM items"):
+            for t in (row[0] or "").split(","):
+                t = t.strip()
+                if t:
+                    out[t] = out.get(t, 0) + 1
+    return dict(sorted(out.items(), key=lambda kv: -kv[1]))
+
+
+def retype(limit=None):
+    """Пересчитать типы у уже сохранённых записей."""
+    n = 0
+    with conn() as c:
+        q = "SELECT id,title,text FROM items"
+        if limit:
+            q += " LIMIT %d" % int(limit)
+        for iid, title, text in c.execute(q).fetchall():
+            c.execute("UPDATE items SET types=? WHERE id=?",
+                      (",".join(taxonomy.classify(title, text)), iid))
+            n += 1
+    return n
+
+
 def count_by_source():
     with conn() as c:
         return dict(c.execute("SELECT source, COUNT(*) FROM items GROUP BY source").fetchall())