abuse: классификатор ловит панели/github/hn и мульти-теги, реальный total из индекса, кап API поднят до 3000

omni/abuse.py

@@ -141,24 +141,27 @@ def _qualifies(source, title, text, tags):
         sc -= 26
         if not hits:
             return 0, []
-    raw_key = bool(re.search(r"(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
-    if NOISE.search(source or "") and not raw_key:
+    raw_key = bool(re.search(r"\b(sk-[A-Za-z0-9_\-]{20,}|gh[pousr]_[A-Za-z0-9]{20,})", blob))
+    src = str(source or "")
+    known_prefix = src.startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo", "pn_", "gh_", "hn_"))
+    if NOISE.search(src) and not (raw_key or known_prefix):
         return 0, []
-    if cat is None:
-        # старые записи хранят категорию вместо имени источника
-        if not raw_key and not str(source or "").startswith(("ab_", "tg_", "rad_", "reddit_", "discourse_", "xenforo")):
-            return 0, []
-    elif cat not in ABUSE_CATS and not raw_key:
+    if cat is None and not raw_key and not known_prefix:
         return 0, []
-    if cat == "abuse" and sc >= ABUSE_SRC_SC:
+    if cat is not None and cat not in ABUSE_CATS and not (raw_key or known_prefix):
+        return 0, []
+    # панели new-api, github/hn и abuse-источники считаем абузными по факту источника
+    if cat == "abuse" or known_prefix:
+        if sc >= ABUSE_SRC_SC or strong or len(hits) >= 2:
+            return sc or 1, hits
+    if strong and sc >= 16:
         return sc, hits
-    if strong and sc >= 24:
+    if len(hits) >= 2 and sc >= 30:
         return sc, hits
-    if len(hits) >= 2 and sc >= 40:
+    if len(hits) >= 3 and sc >= 20:
         return sc, hits
     return 0, []
 
-
 def _connect():
     con = sqlite3.connect(str(DB), timeout=60)
     con.execute("PRAGMA busy_timeout=60000")
@@ -248,5 +251,26 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
     for it in items:
         for t in it["tags"]:
             counts[t] = counts.get(t, 0) + 1
-    return {"items": items[:limit], "total": len(items), "tags": counts,
+    # total = реальный размер выдачи в индексе, не размер окна чтения
+    total = len(items)
+    try:
+        con = _connect()
+        sql = "SELECT COUNT(*) FROM abuse_idx a JOIN items i ON i.id = a.id"
+        where, args = [], []
+        if tag:
+            where.append("(',' || a.tags || ',') LIKE ?")
+            args.append("%," + tag + ",%")
+        if q:
+            where.append("(i.title LIKE ? OR i.text LIKE ? OR i.source LIKE ?)")
+            args += ["%%%s%%" % q, "%%%s%%" % q, "%%%s%%" % q]
+        if since_days:
+            where.append("i.published >= date('now', ?)")
+            args.append("-%d day" % int(since_days))
+        if where:
+            sql += " WHERE " + " AND ".join(where)
+        total = con.execute(sql, args).fetchone()[0] or total
+        con.close()
+    except Exception:
+        pass
+    return {"items": items[:limit], "total": total, "tags": counts,
             "labels": TAG_LABEL}