абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days

omni/abuse.py

@@ -235,9 +235,11 @@ def _rows(limit=400, tag=None, q=None, since_days=None):
         out = []
         for r in con.execute(sql, args):
             hits = [t for t in (r[6] or "").split(",") if t]
+            body = (r[4] or "")
             out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
-                        "text": (r[4] or "")[:600], "published": r[5],
-                        "tags": hits, "abuse_score": r[8], "rank": r[7]})
+                        "text": body[:600], "published": r[5],
+                        "tags": hits, "abuse_score": r[8], "rank": r[7],
+                        "find": extract((r[3] or "") + " " + body)})
         return out
     finally:
         con.close()
@@ -273,4 +275,43 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
     except Exception:
         pass
     return {"items": items[:limit], "total": total, "tags": counts,
-            "labels": TAG_LABEL}
+            "labels": TAG_LABEL}
+
+_RE_CODE = re.compile(r"(?:промокод|промо-?код|promo\s*code|coupon|купо[нш]|code)[^\n:：]{0,12}[:：\s]+([A-Z0-9][A-Z0-9_\-]{3,23})", re.I)
+_RE_CODE2 = re.compile(r"\b([A-Z]{2,}[0-9]{2,}[A-Z0-9_\-]{0,14})\b")
+_RE_AMT = re.compile(r"(\d[\d\s.,]{0,12})\s*(млрд|млн|тыс|billion|million|k|m|b|токен\w*|tokens?|credits?|\$|₽|юан\w*)", re.I)
+_RE_LINK = re.compile(r"(?:https?://|t\.me/)[^\s)\]\"'<>]{4,120}")
+_RE_DAYS = re.compile(r"(\d{1,3})\s*(дн\w*|day[s]?|час\w*|hour[s]?|недел\w*|week[s]?)", re.I)
+
+
+def extract(text):
+    """Достаёт из текста абуза полезное: промокоды, объём, ссылки, срок."""
+    t = (text or "")[:4000]
+    codes = []
+    for m in _RE_CODE.finditer(t):
+        c = (m.group(1) or "").strip(" .,:;")
+        if c and c.upper() not in {x.upper() for x in codes}:
+            codes.append(c)
+    if not codes and re.search(r"промо|promo|code|код|gift|купо|coupon|подар", t, re.I):
+        for m in _RE_CODE2.finditer(t):
+            c = m.group(1)
+            if c and c.upper() not in {x.upper() for x in codes}:
+                codes.append(c)
+            if len(codes) >= 3:
+                break
+    amt = None
+    m = _RE_AMT.search(t)
+    if m:
+        amt = ("%s %s" % (m.group(1).strip(), m.group(2))).replace("  ", " ")
+    links = []
+    for u in _RE_LINK.findall(t):
+        u = u.rstrip(".,);")
+        if u not in links:
+            links.append(u)
+        if len(links) >= 4:
+            break
+    days = None
+    m = _RE_DAYS.search(t)
+    if m:
+        days = "%s %s" % (m.group(1), m.group(2))
+    return {"codes": codes[:4], "amount": amt, "links": links, "term": days}