абузы: extract промокодов/сумм/ссылок в API и панели, +16 проверенных источников (TG/reddit/discourse/RSS/github/HN), чистка HTML в деталях, клэмп days

2026-10-27 01:50:05 +03:00
parent 8de421b6af
commit b09cb5f873
22 changed files with 552 additions and 2992 deletions
--- a/omni/abuse.py
+++ b/omni/abuse.py
@@ -235,9 +235,11 @@ def _rows(limit=400, tag=None, q=None, since_days=None):
        out = []
        for r in con.execute(sql, args):
            hits = [t for t in (r[6] or "").split(",") if t]
+            body = (r[4] or "")
            out.append({"id": r[0], "source": r[1], "url": r[2], "title": r[3],
-                        "text": (r[4] or "")[:600], "published": r[5],
-                        "tags": hits, "abuse_score": r[8], "rank": r[7]})
+                        "text": body[:600], "published": r[5],
+                        "tags": hits, "abuse_score": r[8], "rank": r[7],
+                        "find": extract((r[3] or "") + " " + body)})
        return out
    finally:
        con.close()
@@ -273,4 +275,43 @@ def scan(limit=120, tag=None, q=None, since_days=None, window=6000):
    except Exception:
        pass
    return {"items": items[:limit], "total": total, "tags": counts,
-            "labels": TAG_LABEL}
+            "labels": TAG_LABEL}
+
+_RE_CODE = re.compile(r"(?:промокод|промо-?код|promo\s*code|coupon|купо[нш]|code)[^\n:：]{0,12}[:：\s]+([A-Z0-9][A-Z0-9_\-]{3,23})", re.I)
+_RE_CODE2 = re.compile(r"\b([A-Z]{2,}[0-9]{2,}[A-Z0-9_\-]{0,14})\b")
+_RE_AMT = re.compile(r"(\d[\d\s.,]{0,12})\s*(млрд|млн|тыс|billion|million|k|m|b|токен\w*|tokens?|credits?|\$|₽|юан\w*)", re.I)
+_RE_LINK = re.compile(r"(?:https?://|t\.me/)[^\s)\]\"'<>]{4,120}")
+_RE_DAYS = re.compile(r"(\d{1,3})\s*(дн\w*|day[s]?|час\w*|hour[s]?|недел\w*|week[s]?)", re.I)
+
+
+def extract(text):
+    """Достаёт из текста абуза полезное: промокоды, объём, ссылки, срок."""
+    t = (text or "")[:4000]
+    codes = []
+    for m in _RE_CODE.finditer(t):
+        c = (m.group(1) or "").strip(" .,:;")
+        if c and c.upper() not in {x.upper() for x in codes}:
+            codes.append(c)
+    if not codes and re.search(r"промо|promo|code|код|gift|купо|coupon|подар", t, re.I):
+        for m in _RE_CODE2.finditer(t):
+            c = m.group(1)
+            if c and c.upper() not in {x.upper() for x in codes}:
+                codes.append(c)
+            if len(codes) >= 3:
+                break
+    amt = None
+    m = _RE_AMT.search(t)
+    if m:
+        amt = ("%s %s" % (m.group(1).strip(), m.group(2))).replace("  ", " ")
+    links = []
+    for u in _RE_LINK.findall(t):
+        u = u.rstrip(".,);")
+        if u not in links:
+            links.append(u)
+        if len(links) >= 4:
+            break
+    days = None
+    m = _RE_DAYS.search(t)
+    if m:
+        days = "%s %s" % (m.group(1), m.group(2))
+    return {"codes": codes[:4], "amount": amt, "links": links, "term": days}
--- a/omni/panel_api.py
+++ b/omni/panel_api.py
@@ -21,9 +21,11 @@ def _clean(v, n=0):
    import html as _h
    if not v:
        return ""
-    t = re.sub(r"<[^>]{1,200}>", " ", str(v))
-    for _ in range(2):
+    t = str(v)
+    # unescape и снятие тегов в цикле: иначе &lt;b&gt; после unescape снова станет тегом
+    for _ in range(3):
        t = _h.unescape(t)
+        t = re.sub(r"<[^>]{1,200}>", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return t[:n].rstrip() if n else t

@@ -39,7 +41,8 @@ def timeline(days=30, cat=None, mode="pub"):
    mode="pub" — по дате публикации (или дате попадания в базу, если публикации нет);
    mode="seen" — строго по времени импорта: «что комбайн принёс за день».
    """
-    days = max(1, min(365, int(days or 30)))
+    days = 30 if days is None else int(days or 0)
+    days = max(1, min(365, days))
    bucket = BUCKET_SEEN if mode == "seen" else BUCKET
    names = [s.name for s in sources.all_sources() if s.category == cat] if cat not in (None, "", "all") else None
    where, args = "", []
--- a/omni/sources.py
+++ b/omni/sources.py
@@ -18,6 +18,7 @@ from .abuse_sources import ABUSE as ABUSE_DEEP
 from .sources_tg_abuse import TG_ABUSE
 from .sources_ext4 import EXTRA5
 from .sources_ext5 import EXTRA6
+from .sources_fresh import fresh as _fresh_sources
 from .sources_x import X_SOURCES


@@ -298,7 +299,16 @@ def _gen_sources():


 def all_sources():
-    """Реестр + пользовательские источники из панели, без дублей по url."""
+    """Реестр + пользовательские источники из панели, без дублей по url и по имени.
+
+    Дедуп по двум ключам обязателен:
+      * url — одна площадка не должна обходиться дважды (двойные записи в базе);
+      * name — имя источника является первичным ключом в source_flags/runs, поэтому
+        зеркала одного и того же ресурса (rss.arxiv vs export.arxiv, old.reddit vs
+        www.reddit) обязаны схлопываться, иначе переключение/состояние «плывут».
+    Побеждает первое вхождение; порядок: реестр → сгенерированные → свежая волна →
+    пользовательские → abuse-radar → panels.json.
+    """
    from . import source_state
    extra = []
    for d in source_state.custom():
@@ -307,19 +317,20 @@ def all_sources():
        d = {k: v for k, v in d.items() if k in
             ("name", "category", "kind", "url", "adapter", "tags", "limit", "detail", "extra")}
        extra.append(Source(**d))
-    seen, out = set(), []
-    for s in SOURCES + _gen_sources() + extra + radar_sources() + panel_sources():
-        # дедуп строго по url: одна площадка = один источник, даже если пришла из
-        # реестра, radar_sources и panels.json одновременно (иначе двойной обход и
-        # двойные записи в базе). Побеждает первое вхождение — порядок списка выше.
+    seen_url, seen_name, out = set(), set(), []
+    for s in SOURCES + _gen_sources() + _fresh_sources(S) + extra + radar_sources() + panel_sources():
        key = (s.url or "").rstrip("/").lower()
-        if key and key in seen:
+        name = (s.name or "").strip().lower()
+        if (key and key in seen_url) or (name and name in seen_name):
            continue
        if key:
-            seen.add(key)
+            seen_url.add(key)
+        if name:
+            seen_name.add(name)
        out.append(s)
    return out

+
 def enabled_sources(include_opt_in=False):
    """Только включённые: выключенные в панели источники не дёргаем.

--- a/omni/sources_abuse.py
+++ b/omni/sources_abuse.py
@@ -71,4 +71,22 @@ ABUSE_LIVE = [
    A("ab_hn_account_generator", "abuse", "site", "https://hn.algolia.com/?query=account+generator", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "account generator"}),
    A("ab_hn_api_key_leak", "abuse", "site", "https://hn.algolia.com/?query=api+key+leak", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "api key leak"}),
    A("ab_hn_chatgpt_free", "abuse", "site", "https://hn.algolia.com/?query=chatgpt+free", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "chatgpt free"}),
+
+    # ---- партия 27.10: проверенные вручную (каждый отдаёт >0 items) ----
+    A("ab_tg_gpt_frees", "abuse", "site", "https://t.me/s/gpt_frees", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_promokody", "abuse", "site", "https://t.me/s/promokody", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_promokodq", "abuse", "site", "https://t.me/s/promokodq", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_cracksoftwux", "abuse", "site", "https://t.me/s/crackSoftiowux", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_taynikstore", "abuse", "site", "https://t.me/s/taynikstoree", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_nodelocrus", "abuse", "site", "https://t.me/s/NODELOCRUS", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_aivision", "abuse", "site", "https://t.me/s/a_vision_ai", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_tg_sms_free", "abuse", "site", "https://t.me/s/freenumbers", "tg_web", ('abuse', 'abuse', 'tg'), limit=20),
+    A("ab_reddit_freebies2", "abuse", "site", "https://www.reddit.com/r/freebies/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20),
+    A("ab_reddit_beermoney", "abuse", "site", "https://www.reddit.com/r/beermoney/.json?limit=25", "reddit_listing", ('abuse', 'abuse', 'reddit'), limit=20),
+    A("ab_dis_n8n", "abuse", "site", "https://community.n8n.io/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
+    A("ab_dis_vercel", "abuse", "site", "https://community.vercel.com/latest.json", "discourse", ('abuse', 'abuse'), limit=20),
+    A("ab_rss_slickdeals2", "abuse", "site", "https://slickdeals.net/newsearch.php?mode=frontpage&searcharea=deals&searchin=first&rss=1", "rss", ('abuse', 'abuse'), limit=20),
+    A("ab_gh_freeapi_keys2", "abuse", "site", "https://github.com/search?q=free+api+keys+pool&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "free api keys pool", "sort": "updated"}),
+    A("ab_gh_leak_pool2", "abuse", "site", "https://github.com/search?q=leaked+api+keys+checker&type=repositories", "github_search", ('abuse', 'abuse', 'github'), limit=15, extra={"q": "leaked api keys checker", "sort": "updated"}),
+    A("ab_hn_free_tokens2", "abuse", "site", "https://hn.algolia.com/?query=free+tokens+api", "hn_search", ('abuse', 'abuse', 'hn'), limit=15, extra={"q": "free tokens api"}),
 ]
--- a/omni/sources_fresh.py
+++ b/omni/sources_fresh.py
@@ -5,6 +5,16 @@
 Импортов из omni.sources нет намеренно (циклический импорт).
 """

+import datetime as _dt
+
+
+def _nvd_week():
+    """NVD 2.0 принимает обе даты и окно <=120 дней: скользящая неделя."""
+    now = _dt.datetime.utcnow()
+    since = (now - _dt.timedelta(days=7)).strftime("%Y-%m-%dT00:00:00.000")
+    return ("https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=30"
+            "&lastModStartDate=%s&lastModEndDate=%s" % (since, now.strftime("%Y-%m-%dT%H:%M:%S.000")))
+

 def fresh(S):
    """S — конструктор Source; возвращает только проверенные сетью источники."""
@@ -33,7 +43,6 @@ def fresh(S):
        S("citizenlab", "security", "rss", "https://citizenlab.ca/feed/", "rss", ("apt", "research")),
        S("csoonline", "security", "rss", "https://www.csoonline.com/feed/", "rss", ("news")),
        S("cybersecuritydive", "security", "rss", "https://www.cybersecuritydive.com/feeds/news/", "rss", ("news")),
-        S("elastic_security_labs", "security", "rss", "https://www.elastic.co/security-labs/rss/feed", "rss", ("research")),
        S("github_advisories_api", "security", "json", "https://api.github.com/advisories?per_page=30&sort=published&direction=desc", "ghsa", ("cve", "ghsa")),
        S("habr_infosecurity", "security", "rss", "https://habr.com/ru/rss/hub/infosecurity/", "rss", ("ru", "research")),
        S("habr_news", "security", "rss", "https://habr.com/ru/rss/news/", "rss", ("ru", "news")),
@@ -45,6 +54,8 @@ def fresh(S):
        S("microsoft_security_blog", "security", "rss", "https://www.microsoft.com/en-us/security/blog/feed/", "rss", ("research")),
        S("mozilla_advisories", "security", "html", "https://www.mozilla.org/en-US/security/advisories/", "html_listing", ("cve", "browser")),
        S("ncsc_uk", "security", "rss", "https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml", "rss", ("gov", "cve")),
+        S("nvd_exploited_kw", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=25&keywordSearch=exploited%20in%20the%20wild", "nvd", ("cve", "exploited")),
+        S("nvd_recent", "security", "json", _nvd_week(), "nvd", ("cve", "nvd")),
        S("paloalto_blog", "security", "rss", "https://www.paloaltonetworks.com/blog/feed/", "rss", ("research")),
        S("seclists_fulldisclosure", "security", "rss", "https://seclists.org/rss/fulldisclosure.rss", "rss", ("mailing", "exploit")),
        S("seclists_nmap", "security", "rss", "https://seclists.org/rss/nmap-dev.rss", "rss", ("mailing")),