# -*- coding: utf-8 -*-
"""Волна 6: CVE-фиды, абуз-борды, трафик/SEO, ещё блоги. Только живые (>=2 записи)."""
import sys, os, json, datetime, concurrent.futures as cf
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
os.chdir(r"C:/Users/User/tmp/omni-parse")
from omni import discover, sources, config
config.load()

C = []


def s(name, cat, kind, url, ad="rss", tags=(), extra=None):
    C.append({"name": name, "category": cat, "kind": kind, "url": url,
              "adapter": ad, "tags": list(tags), "extra": extra or {}})


# --- CVE / уязвимости ---
s("cvefeed_critical", "security", "rss", "https://cvefeed.io/rssfeed/severity/critical", "rss", ("cve", "security"))
s("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",))
s("cisa_kev_json", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "cisa_kev", ("cve", "kev"))
s("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=20", "nvd", ("cve", "nvd"))
s("gh_advisories", "security", "atom", "https://github.com/advisories.atom", "rss", ("cve", "github"))
s("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"))
s("cve_org_blog", "security", "rss", "https://www.cve.org/RSS/News", "rss", ("cve",))
s("msrc_blog", "security", "rss", "https://msrc.microsoft.com/blog/feed", "rss", ("cve", "microsoft"))
s("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",))
s("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"))

# --- абуз-борды и трафик ---
s("ab_seochao_arbitrage", "abuse", "rss", "https://seochao.ru/forums/arbitrazh-trafika.85/index.rss", "rss", ("traffic", "abuse"))
s("ab_seochao_accounts", "abuse", "rss", "https://seochao.ru/forums/akkaunty.60/index.rss", "rss", ("accounts", "abuse"))
s("ab_blackhatworld", "abuse", "rss", "https://www.blackhatworld.com/forums/misc.18/index.rss", "rss", ("traffic", "abuse"))
s("ab_bhw_marketplace", "abuse", "rss", "https://www.blackhatworld.com/forums/marketplace.141/index.rss", "rss", ("market", "abuse"))
s("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"))
s("ab_zennoposter_blog", "abuse", "rss", "https://zennolab.com/discussion/forums/novosti.30/index.rss", "rss", ("automation", "abuse"))

# --- AI / инфра ---
for n, u in [
    ("swebench_blog", "https://www.swebench.com/blog/rss.xml"),
    ("evalplus", "https://github.com/evalplus/evalplus/releases.atom"),
    ("openai_python_rel", "https://github.com/openai/openai-python/releases.atom"),
    ("vllm_blog", "https://blog.vllm.ai/feed.xml"),
    ("pytorch_blog", "https://pytorch.org/feed.xml"),
    ("tensorflow_blog", "https://blog.tensorflow.org/feeds/posts/default"),
    ("ray_blog", "https://www.anyscale.com/blog/rss.xml"),
    ("modal_changelog", "https://modal.com/blog/rss.xml"),
    ("langchain_blog", "https://blog.langchain.dev/rss/"),
    ("llamaindex_blog", "https://www.llamaindex.ai/blog/rss.xml"),
    ("qdrant_blog", "https://qdrant.tech/blog/rss.xml"),
    ("weaviate_blog", "https://weaviate.io/blog/rss.xml"),
    ("chroma_blog", "https://www.trychroma.com/blog/rss.xml"),
    ("pinecone_blog", "https://www.pinecone.io/blog/rss.xml"),
    ("neo4j_blog", "https://neo4j.com/blog/feed/"),
    ("redis_blog", "https://redis.io/blog/feed/"),
    ("clickhouse_blog", "https://clickhouse.com/blog/rss.xml"),
    ("duckdb_news", "https://duckdb.org/news/feed.xml"),
    ("sqlite_news", "https://sqlite.org/news.rss"),
    ("postgres_news", "https://www.postgresql.org/news.rss"),
]:
    s(n, "company", "rss", u, "rss", ("infra", "blog"))

# --- форумы/комьюнити ---
for n, u in [
    ("dc_duckdb", "https://discuss.duckdb.org/latest.json"),
    ("dc_supabase", "https://github.com/supabase/supabase/discussions.atom"),
    ("dc_cloudflare", "https://community.cloudflare.com/latest.json"),
    ("dc_vercel", "https://github.com/vercel/next.js/discussions.atom"),
]:
    s(n, "forum", "json" if u.endswith(".json") else "atom", u,
      "discourse" if u.endswith(".json") else "rss", ("forum",))


def probe_one(row):
    try:
        ok, n, ad = discover.probe(row["url"], row["category"], limit=6)
        return row, ok, n, ad
    except Exception as e:
        return row, False, 0, str(e)[:40]


if __name__ == "__main__":
    have = {x.url.rstrip("/") for x in sources.all_sources()}
    cand = [r for r in C if r["url"].rstrip("/") not in have]
    print("кандидатов:", len(C), "| новых:", len(cand))
    res = []
    with cf.ThreadPoolExecutor(14) as ex:
        for row, ok, n, ad in ex.map(probe_one, cand):
            res.append((row, ok, n, ad))
    live = [(r, n, ad) for r, ok, n, ad in res if ok and n >= 2]
    print("живых:", len(live), "| мёртвых:", len(res) - len(live))
    for r, n, ad in live:
        print("  %-24s %-16s %s %s" % (r["name"], ad, r["category"], n))
    print("--- мёртвые:", ", ".join(r["name"] for r, ok, n, ad in res if not (ok and n >= 2)))
    head = ('"""Волна 6 (авто-проверка, только items >= 2): CVE, абуз-борды, инфра-блоги, форумы.\n'
            'Дата %s, живых %d."""\n\n' % (datetime.date.today(), len(live)))
    body = ['def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):',
            '    return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,',
            '            "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}',
            "", "", "EXTRA6 = ["]
    for r, n, ad in live:
        body.append('    S("%s", "%s", "%s", "%s", "%s", %r, limit=15, extra=%r),\n' %
                    (r["name"], r["category"], r["kind"], r["url"], r["adapter"], tuple(r["tags"]), r["extra"]))
    body.append("]")
    open("omni/sources_ext5.py", "w", encoding="utf-8", newline="\n").write(head + "".join(body).replace("),\n\n", "),\n") + "\n")
    print("written omni/sources_ext5.py")