# -*- coding: utf-8 -*-
"""Волна 5: кандидаты источников -> живая проверка реальным адаптером -> omni/sources_ext4.py.
Пишем только те, что реально отдали >= 2 записи. Никаких "на глаз".
"""
import sys, os, json, concurrent.futures as cf
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
os.chdir(r"C:/Users/User/tmp/omni-parse")

from omni import discover, sources, config
config.load()

C = []  # (name, category, kind, url, adapter, tags)


def s(name, cat, kind, url, ad="rss", tags=()):
    C.append((name, cat, kind, url, ad, tuple(tags)))


# ---------- security ----------
for n, u in [
    ("krebs", "https://krebsonsecurity.com/feed/"),
    ("thn", "https://feeds.feedburner.com/TheHackersNews"),
    ("securityweek", "https://www.securityweek.com/feed/"),
    ("darkreading", "https://www.darkreading.com/rss.xml"),
    ("portswigger_research", "https://portswigger.net/research/rss"),
    ("projectzero", "https://googleprojectzero.blogspot.com/feeds/posts/default"),
    ("talos", "https://blog.talosintelligence.com/rss/"),
    ("sans_isc", "https://isc.sans.edu/rssfeed.xml"),
    ("exploitdb", "https://www.exploit-db.com/rss.xml"),
    ("packetstorm", "https://packetstormsecurity.com/feeds"),
    ("unit42", "https://unit42.paloaltonetworks.com/feed/"),
    ("checkpoint_research", "https://research.checkpoint.com/feed/"),
    ("securelist", "https://securelist.com/feed/"),
    ("welivesecurity", "https://www.welivesecurity.com/en/rss/feed/"),
    ("malwarebytes", "https://www.malwarebytes.com/blog/feed/index.xml"),
    ("sentinelone_labs", "https://www.sentinelone.com/labs/feed/"),
    ("hackread", "https://www.hackread.com/feed/"),
    ("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
    ("gbhackers", "https://gbhackers.com/feed/"),
    ("cybersecuritynews", "https://cybersecuritynews.com/feed/"),
    ("securityaffairs", "https://securityaffairs.com/feed"),
    ("therecord", "https://therecord.media/feed"),
    ("riskybusiness", "https://risky.biz/feeds/risky-business.xml"),
    ("trailofbits", "https://blog.trailofbits.com/feed/"),
    ("snyk_blog", "https://snyk.io/blog/feed/"),
    ("wiz_blog", "https://www.wiz.io/feed/rss.xml"),
    ("aquasec", "https://blog.aquasec.com/rss.xml"),
    ("sysdig", "https://sysdig.com/blog/feed/"),
    ("crowdstrike", "https://www.crowdstrike.com/blog/feed/"),
    ("mandiant", "https://www.mandiant.com/resources/blog/rss.xml"),
    ("recordedfuture", "https://www.recordedfuture.com/feed"),
    ("intigriti", "https://blog.intigriti.com/feed/"),
    ("yeswehack", "https://www.yeswehack.com/feed"),
    ("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
    ("hackerone_blog", "https://www.hackerone.com/blog.rss"),
]:
    s(n, "security", "rss", u, "rss", ("security", "research"))

# ---------- ai / company ----------
for n, u in [
    ("openai_news", "https://openai.com/news/rss.xml"),
    ("deepmind_blog", "https://deepmind.google/blog/rss.xml"),
    ("google_ai_blog", "https://blog.google/technology/ai/rss/"),
    ("hf_blog", "https://huggingface.co/blog/feed.xml"),
    ("the_decoder", "https://the-decoder.com/feed/"),
    ("simonwillison", "https://simonwillison.net/atom/everything/"),
    ("latent_space", "https://www.latent.space/feed"),
    ("importai", "https://importai.substack.com/feed"),
    ("oneusefulthing", "https://www.oneusefulthing.org/feed"),
    ("bair_blog", "https://bair.berkeley.edu/blog/feed.xml"),
    ("lilianweng", "https://lilianweng.github.io/feed.xml"),
    ("jalammar", "https://jalammar.github.io/feed.xml"),
    ("msr_blog", "https://www.microsoft.com/en-us/research/feed/"),
    ("aws_ml_blog", "https://aws.amazon.com/blogs/machine-learning/feed/"),
    ("nvidia_blog", "https://blogs.nvidia.com/feed/"),
    ("meta_ai_blog", "https://ai.meta.com/blog/rss/"),
    ("stability_news", "https://stability.ai/news/rss.xml"),
    ("cohere_blog", "https://cohere.com/blog/rss.xml"),
    ("mistral_news", "https://mistral.ai/news/rss.xml"),
    ("together_blog", "https://www.together.ai/blog/rss.xml"),
    ("eleuther_blog", "https://blog.eleuther.ai/feed.xml"),
    ("karpathy", "https://karpathy.github.io/feed.xml"),
    ("raschka", "https://magazine.sebastianraschka.com/feed"),
    ("interconnects", "https://www.interconnects.ai/feed"),
    ("alignment_forum", "https://www.alignmentforum.org/feed.xml"),
    ("lesswrong", "https://www.lesswrong.com/feed.xml"),
]:
    s(n, "company", "rss", u, "rss", ("ai", "blog"))

# ---------- papers ----------
for n, u in [
    ("arxiv_cs_ai", "https://export.arxiv.org/rss/cs.AI"),
    ("arxiv_cs_cr", "https://export.arxiv.org/rss/cs.CR"),
    ("arxiv_cs_cl", "https://export.arxiv.org/rss/cs.CL"),
    ("arxiv_cs_lg", "https://export.arxiv.org/rss/cs.LG"),
]:
    s(n, "papers", "rss", u, "rss", ("papers", "arxiv"))

# ---------- community / forum (discourse latest.json) ----------
for n, u in [
    ("dc_pytorch", "https://discuss.pytorch.org/latest.json"),
    ("dc_huggingface", "https://discuss.huggingface.co/latest.json"),
    ("dc_elastic", "https://discuss.elastic.co/latest.json"),
    ("dc_gitlab", "https://forum.gitlab.com/latest.json"),
    ("dc_homeassistant", "https://community.home-assistant.io/latest.json"),
    ("dc_manjaro", "https://forum.manjaro.org/latest.json"),
    ("dc_grafana", "https://community.grafana.com/latest.json"),
    ("dc_vue", "https://forum.vuejs.org/latest.json"),
    ("dc_kotlin", "https://discuss.kotlinlang.org/latest.json"),
    ("dc_bitwarden", "https://community.bitwarden.com/latest.json"),
    ("dc_openwrt", "https://forum.openwrt.org/latest.json"),
    ("dc_python", "https://discuss.python.org/latest.json"),
    ("dc_jupyter", "https://discuss.jupyter.org/latest.json"),
    ("dc_fly", "https://community.fly.io/latest.json"),
    ("dc_meta", "https://meta.discourse.org/latest.json"),
]:
    s(n, "forum", "json", u, "discourse", ("forum", "discourse"))

# ---------- reddit ----------
for n, sub in [("r_netsec", "netsec"), ("r_bugbounty", "bugbounty"), ("r_llmdevs", "LLMDevs"),
               ("r_cybersecurity", "cybersecurity"), ("r/sideproject", "SideProject"),
               ("r/SaaS", "SaaS"), ("r/selfhosted", "selfhosted")]:
    s(n, "community", "html", "https://www.reddit.com/r/%s/new/" % sub, "reddit_listing", ("reddit",))

# ---------- telegram ----------
for n, ch in [("tg_seeallochnaya", "seeallochnaya"), ("tg_data_secrets", "data_secrets"),
              ("tg_ai_newz", "ai_newz"), ("tg_addmetoai", "addmetoai")]:
    s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))

# ---------- marketing / traffic forums ----------
s("bhw_index", "forum", "rss", "https://www.blackhatworld.com/forums/-/index.rss", "rss", ("traffic", "seo"))
s("so_llm", "forum", "rss", "https://stackoverflow.com/feeds/tag?tagnames=llm&sort=newest", "rss", ("qa",))

# ---------- releases ----------
for n, repo in [("gh_rel_aider", "Aider-AI/aider"), ("gh_rel_cline", "cline/cline"),
                ("gh_rel_continue", "continuedev/continue"), ("gh_rel_openhands", "All-Hands-AI/OpenHands"),
                ("gh_rel_unsloth", "unslothai/unsloth"), ("gh_rel_axolotl", "axolotl-ai-cloud/axolotl"),
                ("gh_rel_sglang", "sgl-project/sglang"), ("gh_rel_dspy", "stanfordnlp/dspy"),
                ("gh_rel_crewai", "crewAIInc/crewAI"), ("gh_rel_mem0", "mem0ai/mem0"),
                ("gh_rel_graphrag", "microsoft/graphrag"), ("gh_rel_llamaindex", "run-llama/llama_index"),
                ("gh_rel_semgrep", "semgrep/semgrep"), ("gh_rel_nuclei", "projectdiscovery/nuclei")]:
    s(n, "release", "atom", "https://github.com/%s/releases.atom" % repo, "rss", ("release", "github"))

# ---------- cn ----------
for n, u in [("qbitai", "https://www.qbitai.com/feed"), ("jiqizhixin", "https://www.jiqizhixin.com/rss"),
             ("infoq_cn", "https://www.infoq.cn/feed"), ("36kr", "https://36kr.com/feed")]:
    s(n, "company_cn", "rss", u, "rss", ("cn",))


# ---------- волна 5b: инфра/дев-блоги, AI-провайдеры, ещё форумы ----------
for n, u in [
    ("cloudflare_blog", "https://blog.cloudflare.com/rss/"),
    ("vercel_blog", "https://vercel.com/blog/rss.xml"),
    ("github_blog", "https://github.blog/feed/"),
    ("gitlab_blog", "https://about.gitlab.com/atom.xml"),
    ("docker_blog", "https://www.docker.com/blog/feed/"),
    ("supabase_blog", "https://supabase.com/blog/rss.xml"),
    ("neon_blog", "https://neon.tech/blog/rss.xml"),
    ("fly_blog", "https://fly.io/blog/feed.xml"),
    ("stripe_blog", "https://stripe.com/blog/feed.rss"),
    ("elastic_blog", "https://www.elastic.co/blog/feed"),
    ("anthropic_news", "https://www.anthropic.com/news/rss.xml"),
    ("openrouter_blog", "https://openrouter.ai/blog/rss.xml"),
    ("groq_blog", "https://groq.com/blog/rss.xml"),
    ("cerebras_blog", "https://www.cerebras.ai/blog/rss.xml"),
    ("ai2_blog", "https://blog.allenai.org/feed"),
    ("databricks_blog", "https://www.databricks.com/feed"),
    ("wandb_blog", "https://wandb.ai/fully-connected/rss.xml"),
    ("modal_blog", "https://modal.com/blog/feed.xml"),
    ("replicate_blog", "https://replicate.com/blog/rss.xml"),
    ("baseten_blog", "https://www.baseten.co/blog/rss.xml"),
    ("scale_blog", "https://scale.com/blog/rss.xml"),
    ("ooba_blog", "https://blog.ollama.com/rss.xml"),
    ("lmarena_blog", "https://lmarena.ai/blog/rss.xml"),
    ("arxiv_cs_se", "https://export.arxiv.org/rss/cs.SE"),
    ("arxiv_cs_ir", "https://export.arxiv.org/rss/cs.IR"),
    ("arxiv_stat_ml", "https://export.arxiv.org/rss/stat.ML"),
    ("thegradient", "https://thegradient.pub/rss/"),
    ("paperswithcode_blog", "https://paperswithcode.com/blog/rss"),
    ("mlsafe", "https://www.mlsafety.org/rss.xml"),
]:
    s(n, "company", "rss", u, "rss", ("infra", "blog"))

for n, u in [
    ("dc_rust_internals", "https://internals.rust-lang.org/latest.json"),
    ("dc_racket", "https://discourse.racket-lang.org/latest.json"),
    ("dc_nix", "https://discourse.nixos.org/latest.json"),
    ("dc_godot", "https://forum.godotengine.org/latest.json"),
    ("dc_swift", "https://forums.swift.org/latest.json"),
    ("dc_go", "https://forum.golangbridge.org/latest.json"),
    ("dc_raspberry", "https://forums.raspberrypi.com/latest.json"),
    ("dc_tryghost", "https://forum.ghost.org/latest.json"),
    ("dc_blender", "https://devtalk.blender.org/latest.json"),
    ("dc_obsidian", "https://forum.obsidian.md/latest.json"),
]:
    s(n, "forum", "json", u, "discourse", ("forum", "discourse"))

for n, ch in [("tg_ai_machinelearning", "ai_machinelearning"), ("tg_gpt_main", "gpt_main"),
              ("tg_neuralshit", "neuralshit"), ("tg_llm_underhood", "llm_underhood"),
              ("tg_abuz_ai_soft2", "abuz_ai_soft"), ("tg_cybersecurity_ch", "cybersecurity_channel")]:
    s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))


def probe_one(row):
    name, cat, kind, url, ad, tags = row
    try:
        ok, n, used = discover.probe(url, cat, limit=6)
        return (name, cat, kind, url, ad, tags, ok, n, used)
    except Exception as e:
        return (name, cat, kind, url, ad, tags, False, 0, str(e)[:40])


if __name__ == "__main__":
    have = {s_.url.rstrip("/") for s_ in sources.all_sources()}
    cand = [r for r in C if r[3].rstrip("/") not in have]
    print("кандидатов:", len(C), "| новых к проверке:", len(cand))
    res = []
    with cf.ThreadPoolExecutor(16) as ex:
        for r in ex.map(probe_one, cand):
            res.append(r)
    live = [r for r in res if r[6] and r[7] >= 2]
    print("живых:", len(live), "| мёртвых:", len(res) - len(live))
    for r in sorted(res, key=lambda x: -x[7])[:12]:
        print("  %-22s %s %s" % (r[0], r[8], r[7]))
    print("--- skip")
    print(", ".join(x[0] for x in res if x not in live))
    # write module
    head = ('"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):\n'
            'security/AI/форумы/reddit/tg/релизы/CN. Дата прогона %s, живых %d."""\n\n'
            % (__import__("datetime").date.today(), len(live)))
    body = ["def S(name, category, kind, url, adapter=\"rss\", tags=(), limit=15, detail=False, extra=None):",
            "    return {\"name\": name, \"category\": category, \"kind\": kind, \"url\": url, \"adapter\": adapter,",
            "            \"tags\": tuple(tags), \"limit\": limit, \"detail\": detail, \"extra\": extra or {}}",
            "", "", "EXTRA5 = ["]
    for row in live:
        name, cat, kind, url, ad, tags = row[:6]
        body.append('    S("%s", "%s", "%s", "%s", "%s", %r, limit=15),' % (name, cat, kind, url, ad, tags))
    body.append("]")
    open("omni/sources_ext4.py", "w", encoding="utf-8", newline="\n").write(head + "\n".join(body) + "\n")
    print("written omni/sources_ext4.py")