241 lines
13 KiB
Python
241 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Волна 5: кандидаты источников -> живая проверка реальным адаптером -> omni/sources_ext4.py.
|
|
Пишем только те, что реально отдали >= 2 записи. Никаких "на глаз".
|
|
"""
|
|
import sys, os, json, concurrent.futures as cf
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
|
|
os.chdir(r"C:/Users/User/tmp/omni-parse")
|
|
|
|
from omni import discover, sources, config
|
|
config.load()
|
|
|
|
C = [] # (name, category, kind, url, adapter, tags)
|
|
|
|
|
|
def s(name, cat, kind, url, ad="rss", tags=()):
|
|
C.append((name, cat, kind, url, ad, tuple(tags)))
|
|
|
|
|
|
# ---------- security ----------
|
|
for n, u in [
|
|
("krebs", "https://krebsonsecurity.com/feed/"),
|
|
("thn", "https://feeds.feedburner.com/TheHackersNews"),
|
|
("securityweek", "https://www.securityweek.com/feed/"),
|
|
("darkreading", "https://www.darkreading.com/rss.xml"),
|
|
("portswigger_research", "https://portswigger.net/research/rss"),
|
|
("projectzero", "https://googleprojectzero.blogspot.com/feeds/posts/default"),
|
|
("talos", "https://blog.talosintelligence.com/rss/"),
|
|
("sans_isc", "https://isc.sans.edu/rssfeed.xml"),
|
|
("exploitdb", "https://www.exploit-db.com/rss.xml"),
|
|
("packetstorm", "https://packetstormsecurity.com/feeds"),
|
|
("unit42", "https://unit42.paloaltonetworks.com/feed/"),
|
|
("checkpoint_research", "https://research.checkpoint.com/feed/"),
|
|
("securelist", "https://securelist.com/feed/"),
|
|
("welivesecurity", "https://www.welivesecurity.com/en/rss/feed/"),
|
|
("malwarebytes", "https://www.malwarebytes.com/blog/feed/index.xml"),
|
|
("sentinelone_labs", "https://www.sentinelone.com/labs/feed/"),
|
|
("hackread", "https://www.hackread.com/feed/"),
|
|
("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
|
|
("gbhackers", "https://gbhackers.com/feed/"),
|
|
("cybersecuritynews", "https://cybersecuritynews.com/feed/"),
|
|
("securityaffairs", "https://securityaffairs.com/feed"),
|
|
("therecord", "https://therecord.media/feed"),
|
|
("riskybusiness", "https://risky.biz/feeds/risky-business.xml"),
|
|
("trailofbits", "https://blog.trailofbits.com/feed/"),
|
|
("snyk_blog", "https://snyk.io/blog/feed/"),
|
|
("wiz_blog", "https://www.wiz.io/feed/rss.xml"),
|
|
("aquasec", "https://blog.aquasec.com/rss.xml"),
|
|
("sysdig", "https://sysdig.com/blog/feed/"),
|
|
("crowdstrike", "https://www.crowdstrike.com/blog/feed/"),
|
|
("mandiant", "https://www.mandiant.com/resources/blog/rss.xml"),
|
|
("recordedfuture", "https://www.recordedfuture.com/feed"),
|
|
("intigriti", "https://blog.intigriti.com/feed/"),
|
|
("yeswehack", "https://www.yeswehack.com/feed"),
|
|
("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
|
|
("hackerone_blog", "https://www.hackerone.com/blog.rss"),
|
|
]:
|
|
s(n, "security", "rss", u, "rss", ("security", "research"))
|
|
|
|
# ---------- ai / company ----------
|
|
for n, u in [
|
|
("openai_news", "https://openai.com/news/rss.xml"),
|
|
("deepmind_blog", "https://deepmind.google/blog/rss.xml"),
|
|
("google_ai_blog", "https://blog.google/technology/ai/rss/"),
|
|
("hf_blog", "https://huggingface.co/blog/feed.xml"),
|
|
("the_decoder", "https://the-decoder.com/feed/"),
|
|
("simonwillison", "https://simonwillison.net/atom/everything/"),
|
|
("latent_space", "https://www.latent.space/feed"),
|
|
("importai", "https://importai.substack.com/feed"),
|
|
("oneusefulthing", "https://www.oneusefulthing.org/feed"),
|
|
("bair_blog", "https://bair.berkeley.edu/blog/feed.xml"),
|
|
("lilianweng", "https://lilianweng.github.io/feed.xml"),
|
|
("jalammar", "https://jalammar.github.io/feed.xml"),
|
|
("msr_blog", "https://www.microsoft.com/en-us/research/feed/"),
|
|
("aws_ml_blog", "https://aws.amazon.com/blogs/machine-learning/feed/"),
|
|
("nvidia_blog", "https://blogs.nvidia.com/feed/"),
|
|
("meta_ai_blog", "https://ai.meta.com/blog/rss/"),
|
|
("stability_news", "https://stability.ai/news/rss.xml"),
|
|
("cohere_blog", "https://cohere.com/blog/rss.xml"),
|
|
("mistral_news", "https://mistral.ai/news/rss.xml"),
|
|
("together_blog", "https://www.together.ai/blog/rss.xml"),
|
|
("eleuther_blog", "https://blog.eleuther.ai/feed.xml"),
|
|
("karpathy", "https://karpathy.github.io/feed.xml"),
|
|
("raschka", "https://magazine.sebastianraschka.com/feed"),
|
|
("interconnects", "https://www.interconnects.ai/feed"),
|
|
("alignment_forum", "https://www.alignmentforum.org/feed.xml"),
|
|
("lesswrong", "https://www.lesswrong.com/feed.xml"),
|
|
]:
|
|
s(n, "company", "rss", u, "rss", ("ai", "blog"))
|
|
|
|
# ---------- papers ----------
|
|
for n, u in [
|
|
("arxiv_cs_ai", "https://export.arxiv.org/rss/cs.AI"),
|
|
("arxiv_cs_cr", "https://export.arxiv.org/rss/cs.CR"),
|
|
("arxiv_cs_cl", "https://export.arxiv.org/rss/cs.CL"),
|
|
("arxiv_cs_lg", "https://export.arxiv.org/rss/cs.LG"),
|
|
]:
|
|
s(n, "papers", "rss", u, "rss", ("papers", "arxiv"))
|
|
|
|
# ---------- community / forum (discourse latest.json) ----------
|
|
for n, u in [
|
|
("dc_pytorch", "https://discuss.pytorch.org/latest.json"),
|
|
("dc_huggingface", "https://discuss.huggingface.co/latest.json"),
|
|
("dc_elastic", "https://discuss.elastic.co/latest.json"),
|
|
("dc_gitlab", "https://forum.gitlab.com/latest.json"),
|
|
("dc_homeassistant", "https://community.home-assistant.io/latest.json"),
|
|
("dc_manjaro", "https://forum.manjaro.org/latest.json"),
|
|
("dc_grafana", "https://community.grafana.com/latest.json"),
|
|
("dc_vue", "https://forum.vuejs.org/latest.json"),
|
|
("dc_kotlin", "https://discuss.kotlinlang.org/latest.json"),
|
|
("dc_bitwarden", "https://community.bitwarden.com/latest.json"),
|
|
("dc_openwrt", "https://forum.openwrt.org/latest.json"),
|
|
("dc_python", "https://discuss.python.org/latest.json"),
|
|
("dc_jupyter", "https://discuss.jupyter.org/latest.json"),
|
|
("dc_fly", "https://community.fly.io/latest.json"),
|
|
("dc_meta", "https://meta.discourse.org/latest.json"),
|
|
]:
|
|
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
|
|
|
|
# ---------- reddit ----------
|
|
for n, sub in [("r_netsec", "netsec"), ("r_bugbounty", "bugbounty"), ("r_llmdevs", "LLMDevs"),
|
|
("r_cybersecurity", "cybersecurity"), ("r/sideproject", "SideProject"),
|
|
("r/SaaS", "SaaS"), ("r/selfhosted", "selfhosted")]:
|
|
s(n, "community", "html", "https://www.reddit.com/r/%s/new/" % sub, "reddit_listing", ("reddit",))
|
|
|
|
# ---------- telegram ----------
|
|
for n, ch in [("tg_seeallochnaya", "seeallochnaya"), ("tg_data_secrets", "data_secrets"),
|
|
("tg_ai_newz", "ai_newz"), ("tg_addmetoai", "addmetoai")]:
|
|
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
|
|
|
|
# ---------- marketing / traffic forums ----------
|
|
s("bhw_index", "forum", "rss", "https://www.blackhatworld.com/forums/-/index.rss", "rss", ("traffic", "seo"))
|
|
s("so_llm", "forum", "rss", "https://stackoverflow.com/feeds/tag?tagnames=llm&sort=newest", "rss", ("qa",))
|
|
|
|
# ---------- releases ----------
|
|
for n, repo in [("gh_rel_aider", "Aider-AI/aider"), ("gh_rel_cline", "cline/cline"),
|
|
("gh_rel_continue", "continuedev/continue"), ("gh_rel_openhands", "All-Hands-AI/OpenHands"),
|
|
("gh_rel_unsloth", "unslothai/unsloth"), ("gh_rel_axolotl", "axolotl-ai-cloud/axolotl"),
|
|
("gh_rel_sglang", "sgl-project/sglang"), ("gh_rel_dspy", "stanfordnlp/dspy"),
|
|
("gh_rel_crewai", "crewAIInc/crewAI"), ("gh_rel_mem0", "mem0ai/mem0"),
|
|
("gh_rel_graphrag", "microsoft/graphrag"), ("gh_rel_llamaindex", "run-llama/llama_index"),
|
|
("gh_rel_semgrep", "semgrep/semgrep"), ("gh_rel_nuclei", "projectdiscovery/nuclei")]:
|
|
s(n, "release", "atom", "https://github.com/%s/releases.atom" % repo, "rss", ("release", "github"))
|
|
|
|
# ---------- cn ----------
|
|
for n, u in [("qbitai", "https://www.qbitai.com/feed"), ("jiqizhixin", "https://www.jiqizhixin.com/rss"),
|
|
("infoq_cn", "https://www.infoq.cn/feed"), ("36kr", "https://36kr.com/feed")]:
|
|
s(n, "company_cn", "rss", u, "rss", ("cn",))
|
|
|
|
|
|
# ---------- волна 5b: инфра/дев-блоги, AI-провайдеры, ещё форумы ----------
|
|
for n, u in [
|
|
("cloudflare_blog", "https://blog.cloudflare.com/rss/"),
|
|
("vercel_blog", "https://vercel.com/blog/rss.xml"),
|
|
("github_blog", "https://github.blog/feed/"),
|
|
("gitlab_blog", "https://about.gitlab.com/atom.xml"),
|
|
("docker_blog", "https://www.docker.com/blog/feed/"),
|
|
("supabase_blog", "https://supabase.com/blog/rss.xml"),
|
|
("neon_blog", "https://neon.tech/blog/rss.xml"),
|
|
("fly_blog", "https://fly.io/blog/feed.xml"),
|
|
("stripe_blog", "https://stripe.com/blog/feed.rss"),
|
|
("elastic_blog", "https://www.elastic.co/blog/feed"),
|
|
("anthropic_news", "https://www.anthropic.com/news/rss.xml"),
|
|
("openrouter_blog", "https://openrouter.ai/blog/rss.xml"),
|
|
("groq_blog", "https://groq.com/blog/rss.xml"),
|
|
("cerebras_blog", "https://www.cerebras.ai/blog/rss.xml"),
|
|
("ai2_blog", "https://blog.allenai.org/feed"),
|
|
("databricks_blog", "https://www.databricks.com/feed"),
|
|
("wandb_blog", "https://wandb.ai/fully-connected/rss.xml"),
|
|
("modal_blog", "https://modal.com/blog/feed.xml"),
|
|
("replicate_blog", "https://replicate.com/blog/rss.xml"),
|
|
("baseten_blog", "https://www.baseten.co/blog/rss.xml"),
|
|
("scale_blog", "https://scale.com/blog/rss.xml"),
|
|
("ooba_blog", "https://blog.ollama.com/rss.xml"),
|
|
("lmarena_blog", "https://lmarena.ai/blog/rss.xml"),
|
|
("arxiv_cs_se", "https://export.arxiv.org/rss/cs.SE"),
|
|
("arxiv_cs_ir", "https://export.arxiv.org/rss/cs.IR"),
|
|
("arxiv_stat_ml", "https://export.arxiv.org/rss/stat.ML"),
|
|
("thegradient", "https://thegradient.pub/rss/"),
|
|
("paperswithcode_blog", "https://paperswithcode.com/blog/rss"),
|
|
("mlsafe", "https://www.mlsafety.org/rss.xml"),
|
|
]:
|
|
s(n, "company", "rss", u, "rss", ("infra", "blog"))
|
|
|
|
for n, u in [
|
|
("dc_rust_internals", "https://internals.rust-lang.org/latest.json"),
|
|
("dc_racket", "https://discourse.racket-lang.org/latest.json"),
|
|
("dc_nix", "https://discourse.nixos.org/latest.json"),
|
|
("dc_godot", "https://forum.godotengine.org/latest.json"),
|
|
("dc_swift", "https://forums.swift.org/latest.json"),
|
|
("dc_go", "https://forum.golangbridge.org/latest.json"),
|
|
("dc_raspberry", "https://forums.raspberrypi.com/latest.json"),
|
|
("dc_tryghost", "https://forum.ghost.org/latest.json"),
|
|
("dc_blender", "https://devtalk.blender.org/latest.json"),
|
|
("dc_obsidian", "https://forum.obsidian.md/latest.json"),
|
|
]:
|
|
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
|
|
|
|
for n, ch in [("tg_ai_machinelearning", "ai_machinelearning"), ("tg_gpt_main", "gpt_main"),
|
|
("tg_neuralshit", "neuralshit"), ("tg_llm_underhood", "llm_underhood"),
|
|
("tg_abuz_ai_soft2", "abuz_ai_soft"), ("tg_cybersecurity_ch", "cybersecurity_channel")]:
|
|
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
|
|
|
|
|
|
def probe_one(row):
|
|
name, cat, kind, url, ad, tags = row
|
|
try:
|
|
ok, n, used = discover.probe(url, cat, limit=6)
|
|
return (name, cat, kind, url, ad, tags, ok, n, used)
|
|
except Exception as e:
|
|
return (name, cat, kind, url, ad, tags, False, 0, str(e)[:40])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
have = {s_.url.rstrip("/") for s_ in sources.all_sources()}
|
|
cand = [r for r in C if r[3].rstrip("/") not in have]
|
|
print("кандидатов:", len(C), "| новых к проверке:", len(cand))
|
|
res = []
|
|
with cf.ThreadPoolExecutor(16) as ex:
|
|
for r in ex.map(probe_one, cand):
|
|
res.append(r)
|
|
live = [r for r in res if r[6] and r[7] >= 2]
|
|
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
|
|
for r in sorted(res, key=lambda x: -x[7])[:12]:
|
|
print(" %-22s %s %s" % (r[0], r[8], r[7]))
|
|
print("--- skip")
|
|
print(", ".join(x[0] for x in res if x not in live))
|
|
# write module
|
|
head = ('"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):\n'
|
|
'security/AI/форумы/reddit/tg/релизы/CN. Дата прогона %s, живых %d."""\n\n'
|
|
% (__import__("datetime").date.today(), len(live)))
|
|
body = ["def S(name, category, kind, url, adapter=\"rss\", tags=(), limit=15, detail=False, extra=None):",
|
|
" return {\"name\": name, \"category\": category, \"kind\": kind, \"url\": url, \"adapter\": adapter,",
|
|
" \"tags\": tuple(tags), \"limit\": limit, \"detail\": detail, \"extra\": extra or {}}",
|
|
"", "", "EXTRA5 = ["]
|
|
for row in live:
|
|
name, cat, kind, url, ad, tags = row[:6]
|
|
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15),' % (name, cat, kind, url, ad, tags))
|
|
body.append("]")
|
|
open("omni/sources_ext4.py", "w", encoding="utf-8", newline="\n").write(head + "\n".join(body) + "\n")
|
|
print("written omni/sources_ext4.py") |