feat: типизация записей (14 типов, до 3 на запись), чипы и бейджи типов в панели, CSV с типами, /api/types, doctor --only по категориям, волны источников 5-6 (635 всего)
This commit is contained in:
Vlad Gothbreach
241
tools/wave5.py
Normal file
241
tools/wave5.py
Normal file
@@ -0,0 +1,241 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Волна 5: кандидаты источников -> живая проверка реальным адаптером -> omni/sources_ext4.py.
|
||||
Пишем только те, что реально отдали >= 2 записи. Никаких "на глаз".
|
||||
"""
|
||||
import sys, os, json, concurrent.futures as cf
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
|
||||
os.chdir(r"C:/Users/User/tmp/omni-parse")
|
||||
|
||||
from omni import discover, sources, config
|
||||
config.load()
|
||||
|
||||
C = [] # (name, category, kind, url, adapter, tags)
|
||||
|
||||
|
||||
def s(name, cat, kind, url, ad="rss", tags=()):
|
||||
C.append((name, cat, kind, url, ad, tuple(tags)))
|
||||
|
||||
|
||||
# ---------- security ----------
|
||||
for n, u in [
|
||||
("krebs", "https://krebsonsecurity.com/feed/"),
|
||||
("thn", "https://feeds.feedburner.com/TheHackersNews"),
|
||||
("securityweek", "https://www.securityweek.com/feed/"),
|
||||
("darkreading", "https://www.darkreading.com/rss.xml"),
|
||||
("portswigger_research", "https://portswigger.net/research/rss"),
|
||||
("projectzero", "https://googleprojectzero.blogspot.com/feeds/posts/default"),
|
||||
("talos", "https://blog.talosintelligence.com/rss/"),
|
||||
("sans_isc", "https://isc.sans.edu/rssfeed.xml"),
|
||||
("exploitdb", "https://www.exploit-db.com/rss.xml"),
|
||||
("packetstorm", "https://packetstormsecurity.com/feeds"),
|
||||
("unit42", "https://unit42.paloaltonetworks.com/feed/"),
|
||||
("checkpoint_research", "https://research.checkpoint.com/feed/"),
|
||||
("securelist", "https://securelist.com/feed/"),
|
||||
("welivesecurity", "https://www.welivesecurity.com/en/rss/feed/"),
|
||||
("malwarebytes", "https://www.malwarebytes.com/blog/feed/index.xml"),
|
||||
("sentinelone_labs", "https://www.sentinelone.com/labs/feed/"),
|
||||
("hackread", "https://www.hackread.com/feed/"),
|
||||
("thecyberwire", "https://thecyberwire.com/feeds/rss.xml"),
|
||||
("gbhackers", "https://gbhackers.com/feed/"),
|
||||
("cybersecuritynews", "https://cybersecuritynews.com/feed/"),
|
||||
("securityaffairs", "https://securityaffairs.com/feed"),
|
||||
("therecord", "https://therecord.media/feed"),
|
||||
("riskybusiness", "https://risky.biz/feeds/risky-business.xml"),
|
||||
("trailofbits", "https://blog.trailofbits.com/feed/"),
|
||||
("snyk_blog", "https://snyk.io/blog/feed/"),
|
||||
("wiz_blog", "https://www.wiz.io/feed/rss.xml"),
|
||||
("aquasec", "https://blog.aquasec.com/rss.xml"),
|
||||
("sysdig", "https://sysdig.com/blog/feed/"),
|
||||
("crowdstrike", "https://www.crowdstrike.com/blog/feed/"),
|
||||
("mandiant", "https://www.mandiant.com/resources/blog/rss.xml"),
|
||||
("recordedfuture", "https://www.recordedfuture.com/feed"),
|
||||
("intigriti", "https://blog.intigriti.com/feed/"),
|
||||
("yeswehack", "https://www.yeswehack.com/feed"),
|
||||
("bugcrowd_blog", "https://www.bugcrowd.com/blog/feed/"),
|
||||
("hackerone_blog", "https://www.hackerone.com/blog.rss"),
|
||||
]:
|
||||
s(n, "security", "rss", u, "rss", ("security", "research"))
|
||||
|
||||
# ---------- ai / company ----------
|
||||
for n, u in [
|
||||
("openai_news", "https://openai.com/news/rss.xml"),
|
||||
("deepmind_blog", "https://deepmind.google/blog/rss.xml"),
|
||||
("google_ai_blog", "https://blog.google/technology/ai/rss/"),
|
||||
("hf_blog", "https://huggingface.co/blog/feed.xml"),
|
||||
("the_decoder", "https://the-decoder.com/feed/"),
|
||||
("simonwillison", "https://simonwillison.net/atom/everything/"),
|
||||
("latent_space", "https://www.latent.space/feed"),
|
||||
("importai", "https://importai.substack.com/feed"),
|
||||
("oneusefulthing", "https://www.oneusefulthing.org/feed"),
|
||||
("bair_blog", "https://bair.berkeley.edu/blog/feed.xml"),
|
||||
("lilianweng", "https://lilianweng.github.io/feed.xml"),
|
||||
("jalammar", "https://jalammar.github.io/feed.xml"),
|
||||
("msr_blog", "https://www.microsoft.com/en-us/research/feed/"),
|
||||
("aws_ml_blog", "https://aws.amazon.com/blogs/machine-learning/feed/"),
|
||||
("nvidia_blog", "https://blogs.nvidia.com/feed/"),
|
||||
("meta_ai_blog", "https://ai.meta.com/blog/rss/"),
|
||||
("stability_news", "https://stability.ai/news/rss.xml"),
|
||||
("cohere_blog", "https://cohere.com/blog/rss.xml"),
|
||||
("mistral_news", "https://mistral.ai/news/rss.xml"),
|
||||
("together_blog", "https://www.together.ai/blog/rss.xml"),
|
||||
("eleuther_blog", "https://blog.eleuther.ai/feed.xml"),
|
||||
("karpathy", "https://karpathy.github.io/feed.xml"),
|
||||
("raschka", "https://magazine.sebastianraschka.com/feed"),
|
||||
("interconnects", "https://www.interconnects.ai/feed"),
|
||||
("alignment_forum", "https://www.alignmentforum.org/feed.xml"),
|
||||
("lesswrong", "https://www.lesswrong.com/feed.xml"),
|
||||
]:
|
||||
s(n, "company", "rss", u, "rss", ("ai", "blog"))
|
||||
|
||||
# ---------- papers ----------
|
||||
for n, u in [
|
||||
("arxiv_cs_ai", "https://export.arxiv.org/rss/cs.AI"),
|
||||
("arxiv_cs_cr", "https://export.arxiv.org/rss/cs.CR"),
|
||||
("arxiv_cs_cl", "https://export.arxiv.org/rss/cs.CL"),
|
||||
("arxiv_cs_lg", "https://export.arxiv.org/rss/cs.LG"),
|
||||
]:
|
||||
s(n, "papers", "rss", u, "rss", ("papers", "arxiv"))
|
||||
|
||||
# ---------- community / forum (discourse latest.json) ----------
|
||||
for n, u in [
|
||||
("dc_pytorch", "https://discuss.pytorch.org/latest.json"),
|
||||
("dc_huggingface", "https://discuss.huggingface.co/latest.json"),
|
||||
("dc_elastic", "https://discuss.elastic.co/latest.json"),
|
||||
("dc_gitlab", "https://forum.gitlab.com/latest.json"),
|
||||
("dc_homeassistant", "https://community.home-assistant.io/latest.json"),
|
||||
("dc_manjaro", "https://forum.manjaro.org/latest.json"),
|
||||
("dc_grafana", "https://community.grafana.com/latest.json"),
|
||||
("dc_vue", "https://forum.vuejs.org/latest.json"),
|
||||
("dc_kotlin", "https://discuss.kotlinlang.org/latest.json"),
|
||||
("dc_bitwarden", "https://community.bitwarden.com/latest.json"),
|
||||
("dc_openwrt", "https://forum.openwrt.org/latest.json"),
|
||||
("dc_python", "https://discuss.python.org/latest.json"),
|
||||
("dc_jupyter", "https://discuss.jupyter.org/latest.json"),
|
||||
("dc_fly", "https://community.fly.io/latest.json"),
|
||||
("dc_meta", "https://meta.discourse.org/latest.json"),
|
||||
]:
|
||||
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
|
||||
|
||||
# ---------- reddit ----------
|
||||
for n, sub in [("r_netsec", "netsec"), ("r_bugbounty", "bugbounty"), ("r_llmdevs", "LLMDevs"),
|
||||
("r_cybersecurity", "cybersecurity"), ("r/sideproject", "SideProject"),
|
||||
("r/SaaS", "SaaS"), ("r/selfhosted", "selfhosted")]:
|
||||
s(n, "community", "html", "https://www.reddit.com/r/%s/new/" % sub, "reddit_listing", ("reddit",))
|
||||
|
||||
# ---------- telegram ----------
|
||||
for n, ch in [("tg_seeallochnaya", "seeallochnaya"), ("tg_data_secrets", "data_secrets"),
|
||||
("tg_ai_newz", "ai_newz"), ("tg_addmetoai", "addmetoai")]:
|
||||
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
|
||||
|
||||
# ---------- marketing / traffic forums ----------
|
||||
s("bhw_index", "forum", "rss", "https://www.blackhatworld.com/forums/-/index.rss", "rss", ("traffic", "seo"))
|
||||
s("so_llm", "forum", "rss", "https://stackoverflow.com/feeds/tag?tagnames=llm&sort=newest", "rss", ("qa",))
|
||||
|
||||
# ---------- releases ----------
|
||||
for n, repo in [("gh_rel_aider", "Aider-AI/aider"), ("gh_rel_cline", "cline/cline"),
|
||||
("gh_rel_continue", "continuedev/continue"), ("gh_rel_openhands", "All-Hands-AI/OpenHands"),
|
||||
("gh_rel_unsloth", "unslothai/unsloth"), ("gh_rel_axolotl", "axolotl-ai-cloud/axolotl"),
|
||||
("gh_rel_sglang", "sgl-project/sglang"), ("gh_rel_dspy", "stanfordnlp/dspy"),
|
||||
("gh_rel_crewai", "crewAIInc/crewAI"), ("gh_rel_mem0", "mem0ai/mem0"),
|
||||
("gh_rel_graphrag", "microsoft/graphrag"), ("gh_rel_llamaindex", "run-llama/llama_index"),
|
||||
("gh_rel_semgrep", "semgrep/semgrep"), ("gh_rel_nuclei", "projectdiscovery/nuclei")]:
|
||||
s(n, "release", "atom", "https://github.com/%s/releases.atom" % repo, "rss", ("release", "github"))
|
||||
|
||||
# ---------- cn ----------
|
||||
for n, u in [("qbitai", "https://www.qbitai.com/feed"), ("jiqizhixin", "https://www.jiqizhixin.com/rss"),
|
||||
("infoq_cn", "https://www.infoq.cn/feed"), ("36kr", "https://36kr.com/feed")]:
|
||||
s(n, "company_cn", "rss", u, "rss", ("cn",))
|
||||
|
||||
|
||||
# ---------- волна 5b: инфра/дев-блоги, AI-провайдеры, ещё форумы ----------
|
||||
for n, u in [
|
||||
("cloudflare_blog", "https://blog.cloudflare.com/rss/"),
|
||||
("vercel_blog", "https://vercel.com/blog/rss.xml"),
|
||||
("github_blog", "https://github.blog/feed/"),
|
||||
("gitlab_blog", "https://about.gitlab.com/atom.xml"),
|
||||
("docker_blog", "https://www.docker.com/blog/feed/"),
|
||||
("supabase_blog", "https://supabase.com/blog/rss.xml"),
|
||||
("neon_blog", "https://neon.tech/blog/rss.xml"),
|
||||
("fly_blog", "https://fly.io/blog/feed.xml"),
|
||||
("stripe_blog", "https://stripe.com/blog/feed.rss"),
|
||||
("elastic_blog", "https://www.elastic.co/blog/feed"),
|
||||
("anthropic_news", "https://www.anthropic.com/news/rss.xml"),
|
||||
("openrouter_blog", "https://openrouter.ai/blog/rss.xml"),
|
||||
("groq_blog", "https://groq.com/blog/rss.xml"),
|
||||
("cerebras_blog", "https://www.cerebras.ai/blog/rss.xml"),
|
||||
("ai2_blog", "https://blog.allenai.org/feed"),
|
||||
("databricks_blog", "https://www.databricks.com/feed"),
|
||||
("wandb_blog", "https://wandb.ai/fully-connected/rss.xml"),
|
||||
("modal_blog", "https://modal.com/blog/feed.xml"),
|
||||
("replicate_blog", "https://replicate.com/blog/rss.xml"),
|
||||
("baseten_blog", "https://www.baseten.co/blog/rss.xml"),
|
||||
("scale_blog", "https://scale.com/blog/rss.xml"),
|
||||
("ooba_blog", "https://blog.ollama.com/rss.xml"),
|
||||
("lmarena_blog", "https://lmarena.ai/blog/rss.xml"),
|
||||
("arxiv_cs_se", "https://export.arxiv.org/rss/cs.SE"),
|
||||
("arxiv_cs_ir", "https://export.arxiv.org/rss/cs.IR"),
|
||||
("arxiv_stat_ml", "https://export.arxiv.org/rss/stat.ML"),
|
||||
("thegradient", "https://thegradient.pub/rss/"),
|
||||
("paperswithcode_blog", "https://paperswithcode.com/blog/rss"),
|
||||
("mlsafe", "https://www.mlsafety.org/rss.xml"),
|
||||
]:
|
||||
s(n, "company", "rss", u, "rss", ("infra", "blog"))
|
||||
|
||||
for n, u in [
|
||||
("dc_rust_internals", "https://internals.rust-lang.org/latest.json"),
|
||||
("dc_racket", "https://discourse.racket-lang.org/latest.json"),
|
||||
("dc_nix", "https://discourse.nixos.org/latest.json"),
|
||||
("dc_godot", "https://forum.godotengine.org/latest.json"),
|
||||
("dc_swift", "https://forums.swift.org/latest.json"),
|
||||
("dc_go", "https://forum.golangbridge.org/latest.json"),
|
||||
("dc_raspberry", "https://forums.raspberrypi.com/latest.json"),
|
||||
("dc_tryghost", "https://forum.ghost.org/latest.json"),
|
||||
("dc_blender", "https://devtalk.blender.org/latest.json"),
|
||||
("dc_obsidian", "https://forum.obsidian.md/latest.json"),
|
||||
]:
|
||||
s(n, "forum", "json", u, "discourse", ("forum", "discourse"))
|
||||
|
||||
for n, ch in [("tg_ai_machinelearning", "ai_machinelearning"), ("tg_gpt_main", "gpt_main"),
|
||||
("tg_neuralshit", "neuralshit"), ("tg_llm_underhood", "llm_underhood"),
|
||||
("tg_abuz_ai_soft2", "abuz_ai_soft"), ("tg_cybersecurity_ch", "cybersecurity_channel")]:
|
||||
s(n, "community", "html", "https://t.me/s/%s" % ch, "tg_web", ("telegram",))
|
||||
|
||||
|
||||
def probe_one(row):
|
||||
name, cat, kind, url, ad, tags = row
|
||||
try:
|
||||
ok, n, used = discover.probe(url, cat, limit=6)
|
||||
return (name, cat, kind, url, ad, tags, ok, n, used)
|
||||
except Exception as e:
|
||||
return (name, cat, kind, url, ad, tags, False, 0, str(e)[:40])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
have = {s_.url.rstrip("/") for s_ in sources.all_sources()}
|
||||
cand = [r for r in C if r[3].rstrip("/") not in have]
|
||||
print("кандидатов:", len(C), "| новых к проверке:", len(cand))
|
||||
res = []
|
||||
with cf.ThreadPoolExecutor(16) as ex:
|
||||
for r in ex.map(probe_one, cand):
|
||||
res.append(r)
|
||||
live = [r for r in res if r[6] and r[7] >= 2]
|
||||
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
|
||||
for r in sorted(res, key=lambda x: -x[7])[:12]:
|
||||
print(" %-22s %s %s" % (r[0], r[8], r[7]))
|
||||
print("--- skip")
|
||||
print(", ".join(x[0] for x in res if x not in live))
|
||||
# write module
|
||||
head = ('"""Волна 5 (авто-проверка реальным адаптером, только items >= 2):\n'
|
||||
'security/AI/форумы/reddit/tg/релизы/CN. Дата прогона %s, живых %d."""\n\n'
|
||||
% (__import__("datetime").date.today(), len(live)))
|
||||
body = ["def S(name, category, kind, url, adapter=\"rss\", tags=(), limit=15, detail=False, extra=None):",
|
||||
" return {\"name\": name, \"category\": category, \"kind\": kind, \"url\": url, \"adapter\": adapter,",
|
||||
" \"tags\": tuple(tags), \"limit\": limit, \"detail\": detail, \"extra\": extra or {}}",
|
||||
"", "", "EXTRA5 = ["]
|
||||
for row in live:
|
||||
name, cat, kind, url, ad, tags = row[:6]
|
||||
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15),' % (name, cat, kind, url, ad, tags))
|
||||
body.append("]")
|
||||
open("omni/sources_ext4.py", "w", encoding="utf-8", newline="\n").write(head + "\n".join(body) + "\n")
|
||||
print("written omni/sources_ext4.py")
|
||||
106
tools/wave6.py
Normal file
106
tools/wave6.py
Normal file
@@ -0,0 +1,106 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Волна 6: CVE-фиды, абуз-борды, трафик/SEO, ещё блоги. Только живые (>=2 записи)."""
|
||||
import sys, os, json, datetime, concurrent.futures as cf
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
|
||||
os.chdir(r"C:/Users/User/tmp/omni-parse")
|
||||
from omni import discover, sources, config
|
||||
config.load()
|
||||
|
||||
C = []
|
||||
|
||||
|
||||
def s(name, cat, kind, url, ad="rss", tags=(), extra=None):
|
||||
C.append({"name": name, "category": cat, "kind": kind, "url": url,
|
||||
"adapter": ad, "tags": list(tags), "extra": extra or {}})
|
||||
|
||||
|
||||
# --- CVE / уязвимости ---
|
||||
s("cvefeed_critical", "security", "rss", "https://cvefeed.io/rssfeed/severity/critical", "rss", ("cve", "security"))
|
||||
s("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",))
|
||||
s("cisa_kev_json", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "cisa_kev", ("cve", "kev"))
|
||||
s("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=20", "nvd", ("cve", "nvd"))
|
||||
s("gh_advisories", "security", "atom", "https://github.com/advisories.atom", "rss", ("cve", "github"))
|
||||
s("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"))
|
||||
s("cve_org_blog", "security", "rss", "https://www.cve.org/RSS/News", "rss", ("cve",))
|
||||
s("msrc_blog", "security", "rss", "https://msrc.microsoft.com/blog/feed", "rss", ("cve", "microsoft"))
|
||||
s("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",))
|
||||
s("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"))
|
||||
|
||||
# --- абуз-борды и трафик ---
|
||||
s("ab_seochao_arbitrage", "abuse", "rss", "https://seochao.ru/forums/arbitrazh-trafika.85/index.rss", "rss", ("traffic", "abuse"))
|
||||
s("ab_seochao_accounts", "abuse", "rss", "https://seochao.ru/forums/akkaunty.60/index.rss", "rss", ("accounts", "abuse"))
|
||||
s("ab_blackhatworld", "abuse", "rss", "https://www.blackhatworld.com/forums/misc.18/index.rss", "rss", ("traffic", "abuse"))
|
||||
s("ab_bhw_marketplace", "abuse", "rss", "https://www.blackhatworld.com/forums/marketplace.141/index.rss", "rss", ("market", "abuse"))
|
||||
s("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"))
|
||||
s("ab_zennoposter_blog", "abuse", "rss", "https://zennolab.com/discussion/forums/novosti.30/index.rss", "rss", ("automation", "abuse"))
|
||||
|
||||
# --- AI / инфра ---
|
||||
for n, u in [
|
||||
("swebench_blog", "https://www.swebench.com/blog/rss.xml"),
|
||||
("evalplus", "https://github.com/evalplus/evalplus/releases.atom"),
|
||||
("openai_python_rel", "https://github.com/openai/openai-python/releases.atom"),
|
||||
("vllm_blog", "https://blog.vllm.ai/feed.xml"),
|
||||
("pytorch_blog", "https://pytorch.org/feed.xml"),
|
||||
("tensorflow_blog", "https://blog.tensorflow.org/feeds/posts/default"),
|
||||
("ray_blog", "https://www.anyscale.com/blog/rss.xml"),
|
||||
("modal_changelog", "https://modal.com/blog/rss.xml"),
|
||||
("langchain_blog", "https://blog.langchain.dev/rss/"),
|
||||
("llamaindex_blog", "https://www.llamaindex.ai/blog/rss.xml"),
|
||||
("qdrant_blog", "https://qdrant.tech/blog/rss.xml"),
|
||||
("weaviate_blog", "https://weaviate.io/blog/rss.xml"),
|
||||
("chroma_blog", "https://www.trychroma.com/blog/rss.xml"),
|
||||
("pinecone_blog", "https://www.pinecone.io/blog/rss.xml"),
|
||||
("neo4j_blog", "https://neo4j.com/blog/feed/"),
|
||||
("redis_blog", "https://redis.io/blog/feed/"),
|
||||
("clickhouse_blog", "https://clickhouse.com/blog/rss.xml"),
|
||||
("duckdb_news", "https://duckdb.org/news/feed.xml"),
|
||||
("sqlite_news", "https://sqlite.org/news.rss"),
|
||||
("postgres_news", "https://www.postgresql.org/news.rss"),
|
||||
]:
|
||||
s(n, "company", "rss", u, "rss", ("infra", "blog"))
|
||||
|
||||
# --- форумы/комьюнити ---
|
||||
for n, u in [
|
||||
("dc_duckdb", "https://discuss.duckdb.org/latest.json"),
|
||||
("dc_supabase", "https://github.com/supabase/supabase/discussions.atom"),
|
||||
("dc_cloudflare", "https://community.cloudflare.com/latest.json"),
|
||||
("dc_vercel", "https://github.com/vercel/next.js/discussions.atom"),
|
||||
]:
|
||||
s(n, "forum", "json" if u.endswith(".json") else "atom", u,
|
||||
"discourse" if u.endswith(".json") else "rss", ("forum",))
|
||||
|
||||
|
||||
def probe_one(row):
|
||||
try:
|
||||
ok, n, ad = discover.probe(row["url"], row["category"], limit=6)
|
||||
return row, ok, n, ad
|
||||
except Exception as e:
|
||||
return row, False, 0, str(e)[:40]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
have = {x.url.rstrip("/") for x in sources.all_sources()}
|
||||
cand = [r for r in C if r["url"].rstrip("/") not in have]
|
||||
print("кандидатов:", len(C), "| новых:", len(cand))
|
||||
res = []
|
||||
with cf.ThreadPoolExecutor(14) as ex:
|
||||
for row, ok, n, ad in ex.map(probe_one, cand):
|
||||
res.append((row, ok, n, ad))
|
||||
live = [(r, n, ad) for r, ok, n, ad in res if ok and n >= 2]
|
||||
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
|
||||
for r, n, ad in live:
|
||||
print(" %-24s %-16s %s %s" % (r["name"], ad, r["category"], n))
|
||||
print("--- мёртвые:", ", ".join(r["name"] for r, ok, n, ad in res if not (ok and n >= 2)))
|
||||
head = ('"""Волна 6 (авто-проверка, только items >= 2): CVE, абуз-борды, инфра-блоги, форумы.\n'
|
||||
'Дата %s, живых %d."""\n\n' % (datetime.date.today(), len(live)))
|
||||
body = ['def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):',
|
||||
' return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,',
|
||||
' "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}',
|
||||
"", "", "EXTRA6 = ["]
|
||||
for r, n, ad in live:
|
||||
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15, extra=%r),\n' %
|
||||
(r["name"], r["category"], r["kind"], r["url"], r["adapter"], tuple(r["tags"]), r["extra"]))
|
||||
body.append("]")
|
||||
open("omni/sources_ext5.py", "w", encoding="utf-8", newline="\n").write(head + "".join(body).replace("),\n\n", "),\n") + "\n")
|
||||
print("written omni/sources_ext5.py")
|
||||
Reference in New Issue
Block a user