106 lines
6.2 KiB
Python
106 lines
6.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Волна 6: CVE-фиды, абуз-борды, трафик/SEO, ещё блоги. Только живые (>=2 записи)."""
|
|
import sys, os, json, datetime, concurrent.futures as cf
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
sys.path.insert(0, r"C:/Users/User/tmp/omni-parse")
|
|
os.chdir(r"C:/Users/User/tmp/omni-parse")
|
|
from omni import discover, sources, config
|
|
config.load()
|
|
|
|
C = []
|
|
|
|
|
|
def s(name, cat, kind, url, ad="rss", tags=(), extra=None):
|
|
C.append({"name": name, "category": cat, "kind": kind, "url": url,
|
|
"adapter": ad, "tags": list(tags), "extra": extra or {}})
|
|
|
|
|
|
# --- CVE / уязвимости ---
|
|
s("cvefeed_critical", "security", "rss", "https://cvefeed.io/rssfeed/severity/critical", "rss", ("cve", "security"))
|
|
s("vulners_news", "security", "rss", "https://vulners.com/rss.xml", "rss", ("cve",))
|
|
s("cisa_kev_json", "security", "json", "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "cisa_kev", ("cve", "kev"))
|
|
s("nvd_recent", "security", "json", "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=20", "nvd", ("cve", "nvd"))
|
|
s("gh_advisories", "security", "atom", "https://github.com/advisories.atom", "rss", ("cve", "github"))
|
|
s("zerodayinitiative", "security", "rss", "https://www.zerodayinitiative.com/blog?format=rss", "rss", ("cve", "research"))
|
|
s("cve_org_blog", "security", "rss", "https://www.cve.org/RSS/News", "rss", ("cve",))
|
|
s("msrc_blog", "security", "rss", "https://msrc.microsoft.com/blog/feed", "rss", ("cve", "microsoft"))
|
|
s("chromium_blog", "security", "rss", "https://blog.chromium.org/feeds/posts/default", "rss", ("browser",))
|
|
s("github_security_blog", "security", "rss", "https://github.blog/category/security/feed/", "rss", ("github", "security"))
|
|
|
|
# --- абуз-борды и трафик ---
|
|
s("ab_seochao_arbitrage", "abuse", "rss", "https://seochao.ru/forums/arbitrazh-trafika.85/index.rss", "rss", ("traffic", "abuse"))
|
|
s("ab_seochao_accounts", "abuse", "rss", "https://seochao.ru/forums/akkaunty.60/index.rss", "rss", ("accounts", "abuse"))
|
|
s("ab_blackhatworld", "abuse", "rss", "https://www.blackhatworld.com/forums/misc.18/index.rss", "rss", ("traffic", "abuse"))
|
|
s("ab_bhw_marketplace", "abuse", "rss", "https://www.blackhatworld.com/forums/marketplace.141/index.rss", "rss", ("market", "abuse"))
|
|
s("ab_sape_blog", "abuse", "rss", "https://blog.sape.ru/rss/", "rss", ("seo", "abuse"))
|
|
s("ab_zennoposter_blog", "abuse", "rss", "https://zennolab.com/discussion/forums/novosti.30/index.rss", "rss", ("automation", "abuse"))
|
|
|
|
# --- AI / инфра ---
|
|
for n, u in [
|
|
("swebench_blog", "https://www.swebench.com/blog/rss.xml"),
|
|
("evalplus", "https://github.com/evalplus/evalplus/releases.atom"),
|
|
("openai_python_rel", "https://github.com/openai/openai-python/releases.atom"),
|
|
("vllm_blog", "https://blog.vllm.ai/feed.xml"),
|
|
("pytorch_blog", "https://pytorch.org/feed.xml"),
|
|
("tensorflow_blog", "https://blog.tensorflow.org/feeds/posts/default"),
|
|
("ray_blog", "https://www.anyscale.com/blog/rss.xml"),
|
|
("modal_changelog", "https://modal.com/blog/rss.xml"),
|
|
("langchain_blog", "https://blog.langchain.dev/rss/"),
|
|
("llamaindex_blog", "https://www.llamaindex.ai/blog/rss.xml"),
|
|
("qdrant_blog", "https://qdrant.tech/blog/rss.xml"),
|
|
("weaviate_blog", "https://weaviate.io/blog/rss.xml"),
|
|
("chroma_blog", "https://www.trychroma.com/blog/rss.xml"),
|
|
("pinecone_blog", "https://www.pinecone.io/blog/rss.xml"),
|
|
("neo4j_blog", "https://neo4j.com/blog/feed/"),
|
|
("redis_blog", "https://redis.io/blog/feed/"),
|
|
("clickhouse_blog", "https://clickhouse.com/blog/rss.xml"),
|
|
("duckdb_news", "https://duckdb.org/news/feed.xml"),
|
|
("sqlite_news", "https://sqlite.org/news.rss"),
|
|
("postgres_news", "https://www.postgresql.org/news.rss"),
|
|
]:
|
|
s(n, "company", "rss", u, "rss", ("infra", "blog"))
|
|
|
|
# --- форумы/комьюнити ---
|
|
for n, u in [
|
|
("dc_duckdb", "https://discuss.duckdb.org/latest.json"),
|
|
("dc_supabase", "https://github.com/supabase/supabase/discussions.atom"),
|
|
("dc_cloudflare", "https://community.cloudflare.com/latest.json"),
|
|
("dc_vercel", "https://github.com/vercel/next.js/discussions.atom"),
|
|
]:
|
|
s(n, "forum", "json" if u.endswith(".json") else "atom", u,
|
|
"discourse" if u.endswith(".json") else "rss", ("forum",))
|
|
|
|
|
|
def probe_one(row):
|
|
try:
|
|
ok, n, ad = discover.probe(row["url"], row["category"], limit=6)
|
|
return row, ok, n, ad
|
|
except Exception as e:
|
|
return row, False, 0, str(e)[:40]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
have = {x.url.rstrip("/") for x in sources.all_sources()}
|
|
cand = [r for r in C if r["url"].rstrip("/") not in have]
|
|
print("кандидатов:", len(C), "| новых:", len(cand))
|
|
res = []
|
|
with cf.ThreadPoolExecutor(14) as ex:
|
|
for row, ok, n, ad in ex.map(probe_one, cand):
|
|
res.append((row, ok, n, ad))
|
|
live = [(r, n, ad) for r, ok, n, ad in res if ok and n >= 2]
|
|
print("живых:", len(live), "| мёртвых:", len(res) - len(live))
|
|
for r, n, ad in live:
|
|
print(" %-24s %-16s %s %s" % (r["name"], ad, r["category"], n))
|
|
print("--- мёртвые:", ", ".join(r["name"] for r, ok, n, ad in res if not (ok and n >= 2)))
|
|
head = ('"""Волна 6 (авто-проверка, только items >= 2): CVE, абуз-борды, инфра-блоги, форумы.\n'
|
|
'Дата %s, живых %d."""\n\n' % (datetime.date.today(), len(live)))
|
|
body = ['def S(name, category, kind, url, adapter="rss", tags=(), limit=15, detail=False, extra=None):',
|
|
' return {"name": name, "category": category, "kind": kind, "url": url, "adapter": adapter,',
|
|
' "tags": tuple(tags), "limit": limit, "detail": detail, "extra": extra or {}}',
|
|
"", "", "EXTRA6 = ["]
|
|
for r, n, ad in live:
|
|
body.append(' S("%s", "%s", "%s", "%s", "%s", %r, limit=15, extra=%r),\n' %
|
|
(r["name"], r["category"], r["kind"], r["url"], r["adapter"], tuple(r["tags"]), r["extra"]))
|
|
body.append("]")
|
|
open("omni/sources_ext5.py", "w", encoding="utf-8", newline="\n").write(head + "".join(body).replace("),\n\n", "),\n") + "\n")
|
|
print("written omni/sources_ext5.py") |