chore(auto): 2026-10-26T22:21:54Z — items=41703 typed=41703 sources=3412 priced=3971 runs=128 tests=False problems=['тесты: OSError', 'пустых источников 2080/3412']

2026-10-27 01:21:57 +03:00
parent 42a864419b
commit 61f15f88f2
2 changed files with 85 additions and 22 deletions
--- a/.local/diag4.py
+++ b/.local/diag4.py
@@ -0,0 +1,63 @@
+import sys, json
+sys.path.insert(0, '.')
+from concurrent.futures import ThreadPoolExecutor
+from omni import feeds
+from omni.sources import Source
+
+C = [
+    ("debian_dsa", "security", "html", "https://www.debian.org/security/", "html_listing",
+     {"include_prefixes": ["/tracker/DSA-"]}),
+    ("debian_dsa2", "security", "html", "https://www.debian.org/security/", "html_listing",
+     {"include_prefixes": ["/debian-security-announce/"]}),
+    ("openwall_a", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
+     {"include_prefixes": ["/lists/oss-security/2026/"]}),
+    ("openwall_b", "security", "html", "https://www.openwall.com/lists/oss-security/", "html_listing",
+     {"include_prefixes": ["/lists/oss-security/"]}),
+    ("t00ls", "forum", "html", "https://www.t00ls.com/", "html_listing",
+     {"include_prefixes": ["/articles-", "/Vuls-"]}),
+    ("anquanke_html", "security", "html", "https://www.anquanke.com/", "html_listing",
+     {"include_prefixes": ["/post/id/"]}),
+    ("secrss", "security", "html", "https://www.secrss.com/", "html_listing",
+     {"include_prefixes": ["/articles/"], "exclude_exact": ["/articles"]}),
+    ("freebuf_root", "security", "html", "https://www.freebuf.com/", "html_listing",
+     {"include_prefixes": ["/articles/", "/news/"]}),
+    ("freebuf_news", "security", "html", "https://www.freebuf.com/news", "html_listing",
+     {"include_prefixes": ["/articles/", "/news/"]}),
+    ("freebuf_rss2", "security", "rss", "https://www.freebuf.com/rss", "rss", {}),
+    ("freebuf_rss3", "security", "rss", "https://www.freebuf.com/feed/", "rss", {}),
+    ("securitylab_a", "security", "rss", "https://www.securitylab.ru/rss/news.xml", "rss", {}),
+    ("securitylab_b", "security", "rss", "https://www.securitylab.ru/rss/", "rss", {}),
+    ("kaspersky_a", "security", "rss", "https://www.kaspersky.com/blog/feed/", "rss", {}),
+    ("anti_malware_a", "security", "rss", "https://www.anti-malware.ru/rss/news", "rss", {}),
+    ("xakep_tech", "security", "rss", "https://xakep.ru/feed/", "rss", {}),
+    ("codeby_x", "forum", "html", "https://codeby.net/forums/", "xenforo_latest", {}),
+    ("seebug_retry", "security", "rss", "https://paper.seebug.org/rss/", "rss", {}),
+    ("ransomlook_retry", "abuse", "json", "https://www.ransomlook.io/api/recent", "generic_json", {}),
+    ("ransomware_live_g", "abuse", "json", "https://api.ransomware.live/groups", "generic_json", {}),
+    ("cybersecnews_retry", "security", "rss", "https://cybersecuritynews.com/feed/", "rss", {}),
+    ("gbhackers_retry", "security", "rss", "https://gbhackers.com/feed/", "rss", {}),
+    ("huntress_retry", "security", "rss", "https://www.huntress.com/blog/rss.xml", "rss", {}),
+    ("scmedia_a", "security", "rss", "https://www.scmagazine.com/rss/news", "rss", {}),
+    ("trendmicro_a", "security", "rss", "https://www.trendmicro.com/en_us/research.rss", "rss", {}),
+    ("dragos_a", "security", "rss", "https://www.dragos.com/blog/feed/", "rss", {}),
+    ("arcticwolf_a", "security", "rss", "https://arcticwolf.com/feed/", "rss", {}),
+    ("zscaler_a", "security", "rss", "https://www.zscaler.com/blogs/rss.xml", "rss", {}),
+    ("arch_a", "security", "rss", "https://security.archlinux.org/advisory/feed.atom", "rss", {}),
+    ("gentoo_a", "security", "rss", "https://www.gentoo.org/glsa/feed.xml", "rss", {}),
+    ("cisa_all_retry", "security", "rss", "https://www.cisa.gov/cybersecurity-advisories/all.xml", "rss", {}),
+]
+
+
+def run(t):
+    name, cat, kind, url, ad, extra = t
+    src = Source(name=name, category=cat, kind=kind, url=url, adapter=ad, tags=(), limit=5, extra=extra)
+    try:
+        items = feeds.fetch(src, 5)
+        return {"name": name, "n": len(items), "first": (items[0].title[:70] if items else "")}
+    except Exception as e:
+        return {"name": name, "n": 0, "err": str(e)[:80]}
+
+
+with ThreadPoolExecutor(6) as ex:
+    for r in ex.map(run, C):
+        print(json.dumps(r, ensure_ascii=False))
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,35 +1,35 @@
 # omni-parse · автоотчёт

 <!-- STATS:START -->
-_обновлено 2026-10-26T19:42:10+00:00_
+_обновлено 2026-10-26T22:21:54+00:00_

 | метрика | значение |
 |---|---|
-| источников | 869 |
+| источников | 3412 |
 | категорий | 11 |
-| записей | 9889 |
-| с ценой | 421 |
-| типизировано | 9889 |
-| прогонов | 120 |
-| пустых источников | 237 |
+| записей | 41703 |
+| с ценой | 3971 |
+| типизировано | 41703 |
+| прогонов | 128 |
+| пустых источников | 2080 |

-типы: новость 6223 · уязвимость 1354 · софт 921 · релиз 563 · цена 437 · модель 399 · вопрос 347 · гайд 301
+типы: новость 24064 · софт 6073 · цена 4234 · уязвимость 2545 · гайд 2182 · промо 2011 · релиз 1784 · модель 1450

 | источник | записей |
 |---|---|
-| arxiv_cs_ai | 280 |
-| tg_vulnwatch | 220 |
-| tg_bleepingcomputer | 220 |
-| tg_ransomware_news | 218 |
-| community | 215 |
-| reddit_machinelearning | 208 |
-| tg_breachradar | 202 |
-| tg_thehackernews | 200 |
-| tg_poc_in_github | 200 |
-| tg_cvefeed | 200 |
-| tg_xakep_ru | 199 |
-| tg_alstack | 198 |
+| rad_midjourney_free_images | 916 |
+| rad_xanllu | 794 |
+| rad_CVEDetector | 738 |
+| rad_caocongbptg | 732 |
+| rad_pixoradigitalnew | 707 |
+| rad_satu_digital | 520 |
+| rad_Coursevania | 520 |
+| rad_Udemyfree4_u | 513 |
+| rad_localllama_ru | 426 |
+| rad_linuxdoit | 397 |
+| rad_AI_News_CN | 384 |
+| rad_st_parser | 345 |

-тесты: 20 passed in 2.30s
-самоаудит: OK
+тесты: OSError
+самоаудит: тесты: OSError; пустых источников 2080/3412
 <!-- STATS:END -->