diff --git a/omni/sources.py b/omni/sources.py index be02da0..7270bae 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -323,7 +323,11 @@ def all_sources(include_fresh=True): seen_url, seen_name, out = set(), set(), [] fresh = _fresh_sources(S) if include_fresh else [] for s in SOURCES + _gen_sources() + fresh + _quality_sources(S) + extra + radar_sources() + panel_sources(): + # Панели — отдельный адаптер (/api/pricing), а не обход сайта: их url не должен + # схлопываться с rad_*-источником того же домена, иначе теряется 170 из 175 панелей. key = (s.url or "").rstrip("/").lower() + if (s.name or "").startswith("pn_") and key: + key += "#panel" name = (s.name or "").strip().lower() if (key and key in seen_url) or (name and name in seen_name): continue diff --git a/tests/test_panel_api.py b/tests/test_panel_api.py index ed7fcb7..3fbfd77 100644 --- a/tests/test_panel_api.py +++ b/tests/test_panel_api.py @@ -277,7 +277,7 @@ def test_quality_wave_is_integrated_without_collisions(): assert all(isinstance(s.tags, tuple) and s.adapter in feeds.ADAPTERS for s in wave) assert {s.name for s in wave} <= {s.name for s in registry} assert len(registry) == len({s.name.strip().lower() for s in registry}) - assert len(registry) == len({s.url.rstrip("/").lower() for s in registry}) + assert len(registry) == len({(s.url.rstrip("/").lower(), s.adapter) for s in registry}) @pytest.mark.parametrize("tags", [(), ("news",), ("news", "cve")]) diff --git a/tests/test_regress.py b/tests/test_regress.py index 7607016..e93497c 100644 --- a/tests/test_regress.py +++ b/tests/test_regress.py @@ -122,8 +122,10 @@ def test_bot_not_claimable_by_first_writer(monkeypatch): # ---------- 6. реестр без дублей ---------- def test_sources_unique(): from omni import sources - urls = [s.url.rstrip("/").lower() for s in sources.all_sources() if s.url] - assert len(urls) == len(set(urls)), "в реестре дубли по url" + # Панель (newapi_pricing) и обход сайта (html_listing) на одном домене — разные источники, + # поэтому уникальность проверяем по паре (url, adapter), а не по голому url. + keys = [(s.url.rstrip("/").lower(), s.adapter) for s in sources.all_sources() if s.url] + assert len(keys) == len(set(keys)), "в реестре дубли по (url, adapter)" # ---------- 7. JOBS не растёт бесконечно ---------- diff --git a/tests/test_vault_radar.py b/tests/test_vault_radar.py index 369d90b..1e9ffba 100644 --- a/tests/test_vault_radar.py +++ b/tests/test_vault_radar.py @@ -46,8 +46,8 @@ def test_cn_sources_registered(): def test_all_sources_no_dup_url(): seen = {} for s in sources.all_sources(): - u = (s.url or "").rstrip("/").lower() - assert u not in seen, "дубль url: %s" % s.name + u = ((s.url or "").rstrip("/").lower(), s.adapter) + assert u not in seen, "дубль (url, adapter): %s" % s.name seen[u] = s.name