From eb939c857cb577234ed903b178d583f537320fad Mon Sep 17 00:00:00 2001 From: vitaliy zakharkiv Date: Wed, 18 Nov 2026 08:51:56 +0200 Subject: [PATCH] fix: start and stop MultiBroker result backend once (#726) MultiBroker.with_result_backend() assigned the same backend to every inner broker, so each of them started and stopped it too: N+1 calls for N inner brokers, leaking resources opened in startup() and failing on repeated shutdown(). Tasks are registered only on the MultiBroker, so kicks, results and progress always use its backend and inner brokers never need one. Drop the override and keep the backend on the MultiBroker only. --- docs/available-components/brokers.md | 6 ++-- taskiq/brokers/multi_broker.py | 31 +------------------- tests/brokers/test_multi_broker.py | 44 ++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/docs/available-components/brokers.md b/docs/available-components/brokers.md index 22f9cb4d..ea8a87b3 100644 --- a/docs/available-components/brokers.md +++ b/docs/available-components/brokers.md @@ -219,9 +219,9 @@ but if you customize them, make sure they match on both sides. Also, middlewares of inner brokers work only on the kick side, and middlewares of the `MultiBroker` — only on the execution side. -Use `MultiBroker.with_result_backend()` to set a result backend: -it propagates the backend to all inner brokers, keeping client-side -polling and worker-side saving in sync. +Set the result backend on the `MultiBroker` with `with_result_backend()`: +both kicked tasks and the worker use the backend of the `MultiBroker`, +inner brokers don't need one. ::: If all of your queues live in a single RabbitMQ broker, prefer the native diff --git a/taskiq/brokers/multi_broker.py b/taskiq/brokers/multi_broker.py index 766d6ddb..3357f2f3 100644 --- a/taskiq/brokers/multi_broker.py +++ b/taskiq/brokers/multi_broker.py @@ -1,21 +1,12 @@ import contextlib -import sys from collections.abc import AsyncGenerator from logging import getLogger -from typing import TYPE_CHECKING, Any, NoReturn +from typing import Any, NoReturn from taskiq.abc.broker import AckableMessage, AsyncBroker from taskiq.message import BrokerMessage from taskiq.utils import merge_async_iterables -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -if TYPE_CHECKING: # pragma: no cover - from taskiq.abc.result_backend import AsyncResultBackend - logger = getLogger(__name__) @@ -243,23 +234,3 @@ async def shutdown(self) -> None: exc_info=True, ) await super().shutdown() - - def with_result_backend( - self, - result_backend: "AsyncResultBackend[Any]", - ) -> Self: - """ - Set a result backend for the composite and all inner brokers. - - Inner brokers use their result backend to serve - `wait_result` calls on the client side, while the worker - saves results using the result backend of this broker. - Setting the same backend everywhere keeps them in sync. - - :param result_backend: new result backend. - :return: self - """ - self.result_backend = result_backend - for inner_broker in self.brokers: - inner_broker.result_backend = result_backend - return self diff --git a/tests/brokers/test_multi_broker.py b/tests/brokers/test_multi_broker.py index cc6249c3..f11bb19a 100644 --- a/tests/brokers/test_multi_broker.py +++ b/tests/brokers/test_multi_broker.py @@ -269,7 +269,7 @@ async def composite_task() -> None: ... assert broker.get_all_tasks()[composite_task.task_name] is composite_task -async def test_with_result_backend_propagates_to_inner_brokers() -> None: +async def test_with_result_backend_is_not_shared_with_inner_brokers() -> None: first, second = AsyncQueueBroker(), AsyncQueueBroker() broker = MultiBroker(routing={"first": first, "second": second}) backend = InmemoryResultBackend[None]() @@ -277,8 +277,46 @@ async def test_with_result_backend_propagates_to_inner_brokers() -> None: broker.with_result_backend(backend) assert broker.result_backend is backend - assert first.result_backend is backend - assert second.result_backend is backend + assert first.result_backend is not backend + assert second.result_backend is not backend + + +class LifecycleCountingBackend(InmemoryResultBackend[None]): + """Result backend that counts startup and shutdown calls.""" + + def __init__(self) -> None: + super().__init__() + self.startup_calls = 0 + self.shutdown_calls = 0 + + async def startup(self) -> None: + self.startup_calls += 1 + + async def shutdown(self) -> None: + self.shutdown_calls += 1 + + +async def test_shared_result_backend_started_once() -> None: + backend = LifecycleCountingBackend() + broker = MultiBroker( + routing={"first": AsyncQueueBroker(), "second": AsyncQueueBroker()}, + ).with_result_backend(backend) + + await broker.startup() + + assert backend.startup_calls == 1 + + +async def test_shared_result_backend_shut_down_once() -> None: + backend = LifecycleCountingBackend() + broker = MultiBroker( + routing={"first": AsyncQueueBroker(), "second": AsyncQueueBroker()}, + ).with_result_backend(backend) + await broker.startup() + + await broker.shutdown() + + assert backend.shutdown_calls == 1 async def test_receiver_processes_tasks_from_all_brokers() -> None: