Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/available-components/brokers.md
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,9 @@ but if you customize them, make sure they match on both sides.
Also, middlewares of inner brokers work only on the kick side,
and middlewares of the `MultiBroker` — only on the execution side.

Use `MultiBroker.with_result_backend()` to set a result backend:
it propagates the backend to all inner brokers, keeping client-side
polling and worker-side saving in sync.
Set the result backend on the `MultiBroker` with `with_result_backend()`:
both kicked tasks and the worker use the backend of the `MultiBroker`,
inner brokers don't need one.
:::

If all of your queues live in a single RabbitMQ broker, prefer the native
Expand Down
31 changes: 1 addition & 30 deletions taskiq/brokers/multi_broker.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,12 @@
import contextlib
import sys
from collections.abc import AsyncGenerator
from logging import getLogger
from typing import TYPE_CHECKING, Any, NoReturn
from typing import Any, NoReturn

from taskiq.abc.broker import AckableMessage, AsyncBroker
from taskiq.message import BrokerMessage
from taskiq.utils import merge_async_iterables

if sys.version_info >= (3, 11):
from typing import Self
else:
from typing_extensions import Self

if TYPE_CHECKING: # pragma: no cover
from taskiq.abc.result_backend import AsyncResultBackend

logger = getLogger(__name__)


Expand Down Expand Up @@ -243,23 +234,3 @@ async def shutdown(self) -> None:
exc_info=True,
)
await super().shutdown()

def with_result_backend(
self,
result_backend: "AsyncResultBackend[Any]",
) -> Self:
"""
Set a result backend for the composite and all inner brokers.

Inner brokers use their result backend to serve
`wait_result` calls on the client side, while the worker
saves results using the result backend of this broker.
Setting the same backend everywhere keeps them in sync.

:param result_backend: new result backend.
:return: self
"""
self.result_backend = result_backend
for inner_broker in self.brokers:
inner_broker.result_backend = result_backend
return self
44 changes: 41 additions & 3 deletions tests/brokers/test_multi_broker.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,16 +269,54 @@ async def composite_task() -> None: ...
assert broker.get_all_tasks()[composite_task.task_name] is composite_task


async def test_with_result_backend_propagates_to_inner_brokers() -> None:
async def test_with_result_backend_is_not_shared_with_inner_brokers() -> None:
first, second = AsyncQueueBroker(), AsyncQueueBroker()
broker = MultiBroker(routing={"first": first, "second": second})
backend = InmemoryResultBackend[None]()

broker.with_result_backend(backend)

assert broker.result_backend is backend
assert first.result_backend is backend
assert second.result_backend is backend
assert first.result_backend is not backend
assert second.result_backend is not backend


class LifecycleCountingBackend(InmemoryResultBackend[None]):
"""Result backend that counts startup and shutdown calls."""

def __init__(self) -> None:
super().__init__()
self.startup_calls = 0
self.shutdown_calls = 0

async def startup(self) -> None:
self.startup_calls += 1

async def shutdown(self) -> None:
self.shutdown_calls += 1


async def test_shared_result_backend_started_once() -> None:
backend = LifecycleCountingBackend()
broker = MultiBroker(
routing={"first": AsyncQueueBroker(), "second": AsyncQueueBroker()},
).with_result_backend(backend)

await broker.startup()

assert backend.startup_calls == 1


async def test_shared_result_backend_shut_down_once() -> None:
backend = LifecycleCountingBackend()
broker = MultiBroker(
routing={"first": AsyncQueueBroker(), "second": AsyncQueueBroker()},
).with_result_backend(backend)
await broker.startup()

await broker.shutdown()

assert backend.shutdown_calls == 1


async def test_receiver_processes_tasks_from_all_brokers() -> None:
Expand Down
Loading