Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions taskiq/funcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ async def gather(
This function returns tuple of results in the same
order they were passed.

Tasks with the same ID in different result backend instances
are awaited independently. Repeated references to the same backend
instance and task ID reuse the fetched result.

:param tasks: tasks to await.
:param timeout: timeout of awaiting, defaults to -1
:param with_logs: whether you want to fetch logs, defaults to False
Expand All @@ -31,21 +35,23 @@ async def gather(
"""
loop = asyncio.get_event_loop()
start_time = time()
ordered_ids = [task.task_id for task in tasks]
task_ids = {task.task_id for task in tasks}
# A task ID identifies a result only within its backend.
ordered_keys = [(id(task.result_backend), task.task_id) for task in tasks]
task_keys = set(ordered_keys)
results = {}

async def check_task(task: AsyncTaskiqTask[Any]) -> None:
nonlocal results
nonlocal task_ids
nonlocal task_keys
task_key = (id(task.result_backend), task.task_id)
if await task.is_ready():
try:
task_ids.remove(task.task_id)
task_keys.remove(task_key)
except LookupError:
return
results[task.task_id] = await task.get_result(with_logs=with_logs)
results[task_key] = await task.get_result(with_logs=with_logs)

while task_ids:
while task_keys:
if 0 < timeout < time() - start_time:
raise TaskiqResultTimeoutError(timeout=timeout)
check_tasks = []
Expand All @@ -54,4 +60,4 @@ async def check_task(task: AsyncTaskiqTask[Any]) -> None:
await asyncio.gather(*check_tasks)
await asyncio.sleep(periodicity)

return tuple(results[task_id] for task_id in ordered_ids)
return tuple(results[task_key] for task_key in ordered_keys)
136 changes: 136 additions & 0 deletions tests/test_gather_backend_isolation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import asyncio
from typing import Any

import pytest

from taskiq import InMemoryBroker
from taskiq.brokers.inmemory_broker import InmemoryResultBackend
from taskiq.exceptions import TaskiqResultTimeoutError
from taskiq.funcs import gather
from taskiq.result import TaskiqResult
from taskiq.task import AsyncTaskiqTask


@pytest.mark.parametrize("reverse", [False, True])
async def test_gather_keeps_results_from_independent_brokers(reverse: bool) -> None:
"""Parallel branches can use the same business ID in separate result stores."""
research_broker = InMemoryBroker(await_inplace=True)
review_broker = InMemoryBroker(await_inplace=True)

@research_broker.task(task_name="research")
async def research() -> str:
return "research result"

@review_broker.task(task_name="review")
async def review() -> str:
return "review result"

await research_broker.startup()
await review_broker.startup()
try:
research_task = await research.kicker().with_task_id("agent-run-42").kiq()
review_task = await review.kicker().with_task_id("agent-run-42").kiq()
tasks = [research_task, review_task]
expected = ["research result", "review result"]
if reverse:
tasks.reverse()
expected.reverse()

results = await gather(*tasks, tasks[0], periodicity=0)

assert [result.return_value for result in results] == [*expected, expected[0]]
finally:
await review_broker.shutdown()
await research_broker.shutdown()


@pytest.mark.parametrize("ready_index", [0, 1])
async def test_gather_waits_for_same_id_in_each_backend(ready_index: int) -> None:
"""One ready branch must not make an unrelated pending branch look complete."""
backends: list[InmemoryResultBackend[str]] = [
InmemoryResultBackend(),
InmemoryResultBackend(),
]
await backends[ready_index].set_result(
"agent-run-42",
TaskiqResult(is_err=False, return_value="ready branch", execution_time=0),
)
tasks = [AsyncTaskiqTask("agent-run-42", backend) for backend in backends]

with pytest.raises(TaskiqResultTimeoutError):
await gather(*tasks, timeout=0.01, periodicity=0)


class ConsumingResultBackend(InmemoryResultBackend[Any]):
"""A result store that removes results when read, like keep_results=False."""

async def get_result(
self,
task_id: str,
with_logs: bool = False,
) -> TaskiqResult[Any]:
return self.results.pop(task_id)


async def test_gather_reads_duplicate_handles_from_one_backend_once() -> None:
"""Equivalent handles keep their positions without consuming a result twice."""
backend = ConsumingResultBackend()
await backend.set_result(
"agent-run-42",
TaskiqResult(is_err=False, return_value="shared result", execution_time=0),
)
first: AsyncTaskiqTask[Any] = AsyncTaskiqTask("agent-run-42", backend)
equivalent: AsyncTaskiqTask[Any] = AsyncTaskiqTask("agent-run-42", backend)

results = await gather(first, equivalent, first, periodicity=0)

assert [result.return_value for result in results] == ["shared result"] * 3
assert not await backend.is_result_ready("agent-run-42")


class ObservedResultBackend(InmemoryResultBackend[str]):
"""Expose the first poll so a worker can publish its result afterwards."""

def __init__(self) -> None:
super().__init__()
self.polled = asyncio.Event()

async def is_result_ready(self, task_id: str) -> bool:
ready = await super().is_result_ready(task_id)
self.polled.set()
return ready


async def test_gather_collects_same_id_after_second_backend_becomes_ready() -> None:
"""A branch completing between polls contributes its own result."""
first: InmemoryResultBackend[str] = InmemoryResultBackend()
second = ObservedResultBackend()
await first.set_result(
"agent-run-42",
TaskiqResult(is_err=False, return_value="first result", execution_time=0),
)

async def publish_second_result() -> None:
await second.polled.wait()
await second.set_result(
"agent-run-42",
TaskiqResult(is_err=False, return_value="second result", execution_time=0),
)

publisher = asyncio.create_task(publish_second_result())
try:
results = await asyncio.wait_for(
gather(
AsyncTaskiqTask("agent-run-42", first),
AsyncTaskiqTask("agent-run-42", second),
periodicity=0,
),
timeout=1,
)
assert [result.return_value for result in results] == [
"first result",
"second result",
]
finally:
publisher.cancel()
await asyncio.gather(publisher, return_exceptions=True)
Loading