guidellm 0.3.1__py3-none-any.whl → 0.6.0a5__py3-none-any.whl

guidellm/scheduler/worker.py

@@ -1,472 +1,466 @@
-import asyncio
-import math
-import time
-from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator
-from dataclasses import dataclass
-from itertools import islice
-from threading import Event
-from typing import (
-    Any,
-    Generic,
-    Literal,
-    Optional,
-    Union,
-)
+"""
+Worker process implementation for distributed request execution and coordination.
 
-from loguru import logger
-from pydantic import Field
+Manages individual worker processes within the scheduler system, handling request
+lifecycle from queue consumption through backend processing and status publication.
+Workers coordinate with other processes through barriers and events, apply timing
+strategies for request scheduling, maintain concurrency limits, and publish real-time
+status updates throughout request processing.
+"""
 
-from guidellm.backend import (
-    Backend,
-    BackendType,
-    RequestArgs,
-    ResponseSummary,
-    StreamingTextResponse,
-)
-from guidellm.objects import StandardBaseModel
-from guidellm.request import GenerationRequest
-from guidellm.request.types import RequestT, ResponseT
-from guidellm.scheduler.queues import MPQueues, Queue, QueueEmpty
-from guidellm.scheduler.result import (
-    SchedulerRequestInfo,
-    WorkerProcessRequest,
-    WorkerProcessResult,
-)
-from guidellm.scheduler.strategy import SchedulingStrategy
+from __future__ import annotations
 
-__all__ = [
-    "GenerativeRequestsWorker",
-    "GenerativeRequestsWorkerDescription",
-    "RequestsWorker",
-    "ResolveStatus",
-    "WorkerDescription",
-]
+import asyncio
+import time
+import traceback
+from multiprocessing.synchronize import Barrier as ProcessingBarrier
+from multiprocessing.synchronize import Event as ProcessingEvent
+from typing import Annotated, Generic, Literal
 
+try:
+    import uvloop
 
-@dataclass
-class ResolveStatus:
-    requested: bool
-    completed: bool
-    errored: bool
-    canceled: bool
+    HAS_UVLOOP: Annotated[
+        bool, "Flag indicating uvloop availability for event loop optimization"
+    ] = True
+except ImportError:
+    uvloop = None  # type: ignore[assignment] # Optional dependency
 
-    request_start: float
-    request_end: float
+    HAS_UVLOOP = False
 
 
-class WorkerDescription(StandardBaseModel):
-    type_: Literal["worker"] = "worker"
+from guidellm.scheduler.schemas import (
+    BackendInterface,
+    MultiTurnRequestT,
+    RequestT,
+    ResponseT,
+)
+from guidellm.scheduler.strategies import SchedulingStrategy
+from guidellm.schemas import RequestInfo
+from guidellm.utils import (
+    InterProcessMessaging,
+    wait_for_sync_barrier,
+    wait_for_sync_event,
+    wait_for_sync_objects,
+)
 
+__all__ = ["WorkerProcess"]
 
-class RequestsWorker(ABC, Generic[RequestT, ResponseT]):
+
+class WorkerProcess(Generic[RequestT, ResponseT]):
     """
-    An abstract base class for a worker that processes requests.
-    This class defines the interface for a worker that can resolve requests
-    asynchronously or synchronously within the Scheduler class.
-    Subclasses must implement the `resolve` method,
-    which takes a request directly given from the load generator,
-    along with the desired start_time for the request and a timeout_time.
-    The `resolve` method should return the response from the backend.
+    Worker process for distributed request execution in the scheduler system.
+
+    Manages complete request lifecycle including queue consumption, backend processing,
+    timing strategy application, and status publication. Coordinates with other workers
+    through synchronization primitives while maintaining concurrency limits and handling
+    graceful shutdown scenarios including errors and cancellations.
+
+    Example:
+    ::
+        worker = WorkerProcess(
+            worker_index=0,
+            messaging=messaging_interface,
+            backend=backend_instance,
+            strategy=timing_strategy,
+            async_limit=10,
+            fut_scheduling_time_limit=5.0,
+            startup_barrier=barrier,
+            requests_generated_event=generated_event,
+            constraint_reached_event=constraint_event,
+            shutdown_event=shutdown,
+            error_event=error,
+        )
+        worker.run()
     """
 
-    @property
-    @abstractmethod
-    def description(self) -> WorkerDescription:
+    def __init__(
+        self,
+        worker_index: int,
+        messaging: InterProcessMessaging[
+            tuple[
+                ResponseT | None,
+                RequestT | MultiTurnRequestT[RequestT],
+                RequestInfo,
+            ],
+            tuple[
+                RequestT | MultiTurnRequestT[RequestT],
+                RequestInfo,
+            ],
+        ],
+        backend: BackendInterface[RequestT, ResponseT],
+        strategy: SchedulingStrategy,
+        async_limit: int,
+        fut_scheduling_time_limit: float,
+        startup_barrier: ProcessingBarrier,
+        requests_generated_event: ProcessingEvent,
+        constraint_reached_event: ProcessingEvent,
+        shutdown_event: ProcessingEvent,
+        error_event: ProcessingEvent,
+    ):
         """
-        An abstract property that must be implemented by subclasses.
-        This property should return a Serializable class representing the information
-        about the worker instance.
+        Initialize worker process instance.
+
+        :param worker_index: Unique identifier for this worker within the process group
+        :param messaging: Inter-process messaging interface for request coordination
+        :param backend: Backend interface for processing requests
+        :param strategy: Scheduling strategy for determining request timing
+        :param async_limit: Maximum concurrent requests this worker can process
+        :param fut_scheduling_time_limit: Maximum time in seconds to schedule requests
+            into the future
+        :param startup_barrier: Synchronization barrier for coordinated startup
+        :param requests_generated_event: Event signaling request generation completion
+        :param constraint_reached_event: Event signaling processing constraint reached
+        :param shutdown_event: Event signaling graceful shutdown request
+        :param error_event: Event signaling error conditions across processes
         """
-        ...
+        self.worker_index = worker_index
+        self.messaging = messaging
+        self.backend = backend
+        self.strategy = strategy
+        self.async_limit = async_limit
+        self.fut_scheduling_time_limit = fut_scheduling_time_limit
+        self.startup_barrier = startup_barrier
+        self.requests_generated_event = requests_generated_event
+        self.constraint_reached_event = constraint_reached_event
+        self.shutdown_event = shutdown_event
+        self.error_event = error_event
+
+        # Internal states
+        self.startup_completed = False
+        self.backend_started = False
+        self.messaging_started = False
+
+    def run(self):
+        """
+        Main entry point for worker process execution.
+
+        Initializes asyncio event loop with optional uvloop optimization and executes
+        worker async operations. Handles event loop cleanup and error propagation.
 
-    @abstractmethod
-    async def prepare_multiprocessing(self):
+        :raises RuntimeError: If worker encounters unrecoverable error during execution
         """
-        An abstract method that must be implemented by subclasses.
-        This is useful for workers that have instance state that can not
-        be shared across processes and should be cleared out and re-initialized
-        for each new process.
+        try:
+            if HAS_UVLOOP:
+                asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+            asyncio.run(self.run_async())
+        except Exception as err:
+            self.error_event.set()
+            raise RuntimeError(
+                f"Worker process {self.messaging.worker_index} encountered an "
+                f"error: {err}"
+            ) from err
+
+    async def run_async(self):
         """
-        ...
+        Execute main asynchronous worker process logic.
 
-    @abstractmethod
-    async def resolve(
-        self,
-        request: RequestT,
-        timeout_time: float,
-    ) -> tuple[ResolveStatus, ResponseT]:
-        """
-        An abstract method that must be implemented by subclasses.
-        This method should handle the resolution of a request through asyncio,
-        including any necessary backend processing and response handling.
-
-        :param request: The request to be resolved generated by the load generator.
-        :param timeout_time: The timeout time for the request, if there is no timeout
-            given, then this will be math.inf.
-        :return: The response from the worker.
+        Orchestrates concurrent execution of request processing and shutdown monitoring.
+        Handles task cleanup, error propagation, and cancellation coordination when any
+        task completes or encounters an error.
+
+        :raises RuntimeError: If worker tasks encounter unrecoverable errors
+        :raises asyncio.CancelledError: If worker process was cancelled
         """
-        ...
+        stop_task = asyncio.create_task(self._stop_monitor())
+        request_proc_task = asyncio.create_task(self._process_requests())
+        caller_cancelled = False
 
-    async def send_result(
-        self,
-        results_queue: Queue[WorkerProcessResult[RequestT, ResponseT]],
-        result: WorkerProcessResult[RequestT, ResponseT],
-    ):
-        await asyncio.to_thread(results_queue.put, result)  # type: ignore[attr-defined]
+        try:
+            await asyncio.wait(
+                [stop_task, request_proc_task],
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+        except asyncio.CancelledError:
+            caller_cancelled = True
 
-    async def resolve_scheduler_request(
-        self,
-        process_request: WorkerProcessRequest[RequestT, ResponseT],
-        dequeued_time: float,
-        start_time: float,
-        results_queue: Queue[WorkerProcessResult[RequestT, ResponseT]],
-        process_id: int,
-    ):
-        request = process_request.request
-        timeout_time = process_request.timeout_time
-        queued_time = process_request.queued_time
-
-        info = SchedulerRequestInfo(
-            targeted_start_time=start_time,
-            queued_time=queued_time,
-            dequeued_time=dequeued_time,
-            scheduled_time=time.time(),
-            process_id=process_id,
-        )
-        result: WorkerProcessResult[RequestT, ResponseT] = WorkerProcessResult(
-            type_="request_scheduled",
-            request=request,
-            response=None,
-            info=info,
-        )
-        asyncio.create_task(self.send_result(results_queue, result))
+        stop_task.cancel()
+        request_proc_task.cancel()
 
-        if (wait_time := start_time - time.time()) > 0:
-            await asyncio.sleep(wait_time)
+        try:
+            # Ensure all child tasks cancel correctly
+            await asyncio.wait(
+                [stop_task, request_proc_task], return_when=asyncio.ALL_COMPLETED
+            )
+        except asyncio.CancelledError:
+            caller_cancelled = True
+
+        if (
+            task_err := (
+                request_proc_task.exception()
+                if not request_proc_task.cancelled()
+                else stop_task.exception()
+                if not stop_task.cancelled()
+                else None
+            )
+        ) is not None:
+            raise RuntimeError(
+                f"Worker process {self.messaging.worker_index} encountered an "
+                f"error: {task_err}"
+            ) from task_err
 
-        info.worker_start = time.time()
-        result = WorkerProcessResult(
-            type_="request_start",
-            request=request,
-            response=None,
-            info=info,
-        )
-        asyncio.create_task(self.send_result(results_queue, result))
-
-        status, response = await self.resolve(request, timeout_time)
-        info.worker_end = time.time()
-        info.requested = status.requested
-        info.completed = status.completed
-        info.errored = status.errored
-        info.canceled = status.canceled
-        info.request_start = status.request_start
-        info.request_end = status.request_end
-        result = WorkerProcessResult(
-            type_="request_complete",
-            request=request,
-            response=response,
-            info=info,
-        )
-        asyncio.create_task(self.send_result(results_queue, result))
+        if caller_cancelled:
+            raise asyncio.CancelledError("Worker process was cancelled")
 
-    def process_loop_asynchronous(
+    async def _stop_monitor(
         self,
-        queues: MPQueues[RequestT, ResponseT],
-        strategy: SchedulingStrategy,
-        stop_event: Event,
-        max_concurrency: int,
-        process_id: int,
-        num_processes: int,
-    ):
-        async def _process_runner():
-            lock = asyncio.Semaphore(max_concurrency)
-            times_iter = islice(
-                strategy.request_times(),
-                process_id,
-                None,
-                num_processes,
+    ) -> None:
+        """
+        Monitor shutdown and error events for worker termination.
+        :raises RuntimeError if the work process received an error signal.
+        """
+        exit_key = await wait_for_sync_objects(
+            {
+                "error_event": self.error_event,
+                "shutdown_event": self.shutdown_event,
+            },
+            poll_interval=self.messaging.poll_interval,
+        )
+
+        if exit_key == "error_event":
+            raise RuntimeError(
+                f"Worker process {self.messaging.worker_index} received error signal."
             )
 
-            start_time = None
-            while not stop_event.is_set():
-                if start_time is None:
-                    start_time = next(times_iter)
-
-                # Yield control to the event loop. Sleep if we are way ahead
-                await asyncio.sleep(start_time - time.time() - 1)
-                await lock.acquire()
-
-                try:
-                    process_request = queues.requests.get_nowait()
-                    dequeued_time = time.time()
-                except QueueEmpty:
-                    lock.release()
-                    continue
-
-                def _request_callback(
-                    _: asyncio.Future[WorkerProcessRequest[RequestT, ResponseT]],
-                ):
-                    nonlocal lock
-                    lock.release()
-
-                task = asyncio.create_task(
-                    self.resolve_scheduler_request(
-                        process_request=process_request,
-                        dequeued_time=dequeued_time,
-                        start_time=start_time,
-                        results_queue=queues.responses,
-                        process_id=process_id,
-                    )
-                )
-                task.add_done_callback(_request_callback)
-                start_time = None
+    async def _process_requests(self):
+        """
+        Manage request processing lifecycle from startup to shutdown.
 
+        Coordinates startup synchronization, processes requests until constraints are
+        reached, then cancels pending requests until shutdown or error occurs.
+        """
         try:
-            asyncio.run(_process_runner())
-        except Exception as exc:  # noqa: BLE001
-            logger.error(
-                f"Error in worker process {process_id}: {exc}",
-                exc_info=True,
-                stack_info=True,
+            # 1. Start up synchronization (backend, messaging, and other processes)
+            # 2. Messaging startup, receive requests until requests_generated event
+            await self._processing_startup()
+
+            # 3. Run process requests loop until constraint_reached event
+            processing_task = asyncio.create_task(self._process_requests_loop())
+            await wait_for_sync_event(
+                self.constraint_reached_event,
+                poll_interval=self.messaging.poll_interval,
             )
+            processing_task.cancel()
+
+            # 4. Cancel pending requests until proc canceled (manual, shutdown, error)
+            await self._cancel_requests_loop()
+        finally:
+            # 5. On cancel, shut down event, error event, or internal error:
+            #    attempt to shut down this worker cleanly (stop backend and messaging)
+            await self._processing_shutdown()
+
+    async def _processing_startup(self):
+        """Initialize backend, messaging, and synchronize with other workers."""
+        # Get backend ready
+        await self.backend.process_startup()
+        self.backend_started = True
+        await self.backend.validate()
+
+        # Get messaging system ready
+        await self.messaging.start(
+            receive_stop_criteria=[self.requests_generated_event]
+        )
+        self.messaging_started = True
 
+        # Wait for all processes to be ready
+        await wait_for_sync_barrier(
+            self.startup_barrier,
+            poll_interval=self.messaging.poll_interval,
+        )
 
-class GenerativeRequestsWorkerDescription(WorkerDescription):
-    type_: Literal["generative_requests_worker"] = "generative_requests_worker"  # type: ignore[assignment]
-    backend_type: BackendType
-    backend_target: str
-    backend_model: str
-    backend_info: dict[str, Any] = Field(
-        default_factory=dict,
-    )
+        self.startup_completed = True
 
+    async def _processing_shutdown(self):
+        if self.backend_started:
+            await self.backend.process_shutdown()
+            self.backend_started = False
 
-class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary]):
-    """
-    A class that handles the execution of requests using a backend.
-    This class is responsible for sending requests to the backend,
-    handling responses, and managing errors.
+        if self.messaging_started:
+            await self.messaging.stop()
+            self.messaging_started = False
 
-    :param backend: The backend to use for handling requests.
-        This should be an instance of Backend such as an OpenAIHTTPBackend.
-    """
+        self.startup_completed = False
 
-    def __init__(self, backend: Backend):
-        self.backend = backend
-
-    @property
-    def description(self) -> GenerativeRequestsWorkerDescription:
-        """
-        Get the description of the worker.
-        :return: The description of the worker.
+    async def _process_requests_loop(self):
         """
-        return GenerativeRequestsWorkerDescription(
-            backend_type=self.backend.type_,
-            backend_target=self.backend.target,
-            backend_model=self.backend.model or "None",
-            backend_info=self.backend.info,
-        )
+        Process requests continuously until cancelled with concurrency limits.
 
-    async def prepare_multiprocessing(self):
+        Schedules and processes requests according to the timing strategy while
+        maintaining the configured concurrency limit through semaphore coordination.
         """
-        Prepare the worker for multiprocessing.
-        This is useful for workers that have instance state that can not
-        be shared across processes and should be cleared out and re-initialized
-        for each new process.
-        """
-        await self.backend.prepare_multiprocessing()
+        try:
+            # Run request processing
+            async_semaphore = asyncio.Semaphore(self.async_limit)
+            pending_tasks: set[asyncio.Task] = set()
+
+            def _task_done(task):
+                pending_tasks.discard(task)
+                async_semaphore.release()
+
+                if not task.cancelled() and (exception := task.exception()):
+                    raise exception
+
+            # Main loop; loop until canceled
+            while True:
+                await async_semaphore.acquire()
+                request_time = await self.strategy.next_request_time(
+                    worker_index=self.worker_index
+                )
 
-    def process_loop_asynchronous(
-        self,
-        queues: MPQueues[GenerationRequest, ResponseSummary],
-        strategy: SchedulingStrategy,
-        stop_event: Event,
-        max_concurrency: int,
-        process_id: int,
-        num_processes: int,
-    ):
-        asyncio.run(self.backend.validate())
-        super().process_loop_asynchronous(
-            queues=queues,
-            strategy=strategy,
-            stop_event=stop_event,
-            max_concurrency=max_concurrency,
-            process_id=process_id,
-            num_processes=num_processes,
-        )
+                if (
+                    time_until := request_time - time.time()
+                ) >= self.fut_scheduling_time_limit:
+                    await asyncio.sleep(time_until - self.fut_scheduling_time_limit)
 
-    async def resolve(
-        self,
-        request: GenerationRequest,
-        timeout_time: float,
-    ) -> tuple[ResolveStatus, ResponseSummary]:
-        """
-        Resolve a request by sending it to the backend and handling the response.
-        This method sends the request to the backend, waits for a response,
-        and handles any errors that may occur during the process.
-
-        :param request: The request to resolve.
-        :param timeout_time: The time to wait for a response before timing out.
-            If timeout_time is math.inf, the request will not timeout.
-        :return: A ResponseSummary object containing the response from the backend.
-            If an error occurs, the ResponseSummary will contain the error message.
+                request_task = asyncio.create_task(
+                    self._process_next_request(target_start=request_time)
+                )
+                pending_tasks.add(request_task)
+                request_task.add_done_callback(_task_done)
+        except asyncio.CancelledError as err:
+            for task in pending_tasks:
+                task.cancel()
+            await asyncio.gather(*pending_tasks, return_exceptions=True)
+
+            raise err
+
+    async def _cancel_requests_loop(self):
+        """Cancel all remaining queued requests until worker process terminates."""
+        while True:
+            try:
+                request: RequestT | MultiTurnRequestT[RequestT]
+                request_info: RequestInfo
+                request, request_info = await self.messaging.get(
+                    timeout=self.messaging.poll_interval
+                )
+            except asyncio.TimeoutError:
+                continue
+
+            request_info.scheduler_node_id = self.messaging.worker_index or -1
+            request_info.error = "Request was cancelled"
+            request_info.timings.resolve_end = time.time()
+            self._send_update("cancelled", None, request, request_info)
+
+    async def _process_next_request(self, target_start: float):
         """
-        resolve_start_time = time.time()
-        response = None
-        error: Optional[str] = None
-        status = ResolveStatus(
-            requested=False,
-            completed=False,
-            errored=False,
-            canceled=False,
-            request_start=-1,
-            request_end=-1,
-        )
+        Process a single request from queue to completion.
 
-        try:
-            if timeout_time < time.time():
-                raise asyncio.TimeoutError(
-                    "The timeout time has already passed."
-                )  # exit early
-
-            status.requested = True
-            request_func, request_kwargs = self._create_request_func_kwargs(request)
-
-            async def _runner():
-                # wrap function so we can enforce timeout and
-                # still return the latest state from the backend
-                async for resp in request_func(**request_kwargs):  # type: ignore[operator]
-                    nonlocal response
-                    response = resp
-
-            await asyncio.wait_for(
-                _runner(),
-                timeout=timeout_time - time.time() if timeout_time < math.inf else None,
-            )
+        Retrieves request from messaging queue, applies timing strategy, processes
+        through backend, and publishes status updates throughout the lifecycle.
 
-            if not response:
-                raise ValueError(
-                    f"No response received for request: {request} "
-                    f"and backend: {self.backend}"
-                )
-            if not isinstance(response, ResponseSummary):
-                raise ValueError(
-                    f"Received no ResponseSummary for request: {request} "
-                    f"and backend: {self.backend}, received: {response}"
-                )
+        :param target_start: Unix timestamp when request should begin processing
+        """
+        request: RequestT | MultiTurnRequestT[RequestT] | None = None
+        request_info: RequestInfo | None = None
+        response: ResponseT | None = None
 
-            status.completed = True
-        except asyncio.TimeoutError:
-            error = "TimeoutError: The request timed out before completing."
-            status.errored = True
-            status.canceled = True
+        try:
+            # Pull request from the queue, update state, and send "pending" update
+            request, request_info = await self._dequeue_next_request(target_start)
+
+            # Schedule the request and send "in_progress" update
+            await self._schedule_request(request, request_info, target_start)
+
+            async for resp, info in self.backend.resolve(  # type: ignore[attr-defined]
+                request, request_info, None
+            ):
+                response = resp
+                request_info = info
+                if request_info is None:
+                    raise RuntimeError("Received invalid request info from backend")
+
+            # Complete the request
+            request_info.timings.resolve_end = time.time()
+            self._send_update("completed", response, request, request_info)
+
+            response = request = request_info = None
+        except asyncio.CancelledError:
+            # Handle cancellation
+            if request is not None and request_info is not None:
+                request_info.error = "Request was cancelled"
+                request_info.timings.resolve_end = time.time()
+                self._send_update("cancelled", response, request, request_info)
+            raise
         except Exception as exc:  # noqa: BLE001
-            error = str(exc)
-            status.errored = True
-
-        return self._handle_response(
-            status=status,
-            request=request,
-            response=response,
-            error=error,
-            resolve_start_time=resolve_start_time,
-        )
+            if request is not None and request_info is not None:
+                request_info.error = repr(exc)
+                request_info.traceback = traceback.format_exc()
+                request_info.timings.resolve_end = time.time()
+                self._send_update("errored", response, request, request_info)
+        finally:
+            if request_info is not None:
+                self.strategy.request_completed(request_info)
+
+    async def _dequeue_next_request(
+        self, target_start: float
+    ) -> tuple[RequestT, RequestInfo]:
+        request, request_info = await self.messaging.get()
+        dequeued_time = time.time()  # Ensure accurate dequeue timing
+        if request is None or request_info is None:
+            raise RuntimeError("Received invalid request or request info")
+        if isinstance(request, list | tuple):
+            raise NotImplementedError("Multi-turn requests are not yet supported")
+
+        request_info.timings.dequeued = dequeued_time
+        request_info.scheduler_node_id = self.messaging.worker_index or -1
+        request_info.timings.targeted_start = target_start
+        self._send_update("pending", None, request, request_info)
+        return request, request_info
+
+    async def _schedule_request(
+        self, request: RequestT, request_info: RequestInfo, target_start: float
+    ):
+        request_info.timings.scheduled_at = request_info.timings.dequeued
+        if target_start > (current_time := time.time()):
+            await asyncio.sleep(target_start - current_time)
+            # Adapt delay so that scheduled at reflects the sleep time
+            request_info.timings.scheduled_at = target_start
+
+        # Process the request with the backend
+        request_info.timings.resolve_start = time.time()
+        self._send_update("in_progress", None, request, request_info)
 
-    def _create_request_func_kwargs(
+    def _send_update(
         self,
-        request: GenerationRequest,
-    ) -> tuple[
-        AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None],
-        dict[str, Any],
-    ]:
-        request_func: AsyncGenerator[
-            Union[StreamingTextResponse, ResponseSummary], None
-        ]
-        request_kwargs: dict[str, Any]
-
-        if request.request_type == "text_completions":
-            request_func = self.backend.text_completions  # type: ignore[assignment]
-            request_kwargs = {
-                "prompt": request.content,
-                "request_id": request.request_id,
-                "prompt_token_count": request.stats.get("prompt_tokens", None),
-                "output_token_count": request.constraints.get("output_tokens", None),
-                **request.params,
-            }
-        elif request.request_type == "chat_completions":
-            request_func = self.backend.chat_completions  # type: ignore[assignment]
-            request_kwargs = {
-                "content": request.content,
-                "request_id": request.request_id,
-                "prompt_token_count": request.stats.get("prompt_tokens", None),
-                "output_token_count": request.constraints.get("output_tokens", None),
-                **request.params,
-            }
-        else:
-            raise ValueError(
-                f"Invalid request type: {request.request_type} for {request}"
-            )
+        new_status: Literal[
+            "pending", "in_progress", "completed", "errored", "cancelled"
+        ],
+        response: ResponseT | None,
+        request: RequestT | MultiTurnRequestT[RequestT],
+        request_info: RequestInfo,
+    ):
+        """
+        Publish request status update through messaging system.
 
-        return request_func, request_kwargs
+        Updates request status and publishes to messaging queue for coordinator
+        consumption. Prevents duplicate status updates for the same state.
 
-    def _handle_response(
-        self,
-        status: ResolveStatus,
-        request: GenerationRequest,
-        response: Any,
-        error: Optional[str],
-        resolve_start_time: float,
-    ) -> tuple[ResolveStatus, ResponseSummary]:
-        if response is None or not isinstance(
-            response, (ResponseSummary, StreamingTextResponse)
-        ):
-            # nothing received or invalid response, fill in defaults for error
-            if response:
-                error = str(
-                    ValueError(
-                        f"Invalid response: {type(response)} for request: {request}; "
-                    )
-                ) + (error or "")
-
-            response = ResponseSummary(
-                value="",
-                request_args=RequestArgs(
-                    target=self.backend.target,
-                    headers={},
-                    params={},
-                    payload={},
-                ),
-                start_time=resolve_start_time,
-                end_time=status.request_end,
-                first_iter_time=None,
-                last_iter_time=None,
-                request_id=request.request_id,
-                error=error or "Unknown error",
-            )
-        elif isinstance(response, StreamingTextResponse):
-            response = ResponseSummary(
-                value=response.value,
-                request_args=RequestArgs(
-                    target=self.backend.target,
-                    headers={},
-                    params={},
-                    payload={},
-                ),
-                start_time=response.start_time,
-                end_time=time.time(),
-                first_iter_time=response.first_iter_time,
-                last_iter_time=response.time if response.iter_count > 0 else None,
-                request_prompt_tokens=request.stats.get("prompt_tokens", None),
-                request_output_tokens=request.constraints.get("output_tokens", None),
-                response_prompt_tokens=None,
-                response_output_tokens=response.iter_count,
-                request_id=request.request_id,
-                error=error or "Unknown error",
-            )
+        :param new_status: New status for the request
+        :param response: Response object if available, None otherwise
+        :param request: Request object being processed
+        :param request_info: Request metadata and timing information
+        :raises Exception: If messaging system fails to publish the update
+        """
+        prev_status = request_info.status
 
-        response.error = error
-        status.request_start = response.start_time
-        status.request_end = response.end_time
+        if new_status == prev_status:
+            # already sent this update, don't send again
+            return
 
-        return status, response
+        try:
+            request_info.status = new_status
+            request_info = (
+                request_info.model_copy()
+                if new_status not in {"completed", "errored", "cancelled"}
+                else request_info  # last update, don't need to copy
+            )
+            self.messaging.put_sync(
+                (response, request, request_info),
+                timeout=-1,
+            )
+            prev_status = new_status
+        except Exception as exc:
+            # Reset status to last one that succeeded or started function with
+            # Calling logic can retry after handling error, if possible
+            request_info.status = prev_status
+            raise exc