vellum-ai 0.14.37__py3-none-any.whl → 0.14.38__py3-none-any.whl

vellum/workflows/nodes/displayable/prompt_deployment_node/tests/test_node.py

@@ -5,6 +5,8 @@ from typing import Any, Iterator, List
 
 from httpx import Response
 
+from vellum import RejectedExecutePromptEvent
+from vellum.client import ApiError
 from vellum.client.types.chat_history_input_request import ChatHistoryInputRequest
 from vellum.client.types.chat_message import ChatMessage
 from vellum.client.types.chat_message_request import ChatMessageRequest
@@ -15,6 +17,8 @@ from vellum.client.types.json_input_request import JsonInputRequest
 from vellum.client.types.prompt_output import PromptOutput
 from vellum.client.types.string_vellum_value import StringVellumValue
 from vellum.workflows.context import execution_context
+from vellum.workflows.errors import WorkflowErrorCode
+from vellum.workflows.exceptions import NodeException
 from vellum.workflows.nodes.displayable.prompt_deployment_node.node import PromptDeploymentNode
 
 
@@ -194,3 +198,296 @@ def test_prompt_deployment_node__json_output(vellum_client):
     json_output = outputs[2]
     assert json_output.name == "json"
     assert json_output.value == expected_json
+
+
+def test_prompt_deployment_node__all_fallbacks_fail(vellum_client):
+    # GIVEN a Prompt Deployment Node with fallback models
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {"query": "test query"}
+        ml_model_fallbacks = ["fallback_model_1", "fallback_model_2"]
+
+    # AND all models fail with 404 errors
+    primary_error = ApiError(
+        body={"detail": "Failed to find model 'primary_model'"},
+        status_code=404,
+    )
+    fallback1_error = ApiError(
+        body={"detail": "Failed to find model 'fallback_model_1'"},
+        status_code=404,
+    )
+    fallback2_error = ApiError(
+        body={"detail": "Failed to find model 'fallback_model_2'"},
+        status_code=404,
+    )
+
+    vellum_client.execute_prompt_stream.side_effect = [primary_error, fallback1_error, fallback2_error]
+
+    # WHEN we run the node
+    node = TestPromptDeploymentNode()
+
+    # THEN an exception should be raised
+    with pytest.raises(NodeException) as exc_info:
+        list(node.run())
+
+    # AND the client should have been called three times
+    assert vellum_client.execute_prompt_stream.call_count == 3
+
+    # AND we get the expected error message
+    assert (
+        exc_info.value.message
+        == "Failed to execute prompts with these fallbacks: ['fallback_model_1', 'fallback_model_2']"
+    )
+
+
+def test_prompt_deployment_node__fallback_success(vellum_client):
+    # GIVEN a Prompt Deployment Node with fallback models
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {"query": "test query"}
+        ml_model_fallbacks = ["fallback_model_1", "fallback_model_2"]
+
+    # AND the primary model fails with a 404 error
+    primary_error = ApiError(
+        body={"detail": "Failed to find model 'primary_model'"},
+        status_code=404,
+    )
+
+    # AND the first fallback model succeeds
+    def generate_successful_stream():
+        execution_id = str(uuid4())
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            FulfilledExecutePromptEvent(
+                execution_id=execution_id, outputs=[StringVellumValue(value="Fallback response")]
+            ),
+        ]
+        return iter(events)
+
+    # Set up the mock to fail on primary but succeed on first fallback
+    vellum_client.execute_prompt_stream.side_effect = [primary_error, generate_successful_stream()]
+
+    # WHEN we run the node
+    node = TestPromptDeploymentNode()
+    outputs = list(node.run())
+
+    # THEN the node should complete successfully using the fallback model
+    assert len(outputs) > 0
+    assert outputs[-1].value == "Fallback response"
+
+    # AND the client should have been called twice (once for primary, once for fallback)
+    assert vellum_client.execute_prompt_stream.call_count == 2
+
+    # AND the second call should include the fallback model override
+    second_call_kwargs = vellum_client.execute_prompt_stream.call_args_list[1][1]
+    body_params = second_call_kwargs["request_options"]["additional_body_parameters"]
+    assert body_params["overrides"]["ml_model_fallback"] == "fallback_model_1"
+
+
+def test_prompt_deployment_node__provider_error_with_fallbacks(vellum_client):
+    # GIVEN a Prompt Deployment Node with fallback models
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {}
+        ml_model_fallbacks = ["gpt-4o", "gemini-1.5-flash-latest"]
+
+    # AND the primary model starts but then fails with a provider error
+    def generate_primary_events():
+        execution_id = str(uuid4())
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            RejectedExecutePromptEvent(
+                execution_id=execution_id,
+                error={
+                    "code": "PROVIDER_ERROR",
+                    "message": "The model provider encountered an error",
+                },
+            ),
+        ]
+        return iter(events)
+
+    # AND the fallback model succeeds
+    def generate_fallback_events():
+        execution_id = str(uuid4())
+        expected_outputs: List[PromptOutput] = [StringVellumValue(value="Fallback response")]
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            FulfilledExecutePromptEvent(execution_id=execution_id, outputs=expected_outputs),
+        ]
+        return iter(events)
+
+    vellum_client.execute_prompt_stream.side_effect = [generate_primary_events(), generate_fallback_events()]
+
+    # WHEN we run the node
+    node = TestPromptDeploymentNode()
+    outputs = list(node.run())
+
+    # THEN the node should complete successfully using the fallback model
+    assert len(outputs) > 0
+    assert outputs[-1].value == "Fallback response"
+
+    # AND the client should have been called twice
+    assert vellum_client.execute_prompt_stream.call_count == 2
+
+    # AND the second call should include the fallback model override
+    second_call_kwargs = vellum_client.execute_prompt_stream.call_args_list[1][1]
+    body_params = second_call_kwargs["request_options"]["additional_body_parameters"]
+    assert body_params["overrides"]["ml_model_fallback"] == "gpt-4o"
+
+
+def test_prompt_deployment_node__multiple_fallbacks_mixed_errors(vellum_client):
+    """
+    This test case is when the primary model fails with an api error and
+    the first fallback fails with a provider error
+    """
+
+    # GIVEN a Prompt Deployment Node with multiple fallback models
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {}
+        ml_model_fallbacks = ["gpt-4o", "gemini-1.5-flash-latest"]
+
+    # AND the primary model fails with an API error
+    primary_error = ApiError(
+        body={"detail": "Failed to find model 'primary_model'"},
+        status_code=404,
+    )
+
+    # AND the first fallback model fails with a provider error
+    def generate_fallback1_events():
+        execution_id = str(uuid4())
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            RejectedExecutePromptEvent(
+                execution_id=execution_id,
+                error={
+                    "code": "PROVIDER_ERROR",
+                    "message": "The first fallback provider encountered an error",
+                },
+            ),
+        ]
+        return iter(events)
+
+    # AND the second fallback model succeeds
+    def generate_fallback2_events():
+        execution_id = str(uuid4())
+        expected_outputs: List[PromptOutput] = [StringVellumValue(value="Second fallback response")]
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            FulfilledExecutePromptEvent(execution_id=execution_id, outputs=expected_outputs),
+        ]
+        return iter(events)
+
+    vellum_client.execute_prompt_stream.side_effect = [
+        primary_error,
+        generate_fallback1_events(),
+        generate_fallback2_events(),
+    ]
+
+    # WHEN we run the node
+    node = TestPromptDeploymentNode()
+    outputs = list(node.run())
+
+    # THEN the node should complete successfully using the second fallback model
+    assert len(outputs) > 0
+    assert outputs[-1].value == "Second fallback response"
+
+    # AND the client should have been called three times
+    assert vellum_client.execute_prompt_stream.call_count == 3
+
+    # AND the calls should include the correct model overrides
+    first_fallback_call = vellum_client.execute_prompt_stream.call_args_list[1][1]
+    first_fallback_params = first_fallback_call["request_options"]["additional_body_parameters"]
+    assert first_fallback_params["overrides"]["ml_model_fallback"] == "gpt-4o"
+
+    second_fallback_call = vellum_client.execute_prompt_stream.call_args_list[2][1]
+    second_fallback_params = second_fallback_call["request_options"]["additional_body_parameters"]
+    assert second_fallback_params["overrides"]["ml_model_fallback"] == "gemini-1.5-flash-latest"
+
+
+def test_prompt_deployment_node_multiple_provider_errors(vellum_client):
+    # GIVEN a Prompt Deployment Node with a single fallback model
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {}
+        ml_model_fallbacks = ["gpt-4o"]
+
+    # AND the primary model fails with a provider error
+    def generate_primary_events():
+        execution_id = str(uuid4())
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            RejectedExecutePromptEvent(
+                execution_id=execution_id,
+                error={
+                    "code": "PROVIDER_ERROR",
+                    "message": "The primary provider encountered an error",
+                },
+            ),
+        ]
+        return iter(events)
+
+    # AND the fallback model also fails with a provider error
+    def generate_fallback1_events():
+        execution_id = str(uuid4())
+        events = [
+            InitiatedExecutePromptEvent(execution_id=execution_id),
+            RejectedExecutePromptEvent(
+                execution_id=execution_id,
+                error={
+                    "code": "PROVIDER_ERROR",
+                    "message": "The first fallback provider encountered an error",
+                },
+            ),
+        ]
+        return iter(events)
+
+    vellum_client.execute_prompt_stream.side_effect = [
+        generate_primary_events(),
+        generate_fallback1_events(),
+    ]
+
+    # WHEN we run the node
+    with pytest.raises(NodeException) as exc_info:
+        node = TestPromptDeploymentNode()
+        list(node.run())
+
+    # THEN we should get an exception
+    assert exc_info.value.message == "Failed to execute prompts with these fallbacks: ['gpt-4o']"
+
+    # AND the client should have been called two times
+    assert vellum_client.execute_prompt_stream.call_count == 2
+
+    # AND the calls should include the correct model overrides
+    first_fallback_call = vellum_client.execute_prompt_stream.call_args_list[1][1]
+    first_fallback_params = first_fallback_call["request_options"]["additional_body_parameters"]
+    assert first_fallback_params["overrides"]["ml_model_fallback"] == "gpt-4o"
+
+
+def test_prompt_deployment_node__no_fallbacks(vellum_client):
+    # GIVEN a Prompt Deployment Node with no fallback models
+    class TestPromptDeploymentNode(PromptDeploymentNode):
+        deployment = "test_deployment"
+        prompt_inputs = {}
+
+    # AND the primary model fails with an API error
+    primary_error = ApiError(
+        body={"detail": "Failed to find model 'primary_model'"},
+        status_code=404,
+    )
+
+    vellum_client.execute_prompt_stream.side_effect = primary_error
+
+    # WHEN we run the node
+    node = TestPromptDeploymentNode()
+
+    # THEN the node should raise an exception
+    with pytest.raises(NodeException) as exc_info:
+        list(node.run())
+
+    # AND the exception should contain the original error message
+    assert exc_info.value.message == "Failed to find model 'primary_model'"
+    assert exc_info.value.code == WorkflowErrorCode.INVALID_INPUTS
+
+    # AND the client should have been called only once (for the primary model)
+    assert vellum_client.execute_prompt_stream.call_count == 1

vellum/workflows/runner/runner.py

@@ -146,7 +146,6 @@ class WorkflowRunner(Generic[StateType]):
         self._active_nodes_by_execution_id: Dict[UUID, ActiveNode[StateType]] = {}
         self._cancel_signal = cancel_signal
         self._execution_context = init_execution_context or get_execution_context()
-        self._parent_context = self._execution_context.parent_context
 
         setattr(
             self._initial_state,
@@ -159,13 +158,13 @@ class WorkflowRunner(Generic[StateType]):
     def _snapshot_state(self, state: StateType) -> StateType:
         self._workflow_event_inner_queue.put(
             WorkflowExecutionSnapshottedEvent(
-                trace_id=state.meta.trace_id,
+                trace_id=self._execution_context.trace_id,
                 span_id=state.meta.span_id,
                 body=WorkflowExecutionSnapshottedBody(
                     workflow_definition=self.workflow.__class__,
                     state=state,
                 ),
-                parent=self._parent_context,
+                parent=self._execution_context.parent_context,
             )
         )
         self.workflow._store.append_state_snapshot(state)
@@ -178,16 +177,16 @@ class WorkflowRunner(Generic[StateType]):
         return event
 
     def _run_work_item(self, node: BaseNode[StateType], span_id: UUID) -> None:
-        parent_context = get_parent_context()
+        execution = get_execution_context()
         self._workflow_event_inner_queue.put(
             NodeExecutionInitiatedEvent(
-                trace_id=node.state.meta.trace_id,
+                trace_id=execution.trace_id,
                 span_id=span_id,
                 body=NodeExecutionInitiatedBody(
                     node_definition=node.__class__,
                     inputs=node._inputs,
                 ),
-                parent=parent_context,
+                parent=execution.parent_context,
             )
         )
 
@@ -197,7 +196,7 @@ class WorkflowRunner(Generic[StateType]):
             updated_parent_context = NodeParentContext(
                 span_id=span_id,
                 node_definition=node.__class__,
-                parent=parent_context,
+                parent=execution.parent_context,
             )
             node_run_response: NodeRunResponse
             was_mocked: Optional[bool] = None
@@ -209,7 +208,7 @@ class WorkflowRunner(Generic[StateType]):
                     break
 
             if not was_mocked:
-                with execution_context(parent_context=updated_parent_context, trace_id=node.state.meta.trace_id):
+                with execution_context(parent_context=updated_parent_context, trace_id=execution.trace_id):
                     node_run_response = node.run()
 
             ports = node.Ports()
@@ -232,7 +231,7 @@ class WorkflowRunner(Generic[StateType]):
                 outputs = node.Outputs()
 
                 def initiate_node_streaming_output(output: BaseOutput) -> None:
-                    parent_context = get_parent_context()
+                    execution = get_execution_context()
                     streaming_output_queues[output.name] = Queue()
                     output_descriptor = OutputReference(
                         name=output.name,
@@ -245,18 +244,18 @@ class WorkflowRunner(Generic[StateType]):
                     initiated_ports = initiated_output > ports
                     self._workflow_event_inner_queue.put(
                         NodeExecutionStreamingEvent(
-                            trace_id=node.state.meta.trace_id,
+                            trace_id=execution.trace_id,
                             span_id=span_id,
                             body=NodeExecutionStreamingBody(
                                 node_definition=node.__class__,
                                 output=initiated_output,
                                 invoked_ports=initiated_ports,
                             ),
-                            parent=parent_context,
+                            parent=execution.parent_context,
                         ),
                     )
 
-                with execution_context(parent_context=updated_parent_context, trace_id=node.state.meta.trace_id):
+                with execution_context(parent_context=updated_parent_context, trace_id=execution.trace_id):
                     for output in node_run_response:
                         invoked_ports = output > ports
                         if output.is_initiated:
@@ -268,14 +267,14 @@ class WorkflowRunner(Generic[StateType]):
                             streaming_output_queues[output.name].put(output.delta)
                             self._workflow_event_inner_queue.put(
                                 NodeExecutionStreamingEvent(
-                                    trace_id=node.state.meta.trace_id,
+                                    trace_id=execution.trace_id,
                                     span_id=span_id,
                                     body=NodeExecutionStreamingBody(
                                         node_definition=node.__class__,
                                         output=output,
                                         invoked_ports=invoked_ports,
                                     ),
-                                    parent=parent_context,
+                                    parent=execution.parent_context,
                                 ),
                             )
                         elif output.is_fulfilled:
@@ -285,14 +284,14 @@ class WorkflowRunner(Generic[StateType]):
                             setattr(outputs, output.name, output.value)
                             self._workflow_event_inner_queue.put(
                                 NodeExecutionStreamingEvent(
-                                    trace_id=node.state.meta.trace_id,
+                                    trace_id=execution.trace_id,
                                     span_id=span_id,
                                     body=NodeExecutionStreamingBody(
                                         node_definition=node.__class__,
                                         output=output,
                                         invoked_ports=invoked_ports,
                                     ),
-                                    parent=parent_context,
+                                    parent=execution.parent_context,
                                 )
                             )
 
@@ -309,7 +308,7 @@ class WorkflowRunner(Generic[StateType]):
             invoked_ports = ports(outputs, node.state)
             self._workflow_event_inner_queue.put(
                 NodeExecutionFulfilledEvent(
-                    trace_id=node.state.meta.trace_id,
+                    trace_id=execution.trace_id,
                     span_id=span_id,
                     body=NodeExecutionFulfilledBody(
                         node_definition=node.__class__,
@@ -317,33 +316,33 @@ class WorkflowRunner(Generic[StateType]):
                         invoked_ports=invoked_ports,
                         mocked=was_mocked,
                     ),
-                    parent=parent_context,
+                    parent=execution.parent_context,
                 )
             )
         except NodeException as e:
             logger.info(e)
             self._workflow_event_inner_queue.put(
                 NodeExecutionRejectedEvent(
-                    trace_id=node.state.meta.trace_id,
+                    trace_id=execution.trace_id,
                     span_id=span_id,
                     body=NodeExecutionRejectedBody(
                         node_definition=node.__class__,
                         error=e.error,
                     ),
-                    parent=parent_context,
+                    parent=execution.parent_context,
                 )
             )
         except WorkflowInitializationException as e:
             logger.info(e)
             self._workflow_event_inner_queue.put(
                 NodeExecutionRejectedEvent(
-                    trace_id=node.state.meta.trace_id,
+                    trace_id=execution.trace_id,
                     span_id=span_id,
                     body=NodeExecutionRejectedBody(
                         node_definition=node.__class__,
                         error=e.error,
                     ),
-                    parent=parent_context,
+                    parent=execution.parent_context,
                 )
             )
         except Exception as e:
@@ -351,7 +350,7 @@ class WorkflowRunner(Generic[StateType]):
 
             self._workflow_event_inner_queue.put(
                 NodeExecutionRejectedEvent(
-                    trace_id=node.state.meta.trace_id,
+                    trace_id=execution.trace_id,
                     span_id=span_id,
                     body=NodeExecutionRejectedBody(
                         node_definition=node.__class__,
@@ -360,17 +359,18 @@ class WorkflowRunner(Generic[StateType]):
                             code=WorkflowErrorCode.INTERNAL_ERROR,
                         ),
                     ),
-                    parent=parent_context,
+                    parent=execution.parent_context,
                 ),
             )
 
         logger.debug(f"Finished running node: {node.__class__.__name__}")
 
     def _context_run_work_item(self, node: BaseNode[StateType], span_id: UUID, parent_context=None) -> None:
-        if parent_context is None:
-            parent_context = get_parent_context() or self._parent_context
-
-        with execution_context(parent_context=parent_context, trace_id=node.state.meta.trace_id):
+        execution = get_execution_context()
+        with execution_context(
+            parent_context=parent_context or execution.parent_context,
+            trace_id=execution.trace_id,
+        ):
             self._run_work_item(node, span_id)
 
     def _handle_invoked_ports(self, state: StateType, ports: Optional[Iterable[Port]]) -> None:
@@ -495,66 +495,67 @@ class WorkflowRunner(Generic[StateType]):
 
     def _initiate_workflow_event(self) -> WorkflowExecutionInitiatedEvent:
         return WorkflowExecutionInitiatedEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionInitiatedBody(
                 workflow_definition=self.workflow.__class__,
                 inputs=self._initial_state.meta.workflow_inputs,
             ),
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
         )
 
     def _stream_workflow_event(self, output: BaseOutput) -> WorkflowExecutionStreamingEvent:
         return WorkflowExecutionStreamingEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionStreamingBody(
                 workflow_definition=self.workflow.__class__,
                 output=output,
             ),
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
         )
 
     def _fulfill_workflow_event(self, outputs: OutputsType) -> WorkflowExecutionFulfilledEvent:
         return WorkflowExecutionFulfilledEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionFulfilledBody(
                 workflow_definition=self.workflow.__class__,
                 outputs=outputs,
             ),
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
         )
 
     def _reject_workflow_event(self, error: WorkflowError) -> WorkflowExecutionRejectedEvent:
         return WorkflowExecutionRejectedEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionRejectedBody(
                 workflow_definition=self.workflow.__class__,
                 error=error,
             ),
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
         )
 
     def _resume_workflow_event(self) -> WorkflowExecutionResumedEvent:
         return WorkflowExecutionResumedEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionResumedBody(
                 workflow_definition=self.workflow.__class__,
             ),
+            parent=self._execution_context.parent_context,
         )
 
     def _pause_workflow_event(self, external_inputs: Iterable[ExternalInputReference]) -> WorkflowExecutionPausedEvent:
         return WorkflowExecutionPausedEvent(
-            trace_id=self._initial_state.meta.trace_id,
+            trace_id=self._execution_context.trace_id,
             span_id=self._initial_state.meta.span_id,
             body=WorkflowExecutionPausedBody(
                 workflow_definition=self.workflow.__class__,
                 external_inputs=external_inputs,
             ),
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
         )
 
     def _stream(self) -> None:
@@ -564,13 +565,13 @@ class WorkflowRunner(Generic[StateType]):
         current_parent = WorkflowParentContext(
             span_id=self._initial_state.meta.span_id,
             workflow_definition=self.workflow.__class__,
-            parent=self._parent_context,
+            parent=self._execution_context.parent_context,
             type="WORKFLOW",
         )
         for node_cls in self._entrypoints:
             try:
                 if not self._max_concurrency or len(self._active_nodes_by_execution_id) < self._max_concurrency:
-                    with execution_context(parent_context=current_parent, trace_id=self._initial_state.meta.trace_id):
+                    with execution_context(parent_context=current_parent, trace_id=self._execution_context.trace_id):
                         self._run_node_if_ready(self._initial_state, node_cls)
                 else:
                     self._concurrency_queue.put((self._initial_state, node_cls, None))
@@ -600,7 +601,7 @@ class WorkflowRunner(Generic[StateType]):
 
             self._workflow_event_outer_queue.put(event)
 
-            with execution_context(parent_context=current_parent, trace_id=self._initial_state.meta.trace_id):
+            with execution_context(parent_context=current_parent, trace_id=self._execution_context.trace_id):
                 rejection_error = self._handle_work_item_event(event)
 
             if rejection_error:
@@ -611,7 +612,7 @@ class WorkflowRunner(Generic[StateType]):
             while event := self._workflow_event_inner_queue.get_nowait():
                 self._workflow_event_outer_queue.put(event)
 
-                with execution_context(parent_context=current_parent, trace_id=self._initial_state.meta.trace_id):
+                with execution_context(parent_context=current_parent, trace_id=self._execution_context.trace_id):
                     rejection_error = self._handle_work_item_event(event)
 
                 if rejection_error: