durable-workflow 0.4.45__tar.gz → 0.4.47__tar.gz

{durable_workflow-0.4.45/src/durable_workflow.egg-info → durable_workflow-0.4.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 0.4.45
+Version: 0.4.47
 Summary: Python SDK for the Durable Workflow server (language-neutral HTTP protocol)
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-0.4.45 → durable_workflow-0.4.47}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "0.4.45"
+version = "0.4.47"
 description = "Python SDK for the Durable Workflow server (language-neutral HTTP protocol)"
 readme = "README.md"
 requires-python = ">=3.10"

{durable_workflow-0.4.45 → durable_workflow-0.4.47}/src/durable_workflow/worker.py

@@ -45,7 +45,15 @@ from .client import (
     Client,
     WorkflowExecution,
 )
-from .errors import ActivityCancelled, AvroNotInstalledError, NonRetryableError, QueryFailed, ServerError
+from .errors import (
+    ActivityCancelled,
+    AvroNotInstalledError,
+    DurableWorkflowError,
+    InvalidArgument,
+    NonRetryableError,
+    QueryFailed,
+    ServerError,
+)
 from .external_storage import ExternalPayloadCache, ExternalStorageDriver
 from .interceptors import (
     ActivityInterceptorContext,
@@ -74,6 +82,17 @@ _QUERY_TASK_FINAL_REJECTION_REASONS = {
     "query_task_not_leased",
     "query_task_timed_out",
 }
+_WORKFLOW_TASK_COMPLETION_AMBIGUOUS_REJECTION_REASONS = {
+    "lease_expired",
+    "lease_owner_mismatch",
+    "run_already_closed",
+    "run_closed",
+    "task_not_found",
+    "task_not_leased",
+    "workflow_task_attempt_mismatch",
+}
+_WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS = 3
+_WORKFLOW_TASK_COMPLETION_RETRY_DELAYS = (0.05, 0.2)
 _WORKER_WORKFLOW_FINGERPRINTS: dict[tuple[str, str], str] = {}
 
 
@@ -111,6 +130,26 @@ def _is_final_query_task_rejection(error: BaseException) -> bool:
     )
 
 
+def _should_fail_workflow_task_after_completion_error(error: BaseException) -> bool:
+    if isinstance(error, InvalidArgument):
+        return True
+
+    if isinstance(error, ServerError):
+        if error.status >= 500 or error.status == 429:
+            return False
+
+        return error.reason() not in _WORKFLOW_TASK_COMPLETION_AMBIGUOUS_REJECTION_REASONS
+
+    return isinstance(error, DurableWorkflowError)
+
+
+def _should_retry_workflow_task_completion_error(error: BaseException) -> bool:
+    if isinstance(error, ServerError):
+        return error.status >= 500 or error.status == 429
+
+    return not isinstance(error, DurableWorkflowError)
+
+
 def _signal_arguments_envelope_from_export(
     signal: Mapping[str, Any],
     *,
@@ -767,16 +806,16 @@ class Worker:
                 command["type"],
             )
             try:
-                await self.client.complete_workflow_task(
+                await self._complete_workflow_task_with_retry(
                     task_id=task_id,
-                    lease_owner=self.worker_id,
-                    workflow_task_attempt=attempt,
+                    attempt=attempt,
                     commands=[command],
                 )
             except Exception as e:
                 log.warning("failed to complete workflow update task %s: %s", task_id, e)
-                await self._fail_workflow_task_after_completion_error(task_id, attempt, e)
-                return None
+                if _should_fail_workflow_task_after_completion_error(e):
+                    await self._report_workflow_task_after_completion_error(task_id, attempt, e)
+                    return None
             return [command]
 
         try:
@@ -841,19 +880,55 @@ class Worker:
             [c["type"] for c in commands],
         )
         try:
-            await self.client.complete_workflow_task(
+            await self._complete_workflow_task_with_retry(
                 task_id=task_id,
-                lease_owner=self.worker_id,
-                workflow_task_attempt=attempt,
+                attempt=attempt,
                 commands=commands,
             )
         except Exception as e:
             log.warning("failed to complete workflow task %s: %s", task_id, e)
-            await self._fail_workflow_task_after_completion_error(task_id, attempt, e)
-            return None
+            if _should_fail_workflow_task_after_completion_error(e):
+                await self._report_workflow_task_after_completion_error(task_id, attempt, e)
+                return None
         return commands
 
-    async def _fail_workflow_task_after_completion_error(
+    async def _complete_workflow_task_with_retry(
+        self,
+        *,
+        task_id: str,
+        attempt: int,
+        commands: list[dict[str, Any]],
+    ) -> None:
+        for completion_attempt in range(1, _WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS + 1):
+            try:
+                await self.client.complete_workflow_task(
+                    task_id=task_id,
+                    lease_owner=self.worker_id,
+                    workflow_task_attempt=attempt,
+                    commands=commands,
+                )
+                return
+            except Exception as error:
+                if (
+                    completion_attempt >= _WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS
+                    or not _should_retry_workflow_task_completion_error(error)
+                ):
+                    raise
+
+                delay_index = min(
+                    completion_attempt - 1,
+                    len(_WORKFLOW_TASK_COMPLETION_RETRY_DELAYS) - 1,
+                )
+                log.warning(
+                    "retrying workflow task %s completion after attempt %d/%d failed: %s",
+                    task_id,
+                    completion_attempt,
+                    _WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS,
+                    error,
+                )
+                await asyncio.sleep(_WORKFLOW_TASK_COMPLETION_RETRY_DELAYS[delay_index])
+
+    async def _report_workflow_task_after_completion_error(
         self,
         task_id: str,
         attempt: int,

{durable_workflow-0.4.45 → durable_workflow-0.4.47/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 0.4.45
+Version: 0.4.47
 Summary: Python SDK for the Durable Workflow server (language-neutral HTTP protocol)
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-0.4.45 → durable_workflow-0.4.47}/tests/test_worker.py

@@ -22,7 +22,7 @@ from durable_workflow.client import (
     Client,
     WorkflowExecution,
 )
-from durable_workflow.errors import ServerError
+from durable_workflow.errors import InvalidArgument, ServerError, Unauthorized, WorkflowNotFound
 from durable_workflow.interceptors import (
     ActivityHandler,
     ActivityInterceptorContext,
@@ -32,7 +32,7 @@ from durable_workflow.interceptors import (
     WorkflowTaskHandler,
     WorkflowTaskInterceptorContext,
 )
-from durable_workflow.worker import Worker
+from durable_workflow.worker import Worker, _should_fail_workflow_task_after_completion_error
 
 
 @workflow.defn(name="test-wf")
@@ -185,6 +185,28 @@ def compatible_cluster_info(**overrides: object) -> dict[str, object]:
     return info
 
 
+class TestWorkflowTaskCompletionErrorClassification:
+    @pytest.mark.parametrize(
+        ("error", "should_fail"),
+        [
+            (TimeoutError("completion timed out"), False),
+            (RuntimeError("connection reset"), False),
+            (ServerError(409, {"reason": "lease_expired"}), False),
+            (ServerError(409, {"reason": "workflow_task_attempt_mismatch"}), False),
+            (ServerError(429, {"reason": "rate_limited"}), False),
+            (ServerError(503, {"reason": "server_busy"}), False),
+            (ServerError(409, {"reason": "invalid_commands"}), True),
+            (InvalidArgument("invalid command payload"), True),
+            (Unauthorized("missing bearer token"), True),
+            (WorkflowNotFound("wf-missing"), True),
+        ],
+    )
+    def test_classifies_definite_and_ambiguous_completion_errors(
+        self, error: BaseException, should_fail: bool
+    ) -> None:
+        assert _should_fail_workflow_task_after_completion_error(error) is should_fail
+
+
 class TestWorkerRegistration:
     @pytest.mark.asyncio
     async def test_register(self, mock_client: AsyncMock) -> None:
@@ -502,13 +524,97 @@ class TestWorkflowTaskExecution:
         assert serializer.decode(commands[0]["arguments"]["blob"], codec="json") == ["hello"]
 
     @pytest.mark.asyncio
-    async def test_workflow_task_completion_error_fails_task_for_fast_redispatch(
+    async def test_workflow_task_ambiguous_completion_error_preserves_commands(
+        self, mock_client: AsyncMock
+    ) -> None:
+        mock_client.complete_workflow_task.side_effect = ServerError(409, {"reason": "task_not_leased"})
+        worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
+        task = {
+            "task_id": "t-complete-not-leased",
+            "workflow_type": "test-wf",
+            "workflow_task_attempt": 2,
+            "history_events": [],
+            "arguments": '["hello"]',
+            "payload_codec": "json",
+        }
+
+        result = await worker._run_workflow_task(task)
+
+        assert result is not None
+        assert result[0]["type"] == "schedule_activity"
+        mock_client.complete_workflow_task.assert_awaited_once()
+        mock_client.fail_workflow_task.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_workflow_task_retries_transient_completion_error(
+        self, mock_client: AsyncMock
+    ) -> None:
+        mock_client.complete_workflow_task.side_effect = [
+            TimeoutError("completion timed out"),
+            {"outcome": "completed"},
+        ]
+        worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
+        task = {
+            "task_id": "t-complete-retry",
+            "workflow_type": "test-wf",
+            "workflow_task_attempt": 2,
+            "history_events": [],
+            "arguments": '["hello"]',
+            "payload_codec": "json",
+        }
+
+        result = await worker._run_workflow_task(task)
+
+        assert result is not None
+        assert result[0]["type"] == "schedule_activity"
+        assert mock_client.complete_workflow_task.await_count == 2
+        mock_client.fail_workflow_task.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_workflow_task_definite_completion_rejection_fails_task(
         self, mock_client: AsyncMock
     ) -> None:
-        mock_client.complete_workflow_task.side_effect = TimeoutError("completion timed out")
+        mock_client.complete_workflow_task.side_effect = ServerError(409, {"reason": "invalid_commands"})
         worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
         task = {
-            "task_id": "t-complete-timeout",
+            "task_id": "t-complete-invalid",
+            "workflow_type": "test-wf",
+            "workflow_task_attempt": 2,
+            "history_events": [],
+            "arguments": '["hello"]',
+            "payload_codec": "json",
+        }
+
+        result = await worker._run_workflow_task(task)
+
+        assert result is None
+        mock_client.fail_workflow_task.assert_awaited_once()
+        call_kwargs = mock_client.fail_workflow_task.await_args.kwargs
+        assert call_kwargs["task_id"] == "t-complete-invalid"
+        assert call_kwargs["workflow_task_attempt"] == 2
+        assert call_kwargs["lease_owner"] == worker.worker_id
+        assert call_kwargs["failure_type"] == "ServerError"
+        assert "invalid_commands" in call_kwargs["message"]
+
+    @pytest.mark.parametrize(
+        ("completion_error", "failure_type", "message_fragment"),
+        [
+            (Unauthorized("missing bearer token"), "Unauthorized", "missing bearer token"),
+            (WorkflowNotFound("wf-typed-missing"), "WorkflowNotFound", "wf-typed-missing"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_workflow_task_typed_completion_rejection_fails_task(
+        self,
+        mock_client: AsyncMock,
+        completion_error: Exception,
+        failure_type: str,
+        message_fragment: str,
+    ) -> None:
+        mock_client.complete_workflow_task.side_effect = completion_error
+        worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
+        task = {
+            "task_id": "t-complete-typed-rejection",
             "workflow_type": "test-wf",
             "workflow_task_attempt": 2,
             "history_events": [],
@@ -522,11 +628,33 @@ class TestWorkflowTaskExecution:
         mock_client.complete_workflow_task.assert_awaited_once()
         mock_client.fail_workflow_task.assert_awaited_once()
         call_kwargs = mock_client.fail_workflow_task.await_args.kwargs
-        assert call_kwargs["task_id"] == "t-complete-timeout"
+        assert call_kwargs["task_id"] == "t-complete-typed-rejection"
         assert call_kwargs["workflow_task_attempt"] == 2
         assert call_kwargs["lease_owner"] == worker.worker_id
-        assert call_kwargs["failure_type"] == "TimeoutError"
-        assert "completion timed out" in call_kwargs["message"]
+        assert call_kwargs["failure_type"] == failure_type
+        assert message_fragment in call_kwargs["message"]
+
+    @pytest.mark.asyncio
+    async def test_workflow_task_definite_completion_rejection_stays_failed_when_report_fails(
+        self, mock_client: AsyncMock
+    ) -> None:
+        mock_client.complete_workflow_task.side_effect = ServerError(409, {"reason": "invalid_commands"})
+        mock_client.fail_workflow_task.side_effect = RuntimeError("failure report unavailable")
+        worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
+        task = {
+            "task_id": "t-complete-invalid-report-fails",
+            "workflow_type": "test-wf",
+            "workflow_task_attempt": 2,
+            "history_events": [],
+            "arguments": '["hello"]',
+            "payload_codec": "json",
+        }
+
+        result = await worker._run_workflow_task(task)
+
+        assert result is None
+        mock_client.complete_workflow_task.assert_awaited_once()
+        mock_client.fail_workflow_task.assert_awaited_once()
 
     @pytest.mark.asyncio
     async def test_workflow_command_payload_warning_uses_client_policy(
@@ -701,13 +829,13 @@ class TestWorkflowTaskExecution:
         mock_client.fail_workflow_task.assert_not_called()
 
     @pytest.mark.asyncio
-    async def test_update_task_completion_error_fails_task_for_fast_redispatch(
+    async def test_update_task_ambiguous_completion_error_preserves_command(
         self, mock_client: AsyncMock
     ) -> None:
-        mock_client.complete_workflow_task.side_effect = TimeoutError("update completion timed out")
+        mock_client.complete_workflow_task.side_effect = ServerError(409, {"reason": "task_not_leased"})
         worker = Worker(mock_client, task_queue="q1", workflows=[UpdateWorkflow], activities=[])
         task = {
-            "task_id": "t-update-timeout",
+            "task_id": "t-update-not-leased",
             "workflow_type": "update-wf",
             "workflow_task_attempt": 3,
             "workflow_update_id": "upd-worker-1",
@@ -729,14 +857,47 @@ class TestWorkflowTaskExecution:
 
         result = await worker._run_workflow_task(task)
 
-        assert result is None
+        assert result is not None
+        assert result[0]["type"] == "complete_update"
         mock_client.complete_workflow_task.assert_awaited_once()
-        mock_client.fail_workflow_task.assert_awaited_once()
-        call_kwargs = mock_client.fail_workflow_task.await_args.kwargs
-        assert call_kwargs["task_id"] == "t-update-timeout"
-        assert call_kwargs["workflow_task_attempt"] == 3
-        assert call_kwargs["failure_type"] == "TimeoutError"
-        assert "update completion timed out" in call_kwargs["message"]
+        mock_client.fail_workflow_task.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_update_task_retries_transient_completion_error(
+        self, mock_client: AsyncMock
+    ) -> None:
+        mock_client.complete_workflow_task.side_effect = [
+            ServerError(503, {"reason": "server_busy"}),
+            {"outcome": "completed"},
+        ]
+        worker = Worker(mock_client, task_queue="q1", workflows=[UpdateWorkflow], activities=[])
+        task = {
+            "task_id": "t-update-retry",
+            "workflow_type": "update-wf",
+            "workflow_task_attempt": 3,
+            "workflow_update_id": "upd-worker-1",
+            "workflow_wait_kind": "update",
+            "history_events": [
+                {
+                    "event_type": "UpdateAccepted",
+                    "payload": {
+                        "update_id": "upd-worker-1",
+                        "update_name": "increment",
+                        "arguments": serializer.encode([6], codec="json"),
+                        "payload_codec": "json",
+                    },
+                },
+            ],
+            "arguments": "[]",
+            "payload_codec": "json",
+        }
+
+        result = await worker._run_workflow_task(task)
+
+        assert result is not None
+        assert result[0]["type"] == "complete_update"
+        assert mock_client.complete_workflow_task.await_count == 2
+        mock_client.fail_workflow_task.assert_not_called()
 
     @pytest.mark.asyncio
     async def test_query_task_executes_registered_query(self, mock_client: AsyncMock) -> None: