durable-workflow 2.10.0__tar.gz → 2.10.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. {durable_workflow-2.10.0/src/durable_workflow.egg-info → durable_workflow-2.10.1}/PKG-INFO +1 -1
  2. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/pyproject.toml +3 -3
  3. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_metric_routes.py +1 -0
  4. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/client.py +40 -9
  5. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/errors.py +15 -0
  6. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/worker.py +96 -3
  7. {durable_workflow-2.10.0 → durable_workflow-2.10.1/src/durable_workflow.egg-info}/PKG-INFO +1 -1
  8. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/SOURCES.txt +1 -0
  9. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation_worker.py +8 -3
  10. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_remote_worker.py +19 -5
  11. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_regression_corpus.py +2 -0
  12. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_storage_admission.py +17 -3
  13. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker.py +11 -8
  14. durable_workflow-2.10.1/tests/test_worker_deregistration.py +361 -0
  15. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_observation.py +12 -0
  16. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/LICENSE +0 -0
  17. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/README.md +0 -0
  18. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/setup.cfg +0 -0
  19. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/__init__.py +0 -0
  20. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_activity_lease.py +0 -0
  21. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_activity_process.py +0 -0
  22. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_avro.py +0 -0
  23. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cancellation_scope.py +0 -0
  24. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cancellation_scope_history.py +0 -0
  25. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cooperative_cancellation.py +0 -0
  26. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_prepared_local_activity.py +0 -0
  27. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_query_task_authority.py +0 -0
  28. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_sticky_workflow_cache.py +0 -0
  29. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/activity.py +0 -0
  30. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/auth_composition.py +0 -0
  31. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/cancellation.py +0 -0
  32. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_storage.py +0 -0
  33. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_task_input.py +0 -0
  34. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_task_result.py +0 -0
  35. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/history_bundle_verify.py +0 -0
  36. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/interceptors.py +0 -0
  37. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/invocable.py +0 -0
  38. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/metrics.py +0 -0
  39. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/nexus.py +0 -0
  40. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/py.typed +0 -0
  41. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/python_conformance.py +0 -0
  42. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/replay_conformance.py +0 -0
  43. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/replay_verify.py +0 -0
  44. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/retry_policy.py +0 -0
  45. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/serializer.py +0 -0
  46. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/sync.py +0 -0
  47. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/telemetry.py +0 -0
  48. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/telemetry_carrier.py +0 -0
  49. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/testing.py +0 -0
  50. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/worker_session.py +0 -0
  51. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/workflow.py +0 -0
  52. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/workflow_updates_conformance.py +0 -0
  53. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/dependency_links.txt +0 -0
  54. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/entry_points.txt +0 -0
  55. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/requires.txt +0 -0
  56. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/top_level.txt +0 -0
  57. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_cancellation_policies.py +0 -0
  58. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_context.py +0 -0
  59. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_heartbeat_authority.py +0 -0
  60. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_authority.py +0 -0
  61. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_renewal.py +0 -0
  62. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_worker.py +0 -0
  63. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_process.py +0 -0
  64. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_auth_composition.py +0 -0
  65. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_avro_value_benchmark.py +0 -0
  66. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_avro_value_protocol.py +0 -0
  67. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_context.py +0 -0
  68. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_remaining_time.py +0 -0
  69. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_admission.py +0 -0
  70. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_authoring.py +0 -0
  71. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_boundary.py +0 -0
  72. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_opening.py +0 -0
  73. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_child_workflow_policies.py +0 -0
  74. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_ci_checkout.py +0 -0
  75. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_ci_integration_endpoint.py +0 -0
  76. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client.py +0 -0
  77. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client_telemetry.py +0 -0
  78. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client_timeouts.py +0 -0
  79. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_codec_regression_corpus.py +0 -0
  80. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_descendants.py +0 -0
  81. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_groups.py +0 -0
  82. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_history.py +0 -0
  83. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_replay.py +0 -0
  84. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_condition_replay_drift.py +0 -0
  85. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_control_plane_parity_fixtures.py +0 -0
  86. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation.py +0 -0
  87. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation_client.py +0 -0
  88. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_docstring_cross_references.py +0 -0
  89. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_durable_selection.py +0 -0
  90. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_errors.py +0 -0
  91. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_storage.py +0 -0
  92. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_task_input.py +0 -0
  93. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_task_result.py +0 -0
  94. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_golden_history_replay.py +0 -0
  95. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_bundle_schema2.py +0 -0
  96. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_bundle_verify.py +0 -0
  97. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_event_contract.py +0 -0
  98. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_invocable.py +0 -0
  99. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_message_streams.py +0 -0
  100. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_metric_routes.py +0 -0
  101. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_metrics.py +0 -0
  102. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_durable_wait.py +0 -0
  103. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_namespace.py +0 -0
  104. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_wire.py +0 -0
  105. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_order_processing_example.py +0 -0
  106. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_parallel_completion_order_replay.py +0 -0
  107. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_parallel_saga_authoring.py +0 -0
  108. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_patch_history.py +0 -0
  109. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_pending_operation_replay.py +0 -0
  110. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_poll_failure_backoff.py +0 -0
  111. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_client.py +0 -0
  112. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_groups.py +0 -0
  113. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_policies.py +0 -0
  114. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_worker.py +0 -0
  115. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_public_boundary_scanner.py +0 -0
  116. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_pypi_project_surface.py +0 -0
  117. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_python_conformance.py +0 -0
  118. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_queries.py +0 -0
  119. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_query_task_authority.py +0 -0
  120. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_query_timeout_retry.py +0 -0
  121. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_readme_quickstart.py +0 -0
  122. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_redrive_replay.py +0 -0
  123. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_registration_ambiguity.py +0 -0
  124. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_release_metadata.py +0 -0
  125. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay.py +0 -0
  126. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_clock.py +0 -0
  127. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_conformance.py +0 -0
  128. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_verify.py +0 -0
  129. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_retry_policy.py +0 -0
  130. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_run_cancellation_scope_replay.py +0 -0
  131. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_runtime_external_payload_transport.py +0 -0
  132. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_schedules.py +0 -0
  133. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_scoped_run_cancellation_context.py +0 -0
  134. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sdk_metrics_contract.py +0 -0
  135. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_serializer.py +0 -0
  136. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_signals.py +0 -0
  137. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sleep.py +0 -0
  138. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_standalone_activity_client.py +0 -0
  139. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sticky_execution.py +0 -0
  140. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sync.py +0 -0
  141. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sync_activities.py +0 -0
  142. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_telemetry.py +0 -0
  143. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_telemetry_carrier.py +0 -0
  144. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_failures.py +0 -0
  145. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_harness.py +0 -0
  146. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_nexus.py +0 -0
  147. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_threaded_heartbeat_example.py +0 -0
  148. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_tls_trust.py +0 -0
  149. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_transport_diagnostics.py +0 -0
  150. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_update_signal_condition_replay.py +0 -0
  151. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_updates.py +0 -0
  152. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_wait_condition.py +0 -0
  153. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_session.py +0 -0
  154. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_telemetry_wire.py +0 -0
  155. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_entry_arguments.py +0 -0
  156. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_entrypoint_validation.py +0 -0
  157. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_handler_validation.py +0 -0
  158. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_identity.py +0 -0
  159. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_result_timeout.py +0 -0
  160. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_streams.py +0 -0
  161. {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_updates_conformance.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: durable-workflow
3
- Version: 2.10.0
3
+ Version: 2.10.1
4
4
  Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
5
5
  Author: Durable Workflow Contributors
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "durable-workflow"
7
- version = "2.10.0"
7
+ version = "2.10.1"
8
8
  description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -76,8 +76,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
76
76
  durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
77
77
 
78
78
  [tool.durable-workflow]
79
- product-train = "2.10.0"
80
- registry-version = "2.10.0"
79
+ product-train = "2.10.1"
80
+ registry-version = "2.10.1"
81
81
  supported-server-versions = "2.8.0"
82
82
  worker-protocol-version = "1.20"
83
83
  control-plane-version = "2"
@@ -12,6 +12,7 @@ _ROUTES = (
12
12
  for suffix in ("", "/pass")),
13
13
  "/workers", "/workers/{worker_id}",
14
14
  "/worker/register", "/worker/heartbeat", "/worker/registrations/{worker_id}",
15
+ "/worker/registrations/{worker_id}/deregister",
15
16
  "/worker/sessions", "/worker/sessions/{session_id}", "/worker/sessions/{session_id}/heartbeat",
16
17
  *(f"/worker/{kind}-tasks/poll" for kind in ("workflow", "activity", "query", "update-validation")),
17
18
  *(f"/worker/{kind}-tasks/{{task_id}}/{action}" for kind, actions in (
@@ -1853,6 +1853,7 @@ class Client:
1853
1853
  json: Any = None,
1854
1854
  timeout: float | None = None,
1855
1855
  context: str = "",
1856
+ retry: bool = True,
1856
1857
  ) -> Any:
1857
1858
  if worker and discovery:
1858
1859
  raise ValueError("a request cannot be both worker-plane and cluster discovery")
@@ -1920,7 +1921,7 @@ class Client:
1920
1921
 
1921
1922
  try:
1922
1923
  try:
1923
- resp = await self.retry_policy.execute(_do_request)
1924
+ resp = await self.retry_policy.execute(_do_request) if retry else await _do_request()
1924
1925
  except httpx.HTTPStatusError as exc:
1925
1926
  status_code = str(exc.response.status_code)
1926
1927
  outcome = "http_error"
@@ -5245,39 +5246,69 @@ class Client:
5245
5246
  json={"worker_id": worker_id, "reason": reason},
5246
5247
  ))
5247
5248
 
5248
- async def deregister_worker_registration(self, worker_id: str) -> dict[str, Any]:
5249
+ async def deregister_worker_registration(
5250
+ self, worker_id: str, *, registration_token: str | None = None, timeout: float | None = None,
5251
+ expected_namespace: str | None = None,
5252
+ ) -> dict[str, Any]:
5249
5253
  """Remove this runtime's successful worker-plane registration.
5250
5254
 
5251
5255
  Called by :class:`~durable_workflow.Worker` after its pollers and
5252
5256
  in-flight tasks have drained. This is distinct from the operator-only
5253
5257
  :meth:`deregister_worker` control-plane management operation. Returns
5254
5258
  the worker deregistration envelope, including recovered workflow-task
5255
- count.
5259
+ count. With a registration token, use the runtime's generation-fenced
5260
+ receipt endpoint. Each call makes one attempt: Worker owns safe retries
5261
+ and the remaining shutdown budget. An unfenced DELETE cannot safely be
5262
+ repeated after an uncertain response because the worker ID may be reused.
5263
+ ``expected_namespace`` is a local registration-scope precondition, not
5264
+ a namespace override.
5256
5265
  """
5266
+ if expected_namespace is not None and self.namespace != expected_namespace:
5267
+ raise RuntimeError("worker registration namespace changed; registration removal is unconfirmed")
5268
+ if registration_token is not None and (
5269
+ not isinstance(registration_token, str) or re.fullmatch(r"[a-f0-9]{32}", registration_token) is None
5270
+ ):
5271
+ raise ValueError("registration_token must contain exactly 32 lowercase hexadecimal characters")
5272
+ fenced = registration_token is not None
5257
5273
  try:
5258
5274
  data = await self._request(
5259
- "DELETE",
5260
- f"/worker/registrations/{quote(worker_id, safe='')}",
5275
+ "POST" if fenced else "DELETE",
5276
+ f"/worker/registrations/{quote(worker_id, safe='')}" + ("/deregister" if fenced else ""),
5261
5277
  worker=True,
5262
5278
  context=worker_id,
5279
+ json={"registration_token": registration_token} if fenced else None,
5280
+ timeout=timeout,
5281
+ retry=False,
5263
5282
  )
5264
5283
  except ServerError as error:
5265
- if error.status == 404 and error.reason() == "worker_not_found":
5284
+ if not fenced and error.status == 404 and error.reason() == "worker_not_found":
5266
5285
  return {
5267
5286
  "worker_id": worker_id,
5268
5287
  "outcome": "already_deregistered",
5269
5288
  "recovered_workflow_task_count": 0,
5270
5289
  }
5271
5290
  raise
5272
- if not isinstance(data, dict):
5291
+ if not self._valid_worker_deregistration_response(data, worker_id, registration_token):
5273
5292
  raise ServerError(
5274
5293
  200,
5275
5294
  {
5276
5295
  "reason": "invalid_worker_deregistration_response",
5277
- "message": f"expected JSON object, got {type(data).__name__}",
5296
+ "message": "expected a deregistration receipt for this worker registration",
5278
5297
  },
5279
5298
  )
5280
- return data
5299
+ return cast(dict[str, Any], data)
5300
+
5301
+ @staticmethod
5302
+ def _valid_worker_deregistration_response(data: Any, worker_id: str, token: str | None) -> bool:
5303
+ if not isinstance(data, dict):
5304
+ return False
5305
+ count = data.get("recovered_workflow_task_count")
5306
+ return (
5307
+ data.get("worker_id") == worker_id
5308
+ and data.get("outcome") == "deregistered"
5309
+ and type(count) is int and count >= 0
5310
+ and (token is None or data.get("registration_token") == token)
5311
+ )
5281
5312
 
5282
5313
  async def heartbeat_worker(
5283
5314
  self,
@@ -219,6 +219,21 @@ class ServerError(DurableWorkflowError):
219
219
  return self.body.get("reason")
220
220
  return None
221
221
 
222
+ def worker_deregistration_retry_delay(self, worker_id: str, registration_token: str) -> int | None:
223
+ """Return a delay only for an identity-preserving fenced transient."""
224
+ body = self.body
225
+ if (
226
+ self.status != 503 or not isinstance(body, dict)
227
+ or body.get("reason") not in ("backend_lock_pressure", "backend_unavailable")
228
+ or body.get("operation") != "deregister_worker" or body.get("outcome") != "unknown"
229
+ or body.get("worker_id") != worker_id or body.get("registration_token") != registration_token
230
+ or body.get("retryable") is not True
231
+ or type(body.get("retry_after_seconds")) is not int
232
+ or not 0 < body["retry_after_seconds"] <= 60
233
+ ):
234
+ return None
235
+ return int(body["retry_after_seconds"])
236
+
222
237
  def poll_capacity_backpressure_delay(self, task_kind: str, task_queue: str) -> int | None:
223
238
  """Return the advertised wait only for a complete poll-capacity refusal."""
224
239
  body = self.body
@@ -37,6 +37,8 @@ from functools import wraps
37
37
  from types import FunctionType
38
38
  from typing import Annotated, Any, Concatenate, Literal, ParamSpec, TypeVar, Union, get_args, get_origin, get_type_hints
39
39
 
40
+ import httpx
41
+
40
42
  from . import serializer
41
43
  from ._activity_lease import activity_lease_renewal_supported, current_activity_authority
42
44
  from ._activity_process import CallbackFailure, CallbackInvocation, CallbackProcessLost, SupervisedCallback
@@ -1169,6 +1171,10 @@ class Worker:
1169
1171
  self._in_flight: set[asyncio.Task[Any]] = set()
1170
1172
  self._poller_tasks: set[asyncio.Task[Any]] = set()
1171
1173
  self._registered = False
1174
+ self._deregistration_fenced = False
1175
+ self._registration_token: str | None = None
1176
+ self._registration_worker_id: str | None = None
1177
+ self._registration_namespace: object = None
1172
1178
  self._run_started = False
1173
1179
  self._registration_done = asyncio.Event()
1174
1180
  self._shutdown_task: asyncio.Task[None] | None = None
@@ -1295,6 +1301,18 @@ class Worker:
1295
1301
  _validate_server_compatibility(info)
1296
1302
  protocol = info.get("worker_protocol")
1297
1303
  server_capabilities = protocol.get("server_capabilities") if isinstance(protocol, Mapping) else None
1304
+ deregistration = (
1305
+ server_capabilities.get("worker_deregistration_fencing")
1306
+ if isinstance(server_capabilities, Mapping) else None
1307
+ )
1308
+ self._deregistration_fenced = isinstance(deregistration, Mapping) and deregistration.get("supported") is True
1309
+ if isinstance(deregistration, Mapping) and self._deregistration_fenced and (
1310
+ deregistration.get("schema") != "durable-workflow.v2.worker-deregistration.v1"
1311
+ or deregistration.get("endpoint") != "/worker/registrations/{workerId}/deregister"
1312
+ or type(deregistration.get("receipt_retention_seconds")) is not int
1313
+ or deregistration["receipt_retention_seconds"] <= 0
1314
+ ):
1315
+ raise RuntimeError("Server compatibility error: invalid worker_deregistration_fencing contract")
1298
1316
  sticky_support = (
1299
1317
  server_capabilities.get("sticky_execution") if isinstance(server_capabilities, Mapping) else None
1300
1318
  )
@@ -1449,6 +1467,20 @@ class Worker:
1449
1467
  task_slots=self._current_task_slots(),
1450
1468
  process_metrics=self._current_process_metrics(),
1451
1469
  )
1470
+ if self._deregistration_fenced:
1471
+ token = ack.get("registration_token") if isinstance(ack, dict) else None
1472
+ if (
1473
+ not isinstance(token, str) or len(token) != 32
1474
+ or any(character not in "0123456789abcdef" for character in token)
1475
+ or ack.get("worker_id") != self.worker_id or ack.get("registered") is not True
1476
+ ):
1477
+ raise RuntimeError(
1478
+ "Server returned an invalid registration_token acknowledgement; "
1479
+ "registration may remain active and unfenced cleanup is unsafe"
1480
+ )
1481
+ self._registration_token = token
1482
+ self._registration_worker_id = self.worker_id
1483
+ self._registration_namespace = getattr(self.client, "namespace", None)
1452
1484
  self._registered = True
1453
1485
  # Adapt to the server-advertised cadence when present so a cluster
1454
1486
  # can pin the worker fleet's heartbeat beat without each worker
@@ -4567,8 +4599,9 @@ class Worker:
4567
4599
  async def stop(self) -> None:
4568
4600
  """Stop polling, drain tasks, and remove the worker registration.
4569
4601
 
4570
- Repeated calls share one shutdown result. A failed deregistration is
4571
- therefore propagated to every caller without sending another DELETE.
4602
+ Repeated calls share one shutdown result. Generation-fenced recovery
4603
+ stays within the original shutdown deadline. Unfenced deregistration
4604
+ makes one attempt and does not retry an uncertain outcome.
4572
4605
  """
4573
4606
  self._stop.set()
4574
4607
  if self._shutdown_task is None:
@@ -4664,8 +4697,9 @@ class Worker:
4664
4697
  log.warning("worker session %s close failed: %s", session.options.session_id, error)
4665
4698
 
4666
4699
  if self._registered:
4667
- await self.client.deregister_worker_registration(self.worker_id)
4700
+ await self._deregister_before_deadline(deadline)
4668
4701
  self._registered = False
4702
+ self._registration_token = None
4669
4703
  log.info("worker %s deregistered", self.worker_id)
4670
4704
  self._sticky_cache.clear()
4671
4705
  payload_observers = getattr(self.client, "_workflow_payload_observers", None)
@@ -4676,3 +4710,62 @@ class Worker:
4676
4710
  @staticmethod
4677
4711
  def _remaining_shutdown_time(deadline: float) -> float:
4678
4712
  return max(0.0, deadline - asyncio.get_running_loop().time())
4713
+
4714
+ def _check_deregistration_identity(self) -> None:
4715
+ if self._deregistration_fenced and self._registration_token is None:
4716
+ raise RuntimeError("worker registration token is missing; registration removal is unconfirmed")
4717
+ if self._registration_worker_id is not None and (
4718
+ self.worker_id != self._registration_worker_id
4719
+ or getattr(self.client, "namespace", None) != self._registration_namespace
4720
+ ):
4721
+ raise RuntimeError("worker registration identity changed; registration removal is unconfirmed")
4722
+
4723
+ async def _deregistration_attempt(self, remaining: float) -> tuple[float | None, Exception] | None:
4724
+ kwargs: dict[str, Any] = {}
4725
+ if self._deregistration_fenced:
4726
+ kwargs = {"registration_token": self._registration_token, "timeout": remaining}
4727
+ if isinstance(self._registration_namespace, str):
4728
+ kwargs["expected_namespace"] = self._registration_namespace
4729
+ try:
4730
+ await asyncio.wait_for(
4731
+ self.client.deregister_worker_registration(self.worker_id, **kwargs),
4732
+ timeout=remaining,
4733
+ )
4734
+ except asyncio.TimeoutError as error:
4735
+ raise RuntimeError(
4736
+ "worker shutdown timed out during deregistration; registration removal is unconfirmed"
4737
+ ) from error
4738
+ except ServerError as error:
4739
+ retry_delay = (
4740
+ error.worker_deregistration_retry_delay(self.worker_id, self._registration_token)
4741
+ if self._deregistration_fenced and self._registration_token is not None else None
4742
+ )
4743
+ if retry_delay is None:
4744
+ raise
4745
+ return float(retry_delay), error
4746
+ except (httpx.NetworkError, httpx.TimeoutException, httpx.RemoteProtocolError) as error:
4747
+ if not self._deregistration_fenced:
4748
+ raise
4749
+ return None, error
4750
+ return None
4751
+
4752
+ async def _deregister_before_deadline(self, deadline: float) -> None:
4753
+ attempt = 0
4754
+ while True:
4755
+ self._check_deregistration_identity()
4756
+ remaining = self._remaining_shutdown_time(deadline)
4757
+ if remaining <= 0:
4758
+ raise RuntimeError("worker shutdown deadline exhausted; registration removal is unconfirmed")
4759
+ retry = await self._deregistration_attempt(remaining)
4760
+ if retry is None:
4761
+ return
4762
+ retry_delay, failure = retry
4763
+ delay = min(1.0, 0.1 * 2 ** min(attempt, 4)) if retry_delay is None else retry_delay
4764
+ if delay >= self._remaining_shutdown_time(deadline):
4765
+ raise RuntimeError(
4766
+ "worker shutdown cannot retry within its deadline; registration removal is unconfirmed"
4767
+ ) from failure
4768
+ if attempt == 0:
4769
+ log.warning("worker deregistration is unconfirmed; retrying the fenced request within its deadline")
4770
+ attempt += 1
4771
+ await asyncio.sleep(delay)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: durable-workflow
3
- Version: 2.10.0
3
+ Version: 2.10.1
4
4
  Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
5
5
  Author: Durable Workflow Contributors
6
6
  License-Expression: MIT
@@ -146,6 +146,7 @@ tests/test_update_signal_condition_replay.py
146
146
  tests/test_updates.py
147
147
  tests/test_wait_condition.py
148
148
  tests/test_worker.py
149
+ tests/test_worker_deregistration.py
149
150
  tests/test_worker_observation.py
150
151
  tests/test_worker_session.py
151
152
  tests/test_worker_telemetry_wire.py
@@ -607,8 +607,12 @@ async def test_shutdown_timeout_abandons_cleanup_and_replacement_replays_origina
607
607
  await asyncio.wait_for(entered.wait(), timeout=1)
608
608
  original_history = deepcopy(server.history)
609
609
  heartbeats_before_stop = server.client.heartbeat_workflow_task.await_count
610
- await asyncio.wait_for(worker.stop(), timeout=1)
611
- release_thread.set()
610
+ stopping = worker.stop()
611
+ try:
612
+ with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
613
+ await asyncio.wait_for(stopping, timeout=1)
614
+ finally:
615
+ release_thread.set()
612
616
  await asyncio.wait_for(discarded.wait(), timeout=1)
613
617
  assert fenced_heartbeats == [True]
614
618
  assert server.client.heartbeat_workflow_task.await_count == heartbeats_before_stop
@@ -616,7 +620,8 @@ async def test_shutdown_timeout_abandons_cleanup_and_replacement_replays_origina
616
620
  assert server.history == original_history
617
621
  server.client.complete_workflow_task.assert_not_awaited()
618
622
  server.client.fail_workflow_task.assert_not_awaited()
619
- server.client.deregister_worker_registration.assert_awaited_once_with("cooperative-worker")
623
+ assert worker._registered
624
+ server.client.deregister_worker_registration.assert_not_awaited()
620
625
 
621
626
  replacement = ClaimServer(history=original_history, lease_owner="replacement-worker", workflow_task_attempt=5)
622
627
  cleanup_ids: list[str] = []
@@ -92,8 +92,7 @@ async def application_failure(kind: str) -> None:
92
92
  raise ActivityCancelled()
93
93
 
94
94
 
95
- @pytest_asyncio.fixture
96
- async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def]
95
+ async def make_owner(monkeypatch: pytest.MonkeyPatch) -> tuple[Worker, AsyncMock]:
97
96
  monkeypatch.delenv("DURABLE_WORKFLOW_WORKER_PROTOCOL_VERSION", raising=False)
98
97
  client = AsyncMock(spec=Client)
99
98
  client.get_cluster_info.return_value = compatible_cluster_info(worker_protocol={
@@ -106,6 +105,12 @@ async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def
106
105
  worker = Worker(client, task_queue="queue", worker_id="owner", capabilities=["cooperative_cancellation"],
107
106
  max_concurrent_activity_tasks=1)
108
107
  await worker._register()
108
+ return worker, client
109
+
110
+
111
+ @pytest_asyncio.fixture
112
+ async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def]
113
+ worker, client = await make_owner(monkeypatch)
109
114
  try:
110
115
  yield worker, client
111
116
  finally:
@@ -152,18 +157,27 @@ async def test_authority_loss_stops_and_joins_the_actual_callback_before_reporti
152
157
  client.fail_activity_task.assert_not_awaited()
153
158
 
154
159
 
155
- async def test_shutdown_timeout_joins_the_remote_callback_before_returning(owner: Any, tmp_path: Path) -> None:
156
- worker, client = owner
160
+ async def test_shutdown_timeout_joins_the_remote_callback_before_returning(
161
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
162
+ ) -> None:
163
+ worker, client = await make_owner(monkeypatch)
157
164
  worker._shutdown_timeout = 0.01
158
165
  marker = tmp_path / "callback"
159
166
  worker.activities["remote"] = blocked_async
160
167
  execution = worker._track(worker._run_activity_task(task(str(marker))))
161
168
  await wait_for_file(marker)
162
169
  pid = int(marker.read_text())
163
- await asyncio.wait_for(worker.stop(), timeout=4.0)
170
+ stopping = worker.stop()
171
+ with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed") as failed:
172
+ await asyncio.wait_for(stopping, timeout=4.0)
164
173
  await wait_for_exit(pid)
165
174
  assert execution.cancelled() or execution.result() == "claim_aborted"
166
175
  assert not worker._remote_activity_processes
176
+ assert worker._registered
177
+ client.deregister_worker_registration.assert_not_awaited()
178
+ with pytest.raises(RuntimeError) as repeated:
179
+ await worker.stop()
180
+ assert repeated.value is failed.value
167
181
  client.complete_activity_task.assert_not_awaited()
168
182
  client.fail_activity_task.assert_not_awaited()
169
183
 
@@ -25,6 +25,7 @@ from durable_workflow.workflow import (
25
25
  commands_to_server_commands,
26
26
  query_state,
27
27
  )
28
+ from tests.deregistration_recovery import RecoveredWorkflow as FencedRecoveryWorkflow
28
29
  from tests.test_condition_replay_drift import ConditionIdentityWorkflow
29
30
  from tests.test_golden_history_replay import (
30
31
  GoldenSagaCompensationWorkflow,
@@ -371,6 +372,7 @@ WORKFLOWS = [
371
372
  RepeatedPatchWorkflow,
372
373
  PendingOperation,
373
374
  RecoveredWorkflow,
375
+ FencedRecoveryWorkflow,
374
376
  PagedRecordedSideEffectsWorkflow,
375
377
  PostConditionReceiversWorkflow,
376
378
  RecordedSideEffectWorkflow,
@@ -8,6 +8,7 @@ from contextlib import contextmanager
8
8
  from pathlib import Path
9
9
  from types import SimpleNamespace
10
10
  from typing import Any
11
+ from urllib.parse import unquote
11
12
 
12
13
  import httpx
13
14
  import pytest
@@ -544,7 +545,12 @@ async def test_running_worker_preserves_activity_outcome_without_reexecution(
544
545
  info = compatible_cluster_info()
545
546
  info["worker_protocol"]["server_capabilities"]["query_tasks"] = False
546
547
  return httpx.Response(200, json=info)
547
- if path.endswith("/register") or request.method == "DELETE":
548
+ if request.method == "DELETE":
549
+ return httpx.Response(200, json={
550
+ "worker_id": unquote(path.rsplit("/", 1)[-1]),
551
+ "outcome": "deregistered", "recovered_workflow_task_count": 0,
552
+ })
553
+ if path.endswith("/register"):
548
554
  return httpx.Response(200, json={"registered": True})
549
555
  if path.endswith("/activity-tasks/poll"):
550
556
  claims += 1
@@ -656,7 +662,10 @@ async def test_run_until_timeout_interrupts_storage_pressure(phase: str) -> None
656
662
  return httpx.Response(200, json=info)
657
663
  if request.method == "DELETE":
658
664
  deregistered += 1
659
- return httpx.Response(200, json={})
665
+ return httpx.Response(200, json={
666
+ "worker_id": unquote(path.rsplit("/", 1)[-1]),
667
+ "outcome": "deregistered", "recovered_workflow_task_count": 0,
668
+ })
660
669
  if (
661
670
  (phase == "registration" and path.endswith("/register"))
662
671
  or (phase == "poll" and path.endswith("/poll"))
@@ -813,7 +822,12 @@ async def test_cold_replay_completion_retries_only_its_prepared_acknowledgement(
813
822
  info = compatible_cluster_info()
814
823
  info["worker_protocol"]["server_capabilities"]["query_tasks"] = False
815
824
  return httpx.Response(200, json=info)
816
- if path.endswith("/register") or request.method == "DELETE":
825
+ if request.method == "DELETE":
826
+ return httpx.Response(200, json={
827
+ "worker_id": unquote(path.rsplit("/", 1)[-1]),
828
+ "outcome": "deregistered", "recovered_workflow_task_count": 0,
829
+ })
830
+ if path.endswith("/register"):
817
831
  return httpx.Response(200, json={})
818
832
  if path.endswith("/workflows/cold-replay"):
819
833
  return httpx.Response(200, json={"status": "completed" if completed else "running"})
@@ -3226,7 +3226,7 @@ class TestUpdateValidationTaskExecution:
3226
3226
  assert worker._current_task_slots()["workflow_available"] == 1
3227
3227
 
3228
3228
  @pytest.mark.asyncio
3229
- async def test_shutdown_cancels_validation_and_deregisters_after_releasing_capacity(
3229
+ async def test_shutdown_releases_validation_capacity_without_deregistering_after_deadline(
3230
3230
  self, mock_client: AsyncMock
3231
3231
  ) -> None:
3232
3232
  validation_started = asyncio.Event()
@@ -3279,15 +3279,16 @@ class TestUpdateValidationTaskExecution:
3279
3279
  await asyncio.wait_for(validation_started.wait(), timeout=1.0)
3280
3280
  assert worker._current_task_slots()["workflow_available"] == 0
3281
3281
 
3282
- await worker.stop()
3282
+ with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
3283
+ await worker.stop()
3283
3284
 
3284
3285
  assert validation_cancelled.is_set()
3285
3286
  assert worker._workflow_inflight == 0
3286
3287
  assert worker._workflow_reserved == 0
3287
3288
  assert worker._wf_semaphore._value == 1
3288
3289
  assert worker._current_task_slots()["workflow_available"] == 1
3289
- assert worker._registered is False
3290
- mock_client.deregister_worker_registration.assert_awaited_once_with(worker.worker_id)
3290
+ assert worker._registered is True
3291
+ mock_client.deregister_worker_registration.assert_not_awaited()
3291
3292
 
3292
3293
  @pytest.mark.asyncio
3293
3294
  async def test_cancellation_before_dispatch_starts_releases_admitted_capacity(self, mock_client: AsyncMock) -> None:
@@ -4516,7 +4517,7 @@ class TestWorkerShutdown:
4516
4517
  mock_client.deregister_worker_registration.assert_awaited_once_with(worker.worker_id)
4517
4518
 
4518
4519
  @pytest.mark.asyncio
4519
- async def test_shutdown_deadline_cancels_work_before_deregistering(
4520
+ async def test_shutdown_deadline_cancels_work_without_sending_late_deregistration(
4520
4521
  self, mock_client: AsyncMock
4521
4522
  ) -> None:
4522
4523
  events: list[str] = []
@@ -4539,11 +4540,13 @@ class TestWorkerShutdown:
4539
4540
  tracked_task = worker._track(accepted_work())
4540
4541
  await work_started.wait()
4541
4542
 
4542
- await worker.stop()
4543
+ with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
4544
+ await worker.stop()
4543
4545
 
4544
4546
  assert tracked_task.cancelled()
4545
- assert events == ["work_cancelled", "deregister"]
4546
- mock_client.deregister_worker_registration.assert_awaited_once_with(worker.worker_id)
4547
+ assert events == ["work_cancelled"]
4548
+ assert worker._registered is True
4549
+ mock_client.deregister_worker_registration.assert_not_awaited()
4547
4550
 
4548
4551
  @pytest.mark.asyncio
4549
4552
  async def test_failed_registration_does_not_deregister(self, mock_client: AsyncMock) -> None: