durable-workflow 2.6.2__tar.gz → 2.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. {durable_workflow-2.6.2/src/durable_workflow.egg-info → durable_workflow-2.8.0}/PKG-INFO +15 -1
  2. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/README.md +11 -0
  3. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/pyproject.toml +7 -3
  4. durable_workflow-2.8.0/src/durable_workflow/_activity_lease.py +325 -0
  5. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_activity_process.py +6 -0
  6. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/activity.py +25 -3
  7. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/client.py +238 -19
  8. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/errors.py +85 -3
  9. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/interceptors.py +9 -0
  10. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/retry_policy.py +18 -6
  11. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/sync.py +23 -4
  12. durable_workflow-2.8.0/src/durable_workflow/telemetry.py +341 -0
  13. durable_workflow-2.8.0/src/durable_workflow/telemetry_carrier.py +223 -0
  14. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/worker.py +199 -126
  15. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/workflow_updates_conformance.py +66 -8
  16. {durable_workflow-2.6.2 → durable_workflow-2.8.0/src/durable_workflow.egg-info}/PKG-INFO +15 -1
  17. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/SOURCES.txt +11 -0
  18. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/requires.txt +4 -0
  19. durable_workflow-2.8.0/tests/test_activity_heartbeat_authority.py +155 -0
  20. durable_workflow-2.8.0/tests/test_activity_lease_authority.py +176 -0
  21. durable_workflow-2.8.0/tests/test_activity_lease_renewal.py +131 -0
  22. durable_workflow-2.8.0/tests/test_activity_lease_worker.py +323 -0
  23. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_client.py +246 -3
  24. durable_workflow-2.8.0/tests/test_client_telemetry.py +93 -0
  25. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_errors.py +73 -0
  26. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_regression_corpus.py +62 -1
  27. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_retry_policy.py +12 -0
  28. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sync.py +25 -3
  29. durable_workflow-2.8.0/tests/test_telemetry.py +293 -0
  30. durable_workflow-2.8.0/tests/test_telemetry_carrier.py +158 -0
  31. durable_workflow-2.8.0/tests/test_worker_telemetry_wire.py +205 -0
  32. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_updates_conformance.py +10 -1
  33. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/LICENSE +0 -0
  34. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/setup.cfg +0 -0
  35. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/__init__.py +0 -0
  36. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_avro.py +0 -0
  37. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cancellation_scope.py +0 -0
  38. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cancellation_scope_history.py +0 -0
  39. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cooperative_cancellation.py +0 -0
  40. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_prepared_local_activity.py +0 -0
  41. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_query_task_authority.py +0 -0
  42. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_sticky_workflow_cache.py +0 -0
  43. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/auth_composition.py +0 -0
  44. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/cancellation.py +0 -0
  45. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_storage.py +0 -0
  46. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_task_input.py +0 -0
  47. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_task_result.py +0 -0
  48. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/history_bundle_verify.py +0 -0
  49. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/invocable.py +0 -0
  50. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/metrics.py +0 -0
  51. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/nexus.py +0 -0
  52. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/py.typed +0 -0
  53. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/python_conformance.py +0 -0
  54. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/replay_conformance.py +0 -0
  55. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/replay_verify.py +0 -0
  56. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/serializer.py +0 -0
  57. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/testing.py +0 -0
  58. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/worker_session.py +0 -0
  59. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/workflow.py +0 -0
  60. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/dependency_links.txt +0 -0
  61. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/entry_points.txt +0 -0
  62. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/top_level.txt +0 -0
  63. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_cancellation_policies.py +0 -0
  64. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_context.py +0 -0
  65. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_process.py +0 -0
  66. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_auth_composition.py +0 -0
  67. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_avro_value_benchmark.py +0 -0
  68. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_avro_value_protocol.py +0 -0
  69. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_context.py +0 -0
  70. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_remaining_time.py +0 -0
  71. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_admission.py +0 -0
  72. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_authoring.py +0 -0
  73. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_boundary.py +0 -0
  74. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_opening.py +0 -0
  75. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_child_workflow_policies.py +0 -0
  76. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_ci_checkout.py +0 -0
  77. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_ci_integration_endpoint.py +0 -0
  78. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_client_timeouts.py +0 -0
  79. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_codec_regression_corpus.py +0 -0
  80. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_descendants.py +0 -0
  81. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_groups.py +0 -0
  82. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_history.py +0 -0
  83. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_replay.py +0 -0
  84. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_condition_replay_drift.py +0 -0
  85. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_control_plane_parity_fixtures.py +0 -0
  86. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation.py +0 -0
  87. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation_client.py +0 -0
  88. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation_worker.py +0 -0
  89. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_remote_worker.py +0 -0
  90. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_docstring_cross_references.py +0 -0
  91. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_durable_selection.py +0 -0
  92. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_storage.py +0 -0
  93. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_task_input.py +0 -0
  94. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_task_result.py +0 -0
  95. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_golden_history_replay.py +0 -0
  96. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_history_bundle_verify.py +0 -0
  97. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_history_event_contract.py +0 -0
  98. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_invocable.py +0 -0
  99. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_message_streams.py +0 -0
  100. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_metrics.py +0 -0
  101. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_durable_wait.py +0 -0
  102. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_namespace.py +0 -0
  103. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_wire.py +0 -0
  104. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_order_processing_example.py +0 -0
  105. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_parallel_completion_order_replay.py +0 -0
  106. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_parallel_saga_authoring.py +0 -0
  107. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_patch_history.py +0 -0
  108. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_pending_operation_replay.py +0 -0
  109. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_client.py +0 -0
  110. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_groups.py +0 -0
  111. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_policies.py +0 -0
  112. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_worker.py +0 -0
  113. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_public_boundary_scanner.py +0 -0
  114. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_pypi_project_surface.py +0 -0
  115. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_python_conformance.py +0 -0
  116. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_queries.py +0 -0
  117. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_query_task_authority.py +0 -0
  118. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_query_timeout_retry.py +0 -0
  119. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_readme_quickstart.py +0 -0
  120. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_redrive_replay.py +0 -0
  121. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_registration_ambiguity.py +0 -0
  122. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_release_metadata.py +0 -0
  123. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay.py +0 -0
  124. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_clock.py +0 -0
  125. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_conformance.py +0 -0
  126. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_verify.py +0 -0
  127. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_run_cancellation_scope_replay.py +0 -0
  128. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_runtime_external_payload_transport.py +0 -0
  129. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_schedules.py +0 -0
  130. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_scoped_run_cancellation_context.py +0 -0
  131. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_serializer.py +0 -0
  132. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_signals.py +0 -0
  133. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sleep.py +0 -0
  134. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_standalone_activity_client.py +0 -0
  135. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sticky_execution.py +0 -0
  136. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_storage_admission.py +0 -0
  137. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sync_activities.py +0 -0
  138. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_testing_harness.py +0 -0
  139. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_update_signal_condition_replay.py +0 -0
  140. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_updates.py +0 -0
  141. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_wait_condition.py +0 -0
  142. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker.py +0 -0
  143. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker_observation.py +0 -0
  144. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker_session.py +0 -0
  145. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_entry_arguments.py +0 -0
  146. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_entrypoint_validation.py +0 -0
  147. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_handler_validation.py +0 -0
  148. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_result_timeout.py +0 -0
  149. {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_streams.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: durable-workflow
3
- Version: 2.6.2
3
+ Version: 2.8.0
4
4
  Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
5
5
  Author: Durable Workflow Contributors
6
6
  License-Expression: MIT
@@ -21,6 +21,8 @@ Description-Content-Type: text/markdown
21
21
  License-File: LICENSE
22
22
  Requires-Dist: httpx>=0.27
23
23
  Requires-Dist: fastavro<2,>=1.12.2
24
+ Provides-Extra: telemetry
25
+ Requires-Dist: opentelemetry-api<2,>=1.45.1; extra == "telemetry"
24
26
  Provides-Extra: dev
25
27
  Requires-Dist: pytest>=8.0; extra == "dev"
26
28
  Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
@@ -29,6 +31,7 @@ Requires-Dist: playwright<2,>=1.48; extra == "dev"
29
31
  Requires-Dist: PyYAML>=6.0; extra == "dev"
30
32
  Requires-Dist: ruff>=0.4; extra == "dev"
31
33
  Requires-Dist: tomli>=2; python_version < "3.11" and extra == "dev"
34
+ Requires-Dist: opentelemetry-sdk==1.45.1; extra == "dev"
32
35
  Provides-Extra: prometheus
33
36
  Requires-Dist: prometheus-client>=0.20; extra == "prometheus"
34
37
  Provides-Extra: docs
@@ -182,6 +185,17 @@ independently of application heartbeats. Existing terminal cancellation remains
182
185
  available. See the [cancellation guide](docs/cooperative-cancellation-design.md)
183
186
  for immutable context, operation policies, shielded cleanup and recovery.
184
187
 
188
+ Both `await activity.context().heartbeat()` and synchronous
189
+ `activity.context().heartbeat_sync()` stop at an explicit Server refusal.
190
+ They raise `ActivityLeaseAuthorityLost` with its original `.reason` for an
191
+ elapsed deadline, closed attempt or closed run. Cancellation continues to raise
192
+ `ActivityCancelled`. The worker submits no result or failure for an attempt
193
+ whose heartbeat refused continuation, even if application code catches that
194
+ exception. A heartbeat check acts only when called. Code that never heartbeats
195
+ can continue until it returns, subject to independent lease/cancellation
196
+ supervision; a running native thread cannot be forcibly stopped or its effects
197
+ undone by the SDK.
198
+
185
199
  ## Development
186
200
 
187
201
  The installed `durable-workflow-replay-conformance` command runs an offline
@@ -142,6 +142,17 @@ independently of application heartbeats. Existing terminal cancellation remains
142
142
  available. See the [cancellation guide](docs/cooperative-cancellation-design.md)
143
143
  for immutable context, operation policies, shielded cleanup and recovery.
144
144
 
145
+ Both `await activity.context().heartbeat()` and synchronous
146
+ `activity.context().heartbeat_sync()` stop at an explicit Server refusal.
147
+ They raise `ActivityLeaseAuthorityLost` with its original `.reason` for an
148
+ elapsed deadline, closed attempt or closed run. Cancellation continues to raise
149
+ `ActivityCancelled`. The worker submits no result or failure for an attempt
150
+ whose heartbeat refused continuation, even if application code catches that
151
+ exception. A heartbeat check acts only when called. Code that never heartbeats
152
+ can continue until it returns, subject to independent lease/cancellation
153
+ supervision; a running native thread cannot be forcibly stopped or its effects
154
+ undone by the SDK.
155
+
145
156
  ## Development
146
157
 
147
158
  The installed `durable-workflow-replay-conformance` command runs an offline
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "durable-workflow"
7
- version = "2.6.2"
7
+ version = "2.8.0"
8
8
  description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -38,6 +38,9 @@ dependencies = [
38
38
  ]
39
39
 
40
40
  [project.optional-dependencies]
41
+ telemetry = [
42
+ "opentelemetry-api>=1.45.1,<2",
43
+ ]
41
44
  dev = [
42
45
  "pytest>=8.0",
43
46
  "pytest-asyncio>=0.23",
@@ -46,6 +49,7 @@ dev = [
46
49
  "PyYAML>=6.0",
47
50
  "ruff>=0.4",
48
51
  "tomli>=2; python_version < '3.11'",
52
+ "opentelemetry-sdk==1.45.1",
49
53
  ]
50
54
  prometheus = [
51
55
  "prometheus-client>=0.20",
@@ -71,8 +75,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
71
75
  durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
72
76
 
73
77
  [tool.durable-workflow]
74
- product-train = "2.6.2"
75
- registry-version = "2.6.2"
78
+ product-train = "2.8.0"
79
+ registry-version = "2.8.0"
76
80
  supported-server-versions = "2.8.0"
77
81
  worker-protocol-version = "1.20"
78
82
  control-plane-version = "2"
@@ -0,0 +1,325 @@
1
+ """Runtime lease maintenance for one original activity delivery."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import re
7
+ import time
8
+ from collections.abc import Awaitable, Callable, Mapping
9
+ from contextvars import ContextVar
10
+ from dataclasses import dataclass, field
11
+ from datetime import datetime, timedelta
12
+ from typing import Any, TypeVar
13
+ from urllib.parse import quote
14
+
15
+ import httpx
16
+
17
+ from .errors import ActivityLeaseAuthorityLost, ServerError
18
+
19
+ _T = TypeVar("_T")
20
+ _UTC6 = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\Z")
21
+
22
+
23
+ def _timestamp(value: Any) -> datetime:
24
+ if not isinstance(value, str) or _UTC6.fullmatch(value) is None:
25
+ raise ValueError("activity timing must be UTC with six fractional digits")
26
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
27
+
28
+
29
+ def activity_lease_renewal_supported(client: Any) -> bool:
30
+ """Use the same cached discovery that validates actual renewal requests."""
31
+ info = getattr(client, "_cluster_info", None)
32
+ protocol = info.get("worker_protocol") if isinstance(info, Mapping) else None
33
+ capabilities = protocol.get("server_capabilities") if isinstance(protocol, Mapping) else None
34
+ return isinstance(capabilities, Mapping) and capabilities.get("activity_lease_renewal") is True
35
+
36
+
37
+ def _heartbeat_terms(task: Mapping[str, Any], deadlines: Mapping[str, Any],
38
+ reference: datetime) -> tuple[datetime | None, int | None]:
39
+ heartbeat = _timestamp(deadlines["heartbeat"]) if deadlines.get("heartbeat") is not None else None
40
+ policy = task.get("retry_policy")
41
+ interval = policy.get("heartbeat_timeout") if isinstance(policy, Mapping) else None
42
+ if heartbeat is not None and (type(interval) is not int or interval <= 0):
43
+ raise ValueError("missing recorded activity heartbeat interval")
44
+ interval = interval if type(interval) is int and interval > 0 else None
45
+ if heartbeat is not None and interval is not None and heartbeat - reference > timedelta(seconds=interval):
46
+ raise ValueError("activity heartbeat deadline predates its claimed progress")
47
+ return heartbeat, interval
48
+
49
+
50
+ @dataclass
51
+ class ActivityLeaseAuthority:
52
+ client: Any
53
+ namespace: str
54
+ identity: dict[str, Any]
55
+ task_identity: dict[str, Any]
56
+ captured_at: float
57
+ lease_until: float
58
+ business_until: float | None
59
+ heartbeat_until: float | None
60
+ heartbeat_timeout: int | None
61
+ server_time: datetime
62
+ lease_expiry: datetime
63
+ heartbeat_time: datetime | None
64
+ heartbeat_deadline: datetime | None
65
+ last_renewal: tuple[datetime, datetime, float] | None = None
66
+ lost_reason: str | None = field(default=None, init=False)
67
+
68
+ @classmethod
69
+ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str,
70
+ task_queue: str, server_time: Any) -> ActivityLeaseAuthority:
71
+ # A long poll may wait before a task or its lease exists. Establish the
72
+ # first execution budget with a separate, original-claim confirmation;
73
+ # its entire round trip is charged and it cannot revive an expired lease.
74
+ provisional = cls.delivery(client, namespace, task, owner=owner, task_queue=task_queue,
75
+ server_time=server_time, request_started=time.monotonic())
76
+ started = time.monotonic()
77
+ try:
78
+ reply = await client.renew_activity_task_lease(
79
+ task_id=provisional.task_id, activity_attempt_id=provisional.identity["activity_attempt_id"],
80
+ lease_owner=owner, timeout=min(5.0, provisional.remaining()),
81
+ )
82
+ provisional.acknowledge(reply, started)
83
+ except ActivityLeaseAuthorityLost:
84
+ raise
85
+ except Exception as error:
86
+ reason = error.reason() if isinstance(error, ServerError) else None
87
+ raise ActivityLeaseAuthorityLost(
88
+ provisional.task_id, reason or "activity_initial_lease_unavailable",
89
+ ) from error
90
+ observed = dict(task)
91
+ observed["lease_expires_at"] = reply["lease_expires_at"]
92
+ if provisional.heartbeat_deadline is not None:
93
+ observed["deadlines"] = {**dict(task.get("deadlines", {})),
94
+ "heartbeat": provisional.heartbeat_deadline.isoformat(
95
+ timespec="microseconds").replace("+00:00", "Z")}
96
+ return cls.delivery(client, namespace, observed, owner=owner, task_queue=task_queue,
97
+ server_time=reply["server_time"], request_started=started)
98
+
99
+ @classmethod
100
+ def delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str,
101
+ task_queue: str, server_time: Any, request_started: float) -> ActivityLeaseAuthority:
102
+ task_id = str(task.get("task_id") or "")
103
+ try:
104
+ keys = ("task_id", "workflow_id", "run_id", "activity_execution_id", "activity_attempt_id",
105
+ "lease_owner", "task_queue")
106
+ if (any(not isinstance(task.get(key), str) or not task[key] for key in keys)
107
+ or task["lease_owner"] != owner or task["task_queue"] != task_queue
108
+ or task.get("namespace", namespace) != namespace
109
+ or type(task.get("attempt_number")) is not int or task["attempt_number"] < 1):
110
+ raise ValueError("invalid original activity identity")
111
+ reference = _timestamp(server_time)
112
+ expiry = _timestamp(task.get("lease_expires_at"))
113
+ duration = (expiry - reference).total_seconds()
114
+ if not 0 < duration <= 86400 or not 0 <= request_started <= time.monotonic():
115
+ raise ValueError("invalid original activity lease")
116
+ deadlines = task.get("deadlines", {})
117
+ if not isinstance(deadlines, Mapping):
118
+ raise ValueError("invalid activity deadlines")
119
+ bounds = [_timestamp(deadlines[key]) for key in ("start_to_close", "schedule_to_close", "run", "execution")
120
+ if deadlines.get(key) is not None]
121
+ business = request_started + (min(bounds) - reference).total_seconds() if bounds else None
122
+ heartbeat, interval = _heartbeat_terms(task, deadlines, reference)
123
+ result = cls(
124
+ client, namespace,
125
+ {"workflow_task_id": task["task_id"], "workflow_instance_id": task["workflow_id"],
126
+ "workflow_run_id": task["run_id"], "activity_execution_id": task["activity_execution_id"],
127
+ "activity_attempt_id": task["activity_attempt_id"], "attempt_number": task["attempt_number"],
128
+ "lease_owner": owner},
129
+ {key: task[key] for key in (*keys, "attempt_number")},
130
+ request_started, request_started + duration, business,
131
+ request_started + (heartbeat - reference).total_seconds() if heartbeat is not None else None,
132
+ interval, reference, expiry,
133
+ heartbeat - timedelta(seconds=interval) if heartbeat is not None and interval is not None else None,
134
+ heartbeat,
135
+ )
136
+ except (KeyError, TypeError, ValueError, OverflowError) as error:
137
+ raise ActivityLeaseAuthorityLost(task_id, "invalid_activity_claim") from error
138
+ result.remaining()
139
+ return result
140
+
141
+ @property
142
+ def task_id(self) -> str:
143
+ return str(self.identity["workflow_task_id"])
144
+
145
+ def matches(self, client: Any, namespace: str, task: Mapping[str, Any], owner: str, task_queue: str) -> bool:
146
+ return (self.client is client and self.namespace == namespace and self.identity["lease_owner"] == owner
147
+ and self.task_identity["task_queue"] == task_queue
148
+ and type(task.get("attempt_number")) is int
149
+ and all(task.get(key) == value for key, value in self.task_identity.items()))
150
+
151
+ def governs(self, client: Any, namespace: str, path: str) -> bool:
152
+ prefixes = (f"/worker/activity-tasks/{self.task_id}/",
153
+ f"/worker/activity-tasks/{quote(self.task_id, safe='')}/")
154
+ return (self.client is client and self.namespace == namespace
155
+ and (any(path == prefix + action for prefix in prefixes
156
+ for action in ("complete", "fail", "heartbeat", "renew-lease"))
157
+ or path == "/external-payloads/v1"))
158
+
159
+ def acknowledge_progress(self, reply: Any, request_started: float) -> None:
160
+ """Use an actual application heartbeat, never a runtime liveness ping.
161
+
162
+ The existing heartbeat contract records last_heartbeat_at at the write
163
+ and resets its original recorded timeout interval from that instant.
164
+ Charging the whole request duration consumes storage/network latency.
165
+ """
166
+ self.remaining()
167
+ received = time.monotonic()
168
+ try:
169
+ expected = {"task_id": self.task_id, "activity_attempt_id": self.identity["activity_attempt_id"],
170
+ "lease_owner": self.identity["lease_owner"]}
171
+ if (not isinstance(reply, Mapping) or reply.get("heartbeat_recorded") is not True
172
+ or reply.get("can_continue") is not True or reply.get("cancel_requested") is not False
173
+ or reply.get("reason") is not None
174
+ or any(reply.get(key) != value for key, value in expected.items())
175
+ or not self.captured_at <= request_started <= received):
176
+ raise ValueError("heartbeat did not confirm the original activity")
177
+ progress = _timestamp(reply.get("last_heartbeat_at"))
178
+ expiry = _timestamp(reply.get("lease_expires_at"))
179
+ duration = (expiry - progress).total_seconds()
180
+ if not 0 < duration <= 86400:
181
+ raise ValueError("invalid heartbeat lease grant")
182
+ if self.heartbeat_time is not None and progress <= self.heartbeat_time:
183
+ return
184
+ if request_started + duration <= received:
185
+ raise ValueError("heartbeat lease grant was consumed in transport")
186
+ if self.heartbeat_timeout is not None:
187
+ until = request_started + self.heartbeat_timeout
188
+ if until <= received:
189
+ raise ValueError("heartbeat progress window was consumed in transport")
190
+ self.heartbeat_until = max(self.heartbeat_until or until, until)
191
+ self.heartbeat_deadline = progress + timedelta(seconds=self.heartbeat_timeout)
192
+ self.heartbeat_time = progress
193
+ self.lease_until = max(self.lease_until, request_started + duration)
194
+ self.lease_expiry = max(self.lease_expiry, expiry)
195
+ except (TypeError, ValueError, OverflowError) as error:
196
+ self.lost_reason = "invalid_activity_heartbeat"
197
+ raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
198
+
199
+ def remaining(self) -> float:
200
+ if self.lost_reason is not None:
201
+ raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason)
202
+ bounds = [self.lease_until, *(value for value in (self.business_until, self.heartbeat_until)
203
+ if value is not None)]
204
+ remaining = min(bounds) - time.monotonic()
205
+ if remaining <= 0:
206
+ self.lost_reason = "activity_execution_budget_expired"
207
+ raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason)
208
+ return remaining
209
+
210
+ def acknowledge(self, reply: Any, request_started: float) -> None:
211
+ self.remaining()
212
+ received = time.monotonic()
213
+ try:
214
+ if (not isinstance(reply, Mapping) or reply.get("lease_renewed") is not True
215
+ or reply.get("can_continue") is not True or reply.get("cancel_requested") is not False
216
+ or reply.get("heartbeat_recorded") is not False or reply.get("reason") is not None
217
+ or type(reply.get("attempt_number")) is not int
218
+ or any(reply.get(key) != value for key, value in self.identity.items())
219
+ or not self.captured_at <= request_started <= received):
220
+ raise ValueError("renewal did not preserve the original activity claim")
221
+ server = _timestamp(reply.get("server_time"))
222
+ expiry = _timestamp(reply.get("lease_expires_at"))
223
+ duration = (expiry - server).total_seconds()
224
+ if not 0 < duration <= 86400 or request_started + duration <= received:
225
+ raise ValueError("renewal did not prove a live grant")
226
+ if (self.heartbeat_time is not None and server < self.heartbeat_time and expiry <= self.lease_expiry):
227
+ # An independently acknowledged application heartbeat can
228
+ # overtake an in-flight renewal. Its older reply grants nothing.
229
+ return
230
+ if expiry < self.lease_expiry or server < self.server_time:
231
+ raise ValueError("renewal predates the confirmed grant")
232
+ confirmation = (server, expiry, request_started)
233
+ if self.last_renewal is not None and server <= self.last_renewal[0]:
234
+ if confirmation == self.last_renewal:
235
+ return
236
+ raise ValueError("replayed activity renewal")
237
+ self.lease_until = max(self.lease_until, request_started + duration)
238
+ self._observe_progress(reply, server, request_started, received)
239
+ self.server_time, self.lease_expiry, self.last_renewal = server, expiry, confirmation
240
+ except (TypeError, ValueError, OverflowError) as error:
241
+ self.lost_reason = "invalid_activity_renewal"
242
+ raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
243
+
244
+ def _observe_progress(self, reply: Mapping[str, Any], server: datetime, started: float, received: float) -> None:
245
+ if self.heartbeat_until is None or self.heartbeat_timeout is None:
246
+ return
247
+ try:
248
+ progress = _timestamp(reply.get("last_heartbeat_at"))
249
+ deadline = _timestamp(reply.get("heartbeat_deadline_at"))
250
+ except (TypeError, ValueError):
251
+ return
252
+ until = started + (deadline - server).total_seconds()
253
+ if (progress <= server < deadline and until > received
254
+ and (self.heartbeat_time is None or progress > self.heartbeat_time)
255
+ and (self.heartbeat_deadline is None or deadline > self.heartbeat_deadline)
256
+ and deadline - progress == timedelta(seconds=self.heartbeat_timeout)):
257
+ self.heartbeat_until = max(self.heartbeat_until, until)
258
+ self.heartbeat_time, self.heartbeat_deadline = progress, deadline
259
+
260
+ async def renew(self) -> bool:
261
+ started = time.monotonic()
262
+ try:
263
+ reply = await asyncio.wait_for(self.client.renew_activity_task_lease(
264
+ task_id=self.task_id, activity_attempt_id=self.identity["activity_attempt_id"],
265
+ lease_owner=self.identity["lease_owner"], timeout=min(5.0, self.remaining()),
266
+ ), timeout=min(5.0, self.remaining()))
267
+ self.acknowledge(reply, started)
268
+ return True
269
+ except ActivityLeaseAuthorityLost:
270
+ raise
271
+ except Exception as error:
272
+ if (isinstance(error, (httpx.TransportError, asyncio.TimeoutError))
273
+ or isinstance(error, ServerError) and (error.status >= 500 or error.status == 429)):
274
+ # A failed exchange grants nothing. The existing lease may
275
+ # still authorize work until its original local bound.
276
+ self.remaining()
277
+ return False
278
+ self.lost_reason = "activity_renewal_unavailable"
279
+ raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
280
+
281
+ async def _maintain(self, stopped: asyncio.Event) -> None:
282
+ confirmed = True
283
+ while not stopped.is_set():
284
+ delay = min(30.0 if confirmed else 1.0, max(0.0, self.lease_until - time.monotonic()) / 3)
285
+ if self.heartbeat_timeout is not None:
286
+ delay = min(delay, self.heartbeat_timeout / 3)
287
+ await asyncio.sleep(min(delay, self.remaining()))
288
+ if stopped.is_set():
289
+ return
290
+ confirmed = await self.renew()
291
+
292
+ async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T:
293
+ self.remaining()
294
+ stopped = asyncio.Event()
295
+ execution = asyncio.ensure_future(operation())
296
+ maintenance = asyncio.create_task(self._maintain(stopped))
297
+ try:
298
+ while True:
299
+ done, _ = await asyncio.wait({execution, maintenance}, timeout=self.remaining(),
300
+ return_when=asyncio.FIRST_COMPLETED)
301
+ if maintenance in done:
302
+ await maintenance
303
+ raise ActivityLeaseAuthorityLost(self.task_id, "activity_renewal_stopped")
304
+ if execution in done:
305
+ result = await execution
306
+ self.remaining()
307
+ return result
308
+ except BaseException:
309
+ self.lost_reason = self.lost_reason or "activity_execution_abandoned"
310
+ raise
311
+ finally:
312
+ # An I/O wrapper may complete concurrently with cancel() and consume
313
+ # that cancellation. The settled scope must still stop renewing.
314
+ stopped.set()
315
+ # A Python thread may continue after cancellation. Its existing
316
+ # executor keeps the physical slot until it actually exits, while
317
+ # this context independently fences every late HTTP publication.
318
+ for future in (execution, maintenance):
319
+ future.cancel()
320
+ await asyncio.gather(execution, maintenance, return_exceptions=True)
321
+
322
+
323
+ current_activity_authority: ContextVar[ActivityLeaseAuthority | None] = ContextVar(
324
+ "activity_lease_authority", default=None,
325
+ )
@@ -40,6 +40,10 @@ class CallbackInvocation:
40
40
  info: ActivityInfo
41
41
  task: dict[str, Any]
42
42
  interceptors: tuple[WorkerInterceptor, ...] = ()
43
+ namespace: str | None = None
44
+ observability_supported: bool = False
45
+ remote: bool = False
46
+ execution_context: dict[str, Any] | None = None
43
47
 
44
48
  def serialize(self) -> bytes:
45
49
  try:
@@ -100,6 +104,8 @@ def _callback(directory: str, channel: socket.socket) -> None:
100
104
  context = ActivityInterceptorContext(
101
105
  worker_id=invocation.info.worker_id, task_queue=invocation.info.task_queue,
102
106
  task=invocation.task, activity_type=invocation.info.activity_type, args=invocation.args,
107
+ namespace=invocation.namespace, observability_supported=invocation.observability_supported,
108
+ remote=invocation.remote, execution_context=invocation.execution_context,
103
109
  )
104
110
 
105
111
  async def call(ctx: ActivityInterceptorContext) -> Any:
@@ -7,7 +7,8 @@ from collections.abc import Awaitable, Callable
7
7
  from dataclasses import dataclass
8
8
  from typing import TYPE_CHECKING, Any
9
9
 
10
- from .errors import ActivityCancelled
10
+ from ._activity_lease import current_activity_authority
11
+ from .errors import ActivityCancelled, ActivityLeaseAuthorityLost
11
12
 
12
13
  if TYPE_CHECKING:
13
14
  from .client import Client
@@ -45,6 +46,7 @@ class ActivityContext:
45
46
  self._client = client
46
47
  self._heartbeat_callback = heartbeat_callback
47
48
  self._cancel_requested = False
49
+ self._cannot_continue_reason: str | None = None
48
50
  try:
49
51
  self._event_loop: asyncio.AbstractEventLoop | None = asyncio.get_running_loop()
50
52
  except RuntimeError:
@@ -61,7 +63,7 @@ class ActivityContext:
61
63
  return self._cancel_requested
62
64
 
63
65
  def heartbeat_sync(self, details: dict[str, Any] | None = None) -> None:
64
- """Heartbeat from a synchronous activity thread and observe cancellation.
66
+ """Heartbeat from a synchronous activity thread and check attempt authority.
65
67
 
66
68
  The request runs on the worker's event loop using the original attempt
67
69
  and client. Async handlers must use ``await heartbeat()`` instead.
@@ -84,7 +86,7 @@ class ActivityContext:
84
86
  pending.result()
85
87
 
86
88
  async def heartbeat(self, details: dict[str, Any] | None = None) -> None:
87
- """Report liveness to the server and check for a cancellation request.
89
+ """Report progress and check cancellation and continued attempt authority.
88
90
 
89
91
  Long-running activities should call ``heartbeat()`` periodically so the
90
92
  server can distinguish a slow-but-alive attempt from a dead worker.
@@ -94,7 +96,12 @@ class ActivityContext:
94
96
  Raises :class:`~durable_workflow.errors.ActivityCancelled` when the
95
97
  owning workflow has requested cancellation, so the activity can exit
96
98
  cleanly at its next natural break point.
99
+
100
+ Raises :class:`~durable_workflow.errors.ActivityLeaseAuthorityLost`
101
+ with the original Server reason when the attempt cannot continue,
102
+ including an elapsed deadline, closed attempt or closed run.
97
103
  """
104
+ self._raise_if_cannot_continue()
98
105
  if self._heartbeat_callback is not None:
99
106
  try:
100
107
  await self._heartbeat_callback(details)
@@ -111,6 +118,21 @@ class ActivityContext:
111
118
  if isinstance(resp, dict) and resp.get("cancel_requested"):
112
119
  self._cancel_requested = True
113
120
  raise ActivityCancelled()
121
+ if isinstance(resp, dict) and resp.get("can_continue") is False:
122
+ reason = resp.get("reason")
123
+ self._cannot_continue_reason = reason if isinstance(reason, str) and reason else "activity_cannot_continue"
124
+ self._raise_if_cannot_continue()
125
+
126
+ def _raise_if_cannot_continue(self) -> None:
127
+ if self._cannot_continue_reason is not None:
128
+ authority = current_activity_authority.get()
129
+ if (authority is not None and authority.client is self._client
130
+ and authority.task_id == self._info.task_id
131
+ and authority.identity["activity_attempt_id"] == self._info.activity_attempt_id
132
+ and authority.identity["attempt_number"] == self._info.attempt_number
133
+ and authority.identity["lease_owner"] == self._info.worker_id):
134
+ authority.lost_reason = authority.lost_reason or self._cannot_continue_reason
135
+ raise ActivityLeaseAuthorityLost(self._info.task_id, self._cannot_continue_reason)
114
136
 
115
137
 
116
138
  def context() -> ActivityContext: