durable-workflow 2.6.2__tar.gz → 2.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {durable_workflow-2.6.2/src/durable_workflow.egg-info → durable_workflow-2.8.0}/PKG-INFO +15 -1
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/README.md +11 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/pyproject.toml +7 -3
- durable_workflow-2.8.0/src/durable_workflow/_activity_lease.py +325 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_activity_process.py +6 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/activity.py +25 -3
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/client.py +238 -19
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/errors.py +85 -3
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/interceptors.py +9 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/retry_policy.py +18 -6
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/sync.py +23 -4
- durable_workflow-2.8.0/src/durable_workflow/telemetry.py +341 -0
- durable_workflow-2.8.0/src/durable_workflow/telemetry_carrier.py +223 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/worker.py +199 -126
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/workflow_updates_conformance.py +66 -8
- {durable_workflow-2.6.2 → durable_workflow-2.8.0/src/durable_workflow.egg-info}/PKG-INFO +15 -1
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/SOURCES.txt +11 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/requires.txt +4 -0
- durable_workflow-2.8.0/tests/test_activity_heartbeat_authority.py +155 -0
- durable_workflow-2.8.0/tests/test_activity_lease_authority.py +176 -0
- durable_workflow-2.8.0/tests/test_activity_lease_renewal.py +131 -0
- durable_workflow-2.8.0/tests/test_activity_lease_worker.py +323 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_client.py +246 -3
- durable_workflow-2.8.0/tests/test_client_telemetry.py +93 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_errors.py +73 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_regression_corpus.py +62 -1
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_retry_policy.py +12 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sync.py +25 -3
- durable_workflow-2.8.0/tests/test_telemetry.py +293 -0
- durable_workflow-2.8.0/tests/test_telemetry_carrier.py +158 -0
- durable_workflow-2.8.0/tests/test_worker_telemetry_wire.py +205 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_updates_conformance.py +10 -1
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/LICENSE +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/setup.cfg +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/__init__.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_avro.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cancellation_scope.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cancellation_scope_history.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_cooperative_cancellation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_prepared_local_activity.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_query_task_authority.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/_sticky_workflow_cache.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/auth_composition.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/cancellation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_storage.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_task_input.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/external_task_result.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/history_bundle_verify.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/invocable.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/metrics.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/nexus.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/py.typed +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/python_conformance.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/replay_conformance.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/replay_verify.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/serializer.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/testing.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/worker_session.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow/workflow.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/dependency_links.txt +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/entry_points.txt +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/src/durable_workflow.egg-info/top_level.txt +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_cancellation_policies.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_context.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_activity_process.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_auth_composition.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_avro_value_benchmark.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_avro_value_protocol.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_context.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_remaining_time.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_admission.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_authoring.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_boundary.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cancellation_scope_opening.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_child_workflow_policies.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_ci_checkout.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_ci_integration_endpoint.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_client_timeouts.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_codec_regression_corpus.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_descendants.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_groups.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_history.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_committed_cancellation_scope_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_condition_replay_drift.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_control_plane_parity_fixtures.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation_client.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_cancellation_worker.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_cooperative_remote_worker.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_docstring_cross_references.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_durable_selection.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_storage.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_task_input.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_external_task_result.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_golden_history_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_history_bundle_verify.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_history_event_contract.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_invocable.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_message_streams.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_metrics.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_durable_wait.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_namespace.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_nexus_wire.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_order_processing_example.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_parallel_completion_order_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_parallel_saga_authoring.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_patch_history.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_pending_operation_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_client.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_groups.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_policies.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_prepared_local_activity_worker.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_public_boundary_scanner.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_pypi_project_surface.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_python_conformance.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_queries.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_query_task_authority.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_query_timeout_retry.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_readme_quickstart.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_redrive_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_registration_ambiguity.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_release_metadata.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_clock.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_conformance.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_replay_verify.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_run_cancellation_scope_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_runtime_external_payload_transport.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_schedules.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_scoped_run_cancellation_context.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_serializer.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_signals.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sleep.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_standalone_activity_client.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sticky_execution.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_storage_admission.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_sync_activities.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_testing_harness.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_update_signal_condition_replay.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_updates.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_wait_condition.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker_observation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_worker_session.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_entry_arguments.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_entrypoint_validation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_handler_validation.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_result_timeout.py +0 -0
- {durable_workflow-2.6.2 → durable_workflow-2.8.0}/tests/test_workflow_streams.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: durable-workflow
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.8.0
|
|
4
4
|
Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
|
|
5
5
|
Author: Durable Workflow Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -21,6 +21,8 @@ Description-Content-Type: text/markdown
|
|
|
21
21
|
License-File: LICENSE
|
|
22
22
|
Requires-Dist: httpx>=0.27
|
|
23
23
|
Requires-Dist: fastavro<2,>=1.12.2
|
|
24
|
+
Provides-Extra: telemetry
|
|
25
|
+
Requires-Dist: opentelemetry-api<2,>=1.45.1; extra == "telemetry"
|
|
24
26
|
Provides-Extra: dev
|
|
25
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
26
28
|
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
@@ -29,6 +31,7 @@ Requires-Dist: playwright<2,>=1.48; extra == "dev"
|
|
|
29
31
|
Requires-Dist: PyYAML>=6.0; extra == "dev"
|
|
30
32
|
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
31
33
|
Requires-Dist: tomli>=2; python_version < "3.11" and extra == "dev"
|
|
34
|
+
Requires-Dist: opentelemetry-sdk==1.45.1; extra == "dev"
|
|
32
35
|
Provides-Extra: prometheus
|
|
33
36
|
Requires-Dist: prometheus-client>=0.20; extra == "prometheus"
|
|
34
37
|
Provides-Extra: docs
|
|
@@ -182,6 +185,17 @@ independently of application heartbeats. Existing terminal cancellation remains
|
|
|
182
185
|
available. See the [cancellation guide](docs/cooperative-cancellation-design.md)
|
|
183
186
|
for immutable context, operation policies, shielded cleanup and recovery.
|
|
184
187
|
|
|
188
|
+
Both `await activity.context().heartbeat()` and synchronous
|
|
189
|
+
`activity.context().heartbeat_sync()` stop at an explicit Server refusal.
|
|
190
|
+
They raise `ActivityLeaseAuthorityLost` with its original `.reason` for an
|
|
191
|
+
elapsed deadline, closed attempt or closed run. Cancellation continues to raise
|
|
192
|
+
`ActivityCancelled`. The worker submits no result or failure for an attempt
|
|
193
|
+
whose heartbeat refused continuation, even if application code catches that
|
|
194
|
+
exception. A heartbeat check acts only when called. Code that never heartbeats
|
|
195
|
+
can continue until it returns, subject to independent lease/cancellation
|
|
196
|
+
supervision; a running native thread cannot be forcibly stopped or its effects
|
|
197
|
+
undone by the SDK.
|
|
198
|
+
|
|
185
199
|
## Development
|
|
186
200
|
|
|
187
201
|
The installed `durable-workflow-replay-conformance` command runs an offline
|
|
@@ -142,6 +142,17 @@ independently of application heartbeats. Existing terminal cancellation remains
|
|
|
142
142
|
available. See the [cancellation guide](docs/cooperative-cancellation-design.md)
|
|
143
143
|
for immutable context, operation policies, shielded cleanup and recovery.
|
|
144
144
|
|
|
145
|
+
Both `await activity.context().heartbeat()` and synchronous
|
|
146
|
+
`activity.context().heartbeat_sync()` stop at an explicit Server refusal.
|
|
147
|
+
They raise `ActivityLeaseAuthorityLost` with its original `.reason` for an
|
|
148
|
+
elapsed deadline, closed attempt or closed run. Cancellation continues to raise
|
|
149
|
+
`ActivityCancelled`. The worker submits no result or failure for an attempt
|
|
150
|
+
whose heartbeat refused continuation, even if application code catches that
|
|
151
|
+
exception. A heartbeat check acts only when called. Code that never heartbeats
|
|
152
|
+
can continue until it returns, subject to independent lease/cancellation
|
|
153
|
+
supervision; a running native thread cannot be forcibly stopped or its effects
|
|
154
|
+
undone by the SDK.
|
|
155
|
+
|
|
145
156
|
## Development
|
|
146
157
|
|
|
147
158
|
The installed `durable-workflow-replay-conformance` command runs an offline
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "durable-workflow"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.8.0"
|
|
8
8
|
description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -38,6 +38,9 @@ dependencies = [
|
|
|
38
38
|
]
|
|
39
39
|
|
|
40
40
|
[project.optional-dependencies]
|
|
41
|
+
telemetry = [
|
|
42
|
+
"opentelemetry-api>=1.45.1,<2",
|
|
43
|
+
]
|
|
41
44
|
dev = [
|
|
42
45
|
"pytest>=8.0",
|
|
43
46
|
"pytest-asyncio>=0.23",
|
|
@@ -46,6 +49,7 @@ dev = [
|
|
|
46
49
|
"PyYAML>=6.0",
|
|
47
50
|
"ruff>=0.4",
|
|
48
51
|
"tomli>=2; python_version < '3.11'",
|
|
52
|
+
"opentelemetry-sdk==1.45.1",
|
|
49
53
|
]
|
|
50
54
|
prometheus = [
|
|
51
55
|
"prometheus-client>=0.20",
|
|
@@ -71,8 +75,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
|
|
|
71
75
|
durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
|
|
72
76
|
|
|
73
77
|
[tool.durable-workflow]
|
|
74
|
-
product-train = "2.
|
|
75
|
-
registry-version = "2.
|
|
78
|
+
product-train = "2.8.0"
|
|
79
|
+
registry-version = "2.8.0"
|
|
76
80
|
supported-server-versions = "2.8.0"
|
|
77
81
|
worker-protocol-version = "1.20"
|
|
78
82
|
control-plane-version = "2"
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Runtime lease maintenance for one original activity delivery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import re
|
|
7
|
+
import time
|
|
8
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
9
|
+
from contextvars import ContextVar
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from datetime import datetime, timedelta
|
|
12
|
+
from typing import Any, TypeVar
|
|
13
|
+
from urllib.parse import quote
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from .errors import ActivityLeaseAuthorityLost, ServerError
|
|
18
|
+
|
|
19
|
+
_T = TypeVar("_T")
|
|
20
|
+
_UTC6 = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\Z")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _timestamp(value: Any) -> datetime:
|
|
24
|
+
if not isinstance(value, str) or _UTC6.fullmatch(value) is None:
|
|
25
|
+
raise ValueError("activity timing must be UTC with six fractional digits")
|
|
26
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def activity_lease_renewal_supported(client: Any) -> bool:
|
|
30
|
+
"""Use the same cached discovery that validates actual renewal requests."""
|
|
31
|
+
info = getattr(client, "_cluster_info", None)
|
|
32
|
+
protocol = info.get("worker_protocol") if isinstance(info, Mapping) else None
|
|
33
|
+
capabilities = protocol.get("server_capabilities") if isinstance(protocol, Mapping) else None
|
|
34
|
+
return isinstance(capabilities, Mapping) and capabilities.get("activity_lease_renewal") is True
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _heartbeat_terms(task: Mapping[str, Any], deadlines: Mapping[str, Any],
|
|
38
|
+
reference: datetime) -> tuple[datetime | None, int | None]:
|
|
39
|
+
heartbeat = _timestamp(deadlines["heartbeat"]) if deadlines.get("heartbeat") is not None else None
|
|
40
|
+
policy = task.get("retry_policy")
|
|
41
|
+
interval = policy.get("heartbeat_timeout") if isinstance(policy, Mapping) else None
|
|
42
|
+
if heartbeat is not None and (type(interval) is not int or interval <= 0):
|
|
43
|
+
raise ValueError("missing recorded activity heartbeat interval")
|
|
44
|
+
interval = interval if type(interval) is int and interval > 0 else None
|
|
45
|
+
if heartbeat is not None and interval is not None and heartbeat - reference > timedelta(seconds=interval):
|
|
46
|
+
raise ValueError("activity heartbeat deadline predates its claimed progress")
|
|
47
|
+
return heartbeat, interval
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class ActivityLeaseAuthority:
|
|
52
|
+
client: Any
|
|
53
|
+
namespace: str
|
|
54
|
+
identity: dict[str, Any]
|
|
55
|
+
task_identity: dict[str, Any]
|
|
56
|
+
captured_at: float
|
|
57
|
+
lease_until: float
|
|
58
|
+
business_until: float | None
|
|
59
|
+
heartbeat_until: float | None
|
|
60
|
+
heartbeat_timeout: int | None
|
|
61
|
+
server_time: datetime
|
|
62
|
+
lease_expiry: datetime
|
|
63
|
+
heartbeat_time: datetime | None
|
|
64
|
+
heartbeat_deadline: datetime | None
|
|
65
|
+
last_renewal: tuple[datetime, datetime, float] | None = None
|
|
66
|
+
lost_reason: str | None = field(default=None, init=False)
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str,
|
|
70
|
+
task_queue: str, server_time: Any) -> ActivityLeaseAuthority:
|
|
71
|
+
# A long poll may wait before a task or its lease exists. Establish the
|
|
72
|
+
# first execution budget with a separate, original-claim confirmation;
|
|
73
|
+
# its entire round trip is charged and it cannot revive an expired lease.
|
|
74
|
+
provisional = cls.delivery(client, namespace, task, owner=owner, task_queue=task_queue,
|
|
75
|
+
server_time=server_time, request_started=time.monotonic())
|
|
76
|
+
started = time.monotonic()
|
|
77
|
+
try:
|
|
78
|
+
reply = await client.renew_activity_task_lease(
|
|
79
|
+
task_id=provisional.task_id, activity_attempt_id=provisional.identity["activity_attempt_id"],
|
|
80
|
+
lease_owner=owner, timeout=min(5.0, provisional.remaining()),
|
|
81
|
+
)
|
|
82
|
+
provisional.acknowledge(reply, started)
|
|
83
|
+
except ActivityLeaseAuthorityLost:
|
|
84
|
+
raise
|
|
85
|
+
except Exception as error:
|
|
86
|
+
reason = error.reason() if isinstance(error, ServerError) else None
|
|
87
|
+
raise ActivityLeaseAuthorityLost(
|
|
88
|
+
provisional.task_id, reason or "activity_initial_lease_unavailable",
|
|
89
|
+
) from error
|
|
90
|
+
observed = dict(task)
|
|
91
|
+
observed["lease_expires_at"] = reply["lease_expires_at"]
|
|
92
|
+
if provisional.heartbeat_deadline is not None:
|
|
93
|
+
observed["deadlines"] = {**dict(task.get("deadlines", {})),
|
|
94
|
+
"heartbeat": provisional.heartbeat_deadline.isoformat(
|
|
95
|
+
timespec="microseconds").replace("+00:00", "Z")}
|
|
96
|
+
return cls.delivery(client, namespace, observed, owner=owner, task_queue=task_queue,
|
|
97
|
+
server_time=reply["server_time"], request_started=started)
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str,
|
|
101
|
+
task_queue: str, server_time: Any, request_started: float) -> ActivityLeaseAuthority:
|
|
102
|
+
task_id = str(task.get("task_id") or "")
|
|
103
|
+
try:
|
|
104
|
+
keys = ("task_id", "workflow_id", "run_id", "activity_execution_id", "activity_attempt_id",
|
|
105
|
+
"lease_owner", "task_queue")
|
|
106
|
+
if (any(not isinstance(task.get(key), str) or not task[key] for key in keys)
|
|
107
|
+
or task["lease_owner"] != owner or task["task_queue"] != task_queue
|
|
108
|
+
or task.get("namespace", namespace) != namespace
|
|
109
|
+
or type(task.get("attempt_number")) is not int or task["attempt_number"] < 1):
|
|
110
|
+
raise ValueError("invalid original activity identity")
|
|
111
|
+
reference = _timestamp(server_time)
|
|
112
|
+
expiry = _timestamp(task.get("lease_expires_at"))
|
|
113
|
+
duration = (expiry - reference).total_seconds()
|
|
114
|
+
if not 0 < duration <= 86400 or not 0 <= request_started <= time.monotonic():
|
|
115
|
+
raise ValueError("invalid original activity lease")
|
|
116
|
+
deadlines = task.get("deadlines", {})
|
|
117
|
+
if not isinstance(deadlines, Mapping):
|
|
118
|
+
raise ValueError("invalid activity deadlines")
|
|
119
|
+
bounds = [_timestamp(deadlines[key]) for key in ("start_to_close", "schedule_to_close", "run", "execution")
|
|
120
|
+
if deadlines.get(key) is not None]
|
|
121
|
+
business = request_started + (min(bounds) - reference).total_seconds() if bounds else None
|
|
122
|
+
heartbeat, interval = _heartbeat_terms(task, deadlines, reference)
|
|
123
|
+
result = cls(
|
|
124
|
+
client, namespace,
|
|
125
|
+
{"workflow_task_id": task["task_id"], "workflow_instance_id": task["workflow_id"],
|
|
126
|
+
"workflow_run_id": task["run_id"], "activity_execution_id": task["activity_execution_id"],
|
|
127
|
+
"activity_attempt_id": task["activity_attempt_id"], "attempt_number": task["attempt_number"],
|
|
128
|
+
"lease_owner": owner},
|
|
129
|
+
{key: task[key] for key in (*keys, "attempt_number")},
|
|
130
|
+
request_started, request_started + duration, business,
|
|
131
|
+
request_started + (heartbeat - reference).total_seconds() if heartbeat is not None else None,
|
|
132
|
+
interval, reference, expiry,
|
|
133
|
+
heartbeat - timedelta(seconds=interval) if heartbeat is not None and interval is not None else None,
|
|
134
|
+
heartbeat,
|
|
135
|
+
)
|
|
136
|
+
except (KeyError, TypeError, ValueError, OverflowError) as error:
|
|
137
|
+
raise ActivityLeaseAuthorityLost(task_id, "invalid_activity_claim") from error
|
|
138
|
+
result.remaining()
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def task_id(self) -> str:
|
|
143
|
+
return str(self.identity["workflow_task_id"])
|
|
144
|
+
|
|
145
|
+
def matches(self, client: Any, namespace: str, task: Mapping[str, Any], owner: str, task_queue: str) -> bool:
|
|
146
|
+
return (self.client is client and self.namespace == namespace and self.identity["lease_owner"] == owner
|
|
147
|
+
and self.task_identity["task_queue"] == task_queue
|
|
148
|
+
and type(task.get("attempt_number")) is int
|
|
149
|
+
and all(task.get(key) == value for key, value in self.task_identity.items()))
|
|
150
|
+
|
|
151
|
+
def governs(self, client: Any, namespace: str, path: str) -> bool:
|
|
152
|
+
prefixes = (f"/worker/activity-tasks/{self.task_id}/",
|
|
153
|
+
f"/worker/activity-tasks/{quote(self.task_id, safe='')}/")
|
|
154
|
+
return (self.client is client and self.namespace == namespace
|
|
155
|
+
and (any(path == prefix + action for prefix in prefixes
|
|
156
|
+
for action in ("complete", "fail", "heartbeat", "renew-lease"))
|
|
157
|
+
or path == "/external-payloads/v1"))
|
|
158
|
+
|
|
159
|
+
def acknowledge_progress(self, reply: Any, request_started: float) -> None:
|
|
160
|
+
"""Use an actual application heartbeat, never a runtime liveness ping.
|
|
161
|
+
|
|
162
|
+
The existing heartbeat contract records last_heartbeat_at at the write
|
|
163
|
+
and resets its original recorded timeout interval from that instant.
|
|
164
|
+
Charging the whole request duration consumes storage/network latency.
|
|
165
|
+
"""
|
|
166
|
+
self.remaining()
|
|
167
|
+
received = time.monotonic()
|
|
168
|
+
try:
|
|
169
|
+
expected = {"task_id": self.task_id, "activity_attempt_id": self.identity["activity_attempt_id"],
|
|
170
|
+
"lease_owner": self.identity["lease_owner"]}
|
|
171
|
+
if (not isinstance(reply, Mapping) or reply.get("heartbeat_recorded") is not True
|
|
172
|
+
or reply.get("can_continue") is not True or reply.get("cancel_requested") is not False
|
|
173
|
+
or reply.get("reason") is not None
|
|
174
|
+
or any(reply.get(key) != value for key, value in expected.items())
|
|
175
|
+
or not self.captured_at <= request_started <= received):
|
|
176
|
+
raise ValueError("heartbeat did not confirm the original activity")
|
|
177
|
+
progress = _timestamp(reply.get("last_heartbeat_at"))
|
|
178
|
+
expiry = _timestamp(reply.get("lease_expires_at"))
|
|
179
|
+
duration = (expiry - progress).total_seconds()
|
|
180
|
+
if not 0 < duration <= 86400:
|
|
181
|
+
raise ValueError("invalid heartbeat lease grant")
|
|
182
|
+
if self.heartbeat_time is not None and progress <= self.heartbeat_time:
|
|
183
|
+
return
|
|
184
|
+
if request_started + duration <= received:
|
|
185
|
+
raise ValueError("heartbeat lease grant was consumed in transport")
|
|
186
|
+
if self.heartbeat_timeout is not None:
|
|
187
|
+
until = request_started + self.heartbeat_timeout
|
|
188
|
+
if until <= received:
|
|
189
|
+
raise ValueError("heartbeat progress window was consumed in transport")
|
|
190
|
+
self.heartbeat_until = max(self.heartbeat_until or until, until)
|
|
191
|
+
self.heartbeat_deadline = progress + timedelta(seconds=self.heartbeat_timeout)
|
|
192
|
+
self.heartbeat_time = progress
|
|
193
|
+
self.lease_until = max(self.lease_until, request_started + duration)
|
|
194
|
+
self.lease_expiry = max(self.lease_expiry, expiry)
|
|
195
|
+
except (TypeError, ValueError, OverflowError) as error:
|
|
196
|
+
self.lost_reason = "invalid_activity_heartbeat"
|
|
197
|
+
raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
|
|
198
|
+
|
|
199
|
+
def remaining(self) -> float:
|
|
200
|
+
if self.lost_reason is not None:
|
|
201
|
+
raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason)
|
|
202
|
+
bounds = [self.lease_until, *(value for value in (self.business_until, self.heartbeat_until)
|
|
203
|
+
if value is not None)]
|
|
204
|
+
remaining = min(bounds) - time.monotonic()
|
|
205
|
+
if remaining <= 0:
|
|
206
|
+
self.lost_reason = "activity_execution_budget_expired"
|
|
207
|
+
raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason)
|
|
208
|
+
return remaining
|
|
209
|
+
|
|
210
|
+
def acknowledge(self, reply: Any, request_started: float) -> None:
|
|
211
|
+
self.remaining()
|
|
212
|
+
received = time.monotonic()
|
|
213
|
+
try:
|
|
214
|
+
if (not isinstance(reply, Mapping) or reply.get("lease_renewed") is not True
|
|
215
|
+
or reply.get("can_continue") is not True or reply.get("cancel_requested") is not False
|
|
216
|
+
or reply.get("heartbeat_recorded") is not False or reply.get("reason") is not None
|
|
217
|
+
or type(reply.get("attempt_number")) is not int
|
|
218
|
+
or any(reply.get(key) != value for key, value in self.identity.items())
|
|
219
|
+
or not self.captured_at <= request_started <= received):
|
|
220
|
+
raise ValueError("renewal did not preserve the original activity claim")
|
|
221
|
+
server = _timestamp(reply.get("server_time"))
|
|
222
|
+
expiry = _timestamp(reply.get("lease_expires_at"))
|
|
223
|
+
duration = (expiry - server).total_seconds()
|
|
224
|
+
if not 0 < duration <= 86400 or request_started + duration <= received:
|
|
225
|
+
raise ValueError("renewal did not prove a live grant")
|
|
226
|
+
if (self.heartbeat_time is not None and server < self.heartbeat_time and expiry <= self.lease_expiry):
|
|
227
|
+
# An independently acknowledged application heartbeat can
|
|
228
|
+
# overtake an in-flight renewal. Its older reply grants nothing.
|
|
229
|
+
return
|
|
230
|
+
if expiry < self.lease_expiry or server < self.server_time:
|
|
231
|
+
raise ValueError("renewal predates the confirmed grant")
|
|
232
|
+
confirmation = (server, expiry, request_started)
|
|
233
|
+
if self.last_renewal is not None and server <= self.last_renewal[0]:
|
|
234
|
+
if confirmation == self.last_renewal:
|
|
235
|
+
return
|
|
236
|
+
raise ValueError("replayed activity renewal")
|
|
237
|
+
self.lease_until = max(self.lease_until, request_started + duration)
|
|
238
|
+
self._observe_progress(reply, server, request_started, received)
|
|
239
|
+
self.server_time, self.lease_expiry, self.last_renewal = server, expiry, confirmation
|
|
240
|
+
except (TypeError, ValueError, OverflowError) as error:
|
|
241
|
+
self.lost_reason = "invalid_activity_renewal"
|
|
242
|
+
raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
|
|
243
|
+
|
|
244
|
+
def _observe_progress(self, reply: Mapping[str, Any], server: datetime, started: float, received: float) -> None:
|
|
245
|
+
if self.heartbeat_until is None or self.heartbeat_timeout is None:
|
|
246
|
+
return
|
|
247
|
+
try:
|
|
248
|
+
progress = _timestamp(reply.get("last_heartbeat_at"))
|
|
249
|
+
deadline = _timestamp(reply.get("heartbeat_deadline_at"))
|
|
250
|
+
except (TypeError, ValueError):
|
|
251
|
+
return
|
|
252
|
+
until = started + (deadline - server).total_seconds()
|
|
253
|
+
if (progress <= server < deadline and until > received
|
|
254
|
+
and (self.heartbeat_time is None or progress > self.heartbeat_time)
|
|
255
|
+
and (self.heartbeat_deadline is None or deadline > self.heartbeat_deadline)
|
|
256
|
+
and deadline - progress == timedelta(seconds=self.heartbeat_timeout)):
|
|
257
|
+
self.heartbeat_until = max(self.heartbeat_until, until)
|
|
258
|
+
self.heartbeat_time, self.heartbeat_deadline = progress, deadline
|
|
259
|
+
|
|
260
|
+
async def renew(self) -> bool:
|
|
261
|
+
started = time.monotonic()
|
|
262
|
+
try:
|
|
263
|
+
reply = await asyncio.wait_for(self.client.renew_activity_task_lease(
|
|
264
|
+
task_id=self.task_id, activity_attempt_id=self.identity["activity_attempt_id"],
|
|
265
|
+
lease_owner=self.identity["lease_owner"], timeout=min(5.0, self.remaining()),
|
|
266
|
+
), timeout=min(5.0, self.remaining()))
|
|
267
|
+
self.acknowledge(reply, started)
|
|
268
|
+
return True
|
|
269
|
+
except ActivityLeaseAuthorityLost:
|
|
270
|
+
raise
|
|
271
|
+
except Exception as error:
|
|
272
|
+
if (isinstance(error, (httpx.TransportError, asyncio.TimeoutError))
|
|
273
|
+
or isinstance(error, ServerError) and (error.status >= 500 or error.status == 429)):
|
|
274
|
+
# A failed exchange grants nothing. The existing lease may
|
|
275
|
+
# still authorize work until its original local bound.
|
|
276
|
+
self.remaining()
|
|
277
|
+
return False
|
|
278
|
+
self.lost_reason = "activity_renewal_unavailable"
|
|
279
|
+
raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error
|
|
280
|
+
|
|
281
|
+
async def _maintain(self, stopped: asyncio.Event) -> None:
|
|
282
|
+
confirmed = True
|
|
283
|
+
while not stopped.is_set():
|
|
284
|
+
delay = min(30.0 if confirmed else 1.0, max(0.0, self.lease_until - time.monotonic()) / 3)
|
|
285
|
+
if self.heartbeat_timeout is not None:
|
|
286
|
+
delay = min(delay, self.heartbeat_timeout / 3)
|
|
287
|
+
await asyncio.sleep(min(delay, self.remaining()))
|
|
288
|
+
if stopped.is_set():
|
|
289
|
+
return
|
|
290
|
+
confirmed = await self.renew()
|
|
291
|
+
|
|
292
|
+
async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T:
|
|
293
|
+
self.remaining()
|
|
294
|
+
stopped = asyncio.Event()
|
|
295
|
+
execution = asyncio.ensure_future(operation())
|
|
296
|
+
maintenance = asyncio.create_task(self._maintain(stopped))
|
|
297
|
+
try:
|
|
298
|
+
while True:
|
|
299
|
+
done, _ = await asyncio.wait({execution, maintenance}, timeout=self.remaining(),
|
|
300
|
+
return_when=asyncio.FIRST_COMPLETED)
|
|
301
|
+
if maintenance in done:
|
|
302
|
+
await maintenance
|
|
303
|
+
raise ActivityLeaseAuthorityLost(self.task_id, "activity_renewal_stopped")
|
|
304
|
+
if execution in done:
|
|
305
|
+
result = await execution
|
|
306
|
+
self.remaining()
|
|
307
|
+
return result
|
|
308
|
+
except BaseException:
|
|
309
|
+
self.lost_reason = self.lost_reason or "activity_execution_abandoned"
|
|
310
|
+
raise
|
|
311
|
+
finally:
|
|
312
|
+
# An I/O wrapper may complete concurrently with cancel() and consume
|
|
313
|
+
# that cancellation. The settled scope must still stop renewing.
|
|
314
|
+
stopped.set()
|
|
315
|
+
# A Python thread may continue after cancellation. Its existing
|
|
316
|
+
# executor keeps the physical slot until it actually exits, while
|
|
317
|
+
# this context independently fences every late HTTP publication.
|
|
318
|
+
for future in (execution, maintenance):
|
|
319
|
+
future.cancel()
|
|
320
|
+
await asyncio.gather(execution, maintenance, return_exceptions=True)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
current_activity_authority: ContextVar[ActivityLeaseAuthority | None] = ContextVar(
|
|
324
|
+
"activity_lease_authority", default=None,
|
|
325
|
+
)
|
|
@@ -40,6 +40,10 @@ class CallbackInvocation:
|
|
|
40
40
|
info: ActivityInfo
|
|
41
41
|
task: dict[str, Any]
|
|
42
42
|
interceptors: tuple[WorkerInterceptor, ...] = ()
|
|
43
|
+
namespace: str | None = None
|
|
44
|
+
observability_supported: bool = False
|
|
45
|
+
remote: bool = False
|
|
46
|
+
execution_context: dict[str, Any] | None = None
|
|
43
47
|
|
|
44
48
|
def serialize(self) -> bytes:
|
|
45
49
|
try:
|
|
@@ -100,6 +104,8 @@ def _callback(directory: str, channel: socket.socket) -> None:
|
|
|
100
104
|
context = ActivityInterceptorContext(
|
|
101
105
|
worker_id=invocation.info.worker_id, task_queue=invocation.info.task_queue,
|
|
102
106
|
task=invocation.task, activity_type=invocation.info.activity_type, args=invocation.args,
|
|
107
|
+
namespace=invocation.namespace, observability_supported=invocation.observability_supported,
|
|
108
|
+
remote=invocation.remote, execution_context=invocation.execution_context,
|
|
103
109
|
)
|
|
104
110
|
|
|
105
111
|
async def call(ctx: ActivityInterceptorContext) -> Any:
|
|
@@ -7,7 +7,8 @@ from collections.abc import Awaitable, Callable
|
|
|
7
7
|
from dataclasses import dataclass
|
|
8
8
|
from typing import TYPE_CHECKING, Any
|
|
9
9
|
|
|
10
|
-
from .
|
|
10
|
+
from ._activity_lease import current_activity_authority
|
|
11
|
+
from .errors import ActivityCancelled, ActivityLeaseAuthorityLost
|
|
11
12
|
|
|
12
13
|
if TYPE_CHECKING:
|
|
13
14
|
from .client import Client
|
|
@@ -45,6 +46,7 @@ class ActivityContext:
|
|
|
45
46
|
self._client = client
|
|
46
47
|
self._heartbeat_callback = heartbeat_callback
|
|
47
48
|
self._cancel_requested = False
|
|
49
|
+
self._cannot_continue_reason: str | None = None
|
|
48
50
|
try:
|
|
49
51
|
self._event_loop: asyncio.AbstractEventLoop | None = asyncio.get_running_loop()
|
|
50
52
|
except RuntimeError:
|
|
@@ -61,7 +63,7 @@ class ActivityContext:
|
|
|
61
63
|
return self._cancel_requested
|
|
62
64
|
|
|
63
65
|
def heartbeat_sync(self, details: dict[str, Any] | None = None) -> None:
|
|
64
|
-
"""Heartbeat from a synchronous activity thread and
|
|
66
|
+
"""Heartbeat from a synchronous activity thread and check attempt authority.
|
|
65
67
|
|
|
66
68
|
The request runs on the worker's event loop using the original attempt
|
|
67
69
|
and client. Async handlers must use ``await heartbeat()`` instead.
|
|
@@ -84,7 +86,7 @@ class ActivityContext:
|
|
|
84
86
|
pending.result()
|
|
85
87
|
|
|
86
88
|
async def heartbeat(self, details: dict[str, Any] | None = None) -> None:
|
|
87
|
-
"""Report
|
|
89
|
+
"""Report progress and check cancellation and continued attempt authority.
|
|
88
90
|
|
|
89
91
|
Long-running activities should call ``heartbeat()`` periodically so the
|
|
90
92
|
server can distinguish a slow-but-alive attempt from a dead worker.
|
|
@@ -94,7 +96,12 @@ class ActivityContext:
|
|
|
94
96
|
Raises :class:`~durable_workflow.errors.ActivityCancelled` when the
|
|
95
97
|
owning workflow has requested cancellation, so the activity can exit
|
|
96
98
|
cleanly at its next natural break point.
|
|
99
|
+
|
|
100
|
+
Raises :class:`~durable_workflow.errors.ActivityLeaseAuthorityLost`
|
|
101
|
+
with the original Server reason when the attempt cannot continue,
|
|
102
|
+
including an elapsed deadline, closed attempt or closed run.
|
|
97
103
|
"""
|
|
104
|
+
self._raise_if_cannot_continue()
|
|
98
105
|
if self._heartbeat_callback is not None:
|
|
99
106
|
try:
|
|
100
107
|
await self._heartbeat_callback(details)
|
|
@@ -111,6 +118,21 @@ class ActivityContext:
|
|
|
111
118
|
if isinstance(resp, dict) and resp.get("cancel_requested"):
|
|
112
119
|
self._cancel_requested = True
|
|
113
120
|
raise ActivityCancelled()
|
|
121
|
+
if isinstance(resp, dict) and resp.get("can_continue") is False:
|
|
122
|
+
reason = resp.get("reason")
|
|
123
|
+
self._cannot_continue_reason = reason if isinstance(reason, str) and reason else "activity_cannot_continue"
|
|
124
|
+
self._raise_if_cannot_continue()
|
|
125
|
+
|
|
126
|
+
def _raise_if_cannot_continue(self) -> None:
|
|
127
|
+
if self._cannot_continue_reason is not None:
|
|
128
|
+
authority = current_activity_authority.get()
|
|
129
|
+
if (authority is not None and authority.client is self._client
|
|
130
|
+
and authority.task_id == self._info.task_id
|
|
131
|
+
and authority.identity["activity_attempt_id"] == self._info.activity_attempt_id
|
|
132
|
+
and authority.identity["attempt_number"] == self._info.attempt_number
|
|
133
|
+
and authority.identity["lease_owner"] == self._info.worker_id):
|
|
134
|
+
authority.lost_reason = authority.lost_reason or self._cannot_continue_reason
|
|
135
|
+
raise ActivityLeaseAuthorityLost(self._info.task_id, self._cannot_continue_reason)
|
|
114
136
|
|
|
115
137
|
|
|
116
138
|
def context() -> ActivityContext:
|