durable-workflow 2.10.0__tar.gz → 2.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {durable_workflow-2.10.0/src/durable_workflow.egg-info → durable_workflow-2.10.1}/PKG-INFO +1 -1
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/pyproject.toml +3 -3
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_metric_routes.py +1 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/client.py +40 -9
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/errors.py +15 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/worker.py +96 -3
- {durable_workflow-2.10.0 → durable_workflow-2.10.1/src/durable_workflow.egg-info}/PKG-INFO +1 -1
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/SOURCES.txt +1 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation_worker.py +8 -3
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_remote_worker.py +19 -5
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_regression_corpus.py +2 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_storage_admission.py +17 -3
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker.py +11 -8
- durable_workflow-2.10.1/tests/test_worker_deregistration.py +361 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_observation.py +12 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/LICENSE +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/README.md +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/setup.cfg +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/__init__.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_activity_lease.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_activity_process.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_avro.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cancellation_scope.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cancellation_scope_history.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_cooperative_cancellation.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_prepared_local_activity.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_query_task_authority.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/_sticky_workflow_cache.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/activity.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/auth_composition.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/cancellation.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_storage.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_task_input.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/external_task_result.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/history_bundle_verify.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/interceptors.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/invocable.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/metrics.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/nexus.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/py.typed +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/python_conformance.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/replay_conformance.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/replay_verify.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/retry_policy.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/serializer.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/sync.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/telemetry.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/telemetry_carrier.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/testing.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/worker_session.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/workflow.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow/workflow_updates_conformance.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/dependency_links.txt +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/entry_points.txt +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/requires.txt +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/top_level.txt +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_cancellation_policies.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_context.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_heartbeat_authority.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_authority.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_renewal.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_lease_worker.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_activity_process.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_auth_composition.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_avro_value_benchmark.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_avro_value_protocol.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_context.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_remaining_time.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_admission.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_authoring.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_boundary.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cancellation_scope_opening.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_child_workflow_policies.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_ci_checkout.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_ci_integration_endpoint.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client_telemetry.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_client_timeouts.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_codec_regression_corpus.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_descendants.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_groups.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_history.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_committed_cancellation_scope_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_condition_replay_drift.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_control_plane_parity_fixtures.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation_client.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_docstring_cross_references.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_durable_selection.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_errors.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_storage.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_task_input.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_external_task_result.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_golden_history_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_bundle_schema2.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_bundle_verify.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_history_event_contract.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_invocable.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_message_streams.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_metric_routes.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_metrics.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_durable_wait.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_namespace.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_nexus_wire.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_order_processing_example.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_parallel_completion_order_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_parallel_saga_authoring.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_patch_history.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_pending_operation_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_poll_failure_backoff.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_client.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_groups.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_policies.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_prepared_local_activity_worker.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_public_boundary_scanner.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_pypi_project_surface.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_python_conformance.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_queries.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_query_task_authority.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_query_timeout_retry.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_readme_quickstart.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_redrive_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_registration_ambiguity.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_release_metadata.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_clock.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_conformance.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_replay_verify.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_retry_policy.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_run_cancellation_scope_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_runtime_external_payload_transport.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_schedules.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_scoped_run_cancellation_context.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sdk_metrics_contract.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_serializer.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_signals.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sleep.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_standalone_activity_client.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sticky_execution.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sync.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_sync_activities.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_telemetry.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_telemetry_carrier.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_failures.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_harness.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_testing_nexus.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_threaded_heartbeat_example.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_tls_trust.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_transport_diagnostics.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_update_signal_condition_replay.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_updates.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_wait_condition.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_session.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_worker_telemetry_wire.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_entry_arguments.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_entrypoint_validation.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_handler_validation.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_identity.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_result_timeout.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_streams.py +0 -0
- {durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_workflow_updates_conformance.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "durable-workflow"
|
|
7
|
-
version = "2.10.
|
|
7
|
+
version = "2.10.1"
|
|
8
8
|
description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -76,8 +76,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
|
|
|
76
76
|
durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
|
|
77
77
|
|
|
78
78
|
[tool.durable-workflow]
|
|
79
|
-
product-train = "2.10.
|
|
80
|
-
registry-version = "2.10.
|
|
79
|
+
product-train = "2.10.1"
|
|
80
|
+
registry-version = "2.10.1"
|
|
81
81
|
supported-server-versions = "2.8.0"
|
|
82
82
|
worker-protocol-version = "1.20"
|
|
83
83
|
control-plane-version = "2"
|
|
@@ -12,6 +12,7 @@ _ROUTES = (
|
|
|
12
12
|
for suffix in ("", "/pass")),
|
|
13
13
|
"/workers", "/workers/{worker_id}",
|
|
14
14
|
"/worker/register", "/worker/heartbeat", "/worker/registrations/{worker_id}",
|
|
15
|
+
"/worker/registrations/{worker_id}/deregister",
|
|
15
16
|
"/worker/sessions", "/worker/sessions/{session_id}", "/worker/sessions/{session_id}/heartbeat",
|
|
16
17
|
*(f"/worker/{kind}-tasks/poll" for kind in ("workflow", "activity", "query", "update-validation")),
|
|
17
18
|
*(f"/worker/{kind}-tasks/{{task_id}}/{action}" for kind, actions in (
|
|
@@ -1853,6 +1853,7 @@ class Client:
|
|
|
1853
1853
|
json: Any = None,
|
|
1854
1854
|
timeout: float | None = None,
|
|
1855
1855
|
context: str = "",
|
|
1856
|
+
retry: bool = True,
|
|
1856
1857
|
) -> Any:
|
|
1857
1858
|
if worker and discovery:
|
|
1858
1859
|
raise ValueError("a request cannot be both worker-plane and cluster discovery")
|
|
@@ -1920,7 +1921,7 @@ class Client:
|
|
|
1920
1921
|
|
|
1921
1922
|
try:
|
|
1922
1923
|
try:
|
|
1923
|
-
resp = await self.retry_policy.execute(_do_request)
|
|
1924
|
+
resp = await self.retry_policy.execute(_do_request) if retry else await _do_request()
|
|
1924
1925
|
except httpx.HTTPStatusError as exc:
|
|
1925
1926
|
status_code = str(exc.response.status_code)
|
|
1926
1927
|
outcome = "http_error"
|
|
@@ -5245,39 +5246,69 @@ class Client:
|
|
|
5245
5246
|
json={"worker_id": worker_id, "reason": reason},
|
|
5246
5247
|
))
|
|
5247
5248
|
|
|
5248
|
-
async def deregister_worker_registration(
|
|
5249
|
+
async def deregister_worker_registration(
|
|
5250
|
+
self, worker_id: str, *, registration_token: str | None = None, timeout: float | None = None,
|
|
5251
|
+
expected_namespace: str | None = None,
|
|
5252
|
+
) -> dict[str, Any]:
|
|
5249
5253
|
"""Remove this runtime's successful worker-plane registration.
|
|
5250
5254
|
|
|
5251
5255
|
Called by :class:`~durable_workflow.Worker` after its pollers and
|
|
5252
5256
|
in-flight tasks have drained. This is distinct from the operator-only
|
|
5253
5257
|
:meth:`deregister_worker` control-plane management operation. Returns
|
|
5254
5258
|
the worker deregistration envelope, including recovered workflow-task
|
|
5255
|
-
count.
|
|
5259
|
+
count. With a registration token, use the runtime's generation-fenced
|
|
5260
|
+
receipt endpoint. Each call makes one attempt: Worker owns safe retries
|
|
5261
|
+
and the remaining shutdown budget. An unfenced DELETE cannot safely be
|
|
5262
|
+
repeated after an uncertain response because the worker ID may be reused.
|
|
5263
|
+
``expected_namespace`` is a local registration-scope precondition, not
|
|
5264
|
+
a namespace override.
|
|
5256
5265
|
"""
|
|
5266
|
+
if expected_namespace is not None and self.namespace != expected_namespace:
|
|
5267
|
+
raise RuntimeError("worker registration namespace changed; registration removal is unconfirmed")
|
|
5268
|
+
if registration_token is not None and (
|
|
5269
|
+
not isinstance(registration_token, str) or re.fullmatch(r"[a-f0-9]{32}", registration_token) is None
|
|
5270
|
+
):
|
|
5271
|
+
raise ValueError("registration_token must contain exactly 32 lowercase hexadecimal characters")
|
|
5272
|
+
fenced = registration_token is not None
|
|
5257
5273
|
try:
|
|
5258
5274
|
data = await self._request(
|
|
5259
|
-
"DELETE",
|
|
5260
|
-
f"/worker/registrations/{quote(worker_id, safe='')}",
|
|
5275
|
+
"POST" if fenced else "DELETE",
|
|
5276
|
+
f"/worker/registrations/{quote(worker_id, safe='')}" + ("/deregister" if fenced else ""),
|
|
5261
5277
|
worker=True,
|
|
5262
5278
|
context=worker_id,
|
|
5279
|
+
json={"registration_token": registration_token} if fenced else None,
|
|
5280
|
+
timeout=timeout,
|
|
5281
|
+
retry=False,
|
|
5263
5282
|
)
|
|
5264
5283
|
except ServerError as error:
|
|
5265
|
-
if error.status == 404 and error.reason() == "worker_not_found":
|
|
5284
|
+
if not fenced and error.status == 404 and error.reason() == "worker_not_found":
|
|
5266
5285
|
return {
|
|
5267
5286
|
"worker_id": worker_id,
|
|
5268
5287
|
"outcome": "already_deregistered",
|
|
5269
5288
|
"recovered_workflow_task_count": 0,
|
|
5270
5289
|
}
|
|
5271
5290
|
raise
|
|
5272
|
-
if not
|
|
5291
|
+
if not self._valid_worker_deregistration_response(data, worker_id, registration_token):
|
|
5273
5292
|
raise ServerError(
|
|
5274
5293
|
200,
|
|
5275
5294
|
{
|
|
5276
5295
|
"reason": "invalid_worker_deregistration_response",
|
|
5277
|
-
"message":
|
|
5296
|
+
"message": "expected a deregistration receipt for this worker registration",
|
|
5278
5297
|
},
|
|
5279
5298
|
)
|
|
5280
|
-
return data
|
|
5299
|
+
return cast(dict[str, Any], data)
|
|
5300
|
+
|
|
5301
|
+
@staticmethod
|
|
5302
|
+
def _valid_worker_deregistration_response(data: Any, worker_id: str, token: str | None) -> bool:
|
|
5303
|
+
if not isinstance(data, dict):
|
|
5304
|
+
return False
|
|
5305
|
+
count = data.get("recovered_workflow_task_count")
|
|
5306
|
+
return (
|
|
5307
|
+
data.get("worker_id") == worker_id
|
|
5308
|
+
and data.get("outcome") == "deregistered"
|
|
5309
|
+
and type(count) is int and count >= 0
|
|
5310
|
+
and (token is None or data.get("registration_token") == token)
|
|
5311
|
+
)
|
|
5281
5312
|
|
|
5282
5313
|
async def heartbeat_worker(
|
|
5283
5314
|
self,
|
|
@@ -219,6 +219,21 @@ class ServerError(DurableWorkflowError):
|
|
|
219
219
|
return self.body.get("reason")
|
|
220
220
|
return None
|
|
221
221
|
|
|
222
|
+
def worker_deregistration_retry_delay(self, worker_id: str, registration_token: str) -> int | None:
|
|
223
|
+
"""Return a delay only for an identity-preserving fenced transient."""
|
|
224
|
+
body = self.body
|
|
225
|
+
if (
|
|
226
|
+
self.status != 503 or not isinstance(body, dict)
|
|
227
|
+
or body.get("reason") not in ("backend_lock_pressure", "backend_unavailable")
|
|
228
|
+
or body.get("operation") != "deregister_worker" or body.get("outcome") != "unknown"
|
|
229
|
+
or body.get("worker_id") != worker_id or body.get("registration_token") != registration_token
|
|
230
|
+
or body.get("retryable") is not True
|
|
231
|
+
or type(body.get("retry_after_seconds")) is not int
|
|
232
|
+
or not 0 < body["retry_after_seconds"] <= 60
|
|
233
|
+
):
|
|
234
|
+
return None
|
|
235
|
+
return int(body["retry_after_seconds"])
|
|
236
|
+
|
|
222
237
|
def poll_capacity_backpressure_delay(self, task_kind: str, task_queue: str) -> int | None:
|
|
223
238
|
"""Return the advertised wait only for a complete poll-capacity refusal."""
|
|
224
239
|
body = self.body
|
|
@@ -37,6 +37,8 @@ from functools import wraps
|
|
|
37
37
|
from types import FunctionType
|
|
38
38
|
from typing import Annotated, Any, Concatenate, Literal, ParamSpec, TypeVar, Union, get_args, get_origin, get_type_hints
|
|
39
39
|
|
|
40
|
+
import httpx
|
|
41
|
+
|
|
40
42
|
from . import serializer
|
|
41
43
|
from ._activity_lease import activity_lease_renewal_supported, current_activity_authority
|
|
42
44
|
from ._activity_process import CallbackFailure, CallbackInvocation, CallbackProcessLost, SupervisedCallback
|
|
@@ -1169,6 +1171,10 @@ class Worker:
|
|
|
1169
1171
|
self._in_flight: set[asyncio.Task[Any]] = set()
|
|
1170
1172
|
self._poller_tasks: set[asyncio.Task[Any]] = set()
|
|
1171
1173
|
self._registered = False
|
|
1174
|
+
self._deregistration_fenced = False
|
|
1175
|
+
self._registration_token: str | None = None
|
|
1176
|
+
self._registration_worker_id: str | None = None
|
|
1177
|
+
self._registration_namespace: object = None
|
|
1172
1178
|
self._run_started = False
|
|
1173
1179
|
self._registration_done = asyncio.Event()
|
|
1174
1180
|
self._shutdown_task: asyncio.Task[None] | None = None
|
|
@@ -1295,6 +1301,18 @@ class Worker:
|
|
|
1295
1301
|
_validate_server_compatibility(info)
|
|
1296
1302
|
protocol = info.get("worker_protocol")
|
|
1297
1303
|
server_capabilities = protocol.get("server_capabilities") if isinstance(protocol, Mapping) else None
|
|
1304
|
+
deregistration = (
|
|
1305
|
+
server_capabilities.get("worker_deregistration_fencing")
|
|
1306
|
+
if isinstance(server_capabilities, Mapping) else None
|
|
1307
|
+
)
|
|
1308
|
+
self._deregistration_fenced = isinstance(deregistration, Mapping) and deregistration.get("supported") is True
|
|
1309
|
+
if isinstance(deregistration, Mapping) and self._deregistration_fenced and (
|
|
1310
|
+
deregistration.get("schema") != "durable-workflow.v2.worker-deregistration.v1"
|
|
1311
|
+
or deregistration.get("endpoint") != "/worker/registrations/{workerId}/deregister"
|
|
1312
|
+
or type(deregistration.get("receipt_retention_seconds")) is not int
|
|
1313
|
+
or deregistration["receipt_retention_seconds"] <= 0
|
|
1314
|
+
):
|
|
1315
|
+
raise RuntimeError("Server compatibility error: invalid worker_deregistration_fencing contract")
|
|
1298
1316
|
sticky_support = (
|
|
1299
1317
|
server_capabilities.get("sticky_execution") if isinstance(server_capabilities, Mapping) else None
|
|
1300
1318
|
)
|
|
@@ -1449,6 +1467,20 @@ class Worker:
|
|
|
1449
1467
|
task_slots=self._current_task_slots(),
|
|
1450
1468
|
process_metrics=self._current_process_metrics(),
|
|
1451
1469
|
)
|
|
1470
|
+
if self._deregistration_fenced:
|
|
1471
|
+
token = ack.get("registration_token") if isinstance(ack, dict) else None
|
|
1472
|
+
if (
|
|
1473
|
+
not isinstance(token, str) or len(token) != 32
|
|
1474
|
+
or any(character not in "0123456789abcdef" for character in token)
|
|
1475
|
+
or ack.get("worker_id") != self.worker_id or ack.get("registered") is not True
|
|
1476
|
+
):
|
|
1477
|
+
raise RuntimeError(
|
|
1478
|
+
"Server returned an invalid registration_token acknowledgement; "
|
|
1479
|
+
"registration may remain active and unfenced cleanup is unsafe"
|
|
1480
|
+
)
|
|
1481
|
+
self._registration_token = token
|
|
1482
|
+
self._registration_worker_id = self.worker_id
|
|
1483
|
+
self._registration_namespace = getattr(self.client, "namespace", None)
|
|
1452
1484
|
self._registered = True
|
|
1453
1485
|
# Adapt to the server-advertised cadence when present so a cluster
|
|
1454
1486
|
# can pin the worker fleet's heartbeat beat without each worker
|
|
@@ -4567,8 +4599,9 @@ class Worker:
|
|
|
4567
4599
|
async def stop(self) -> None:
|
|
4568
4600
|
"""Stop polling, drain tasks, and remove the worker registration.
|
|
4569
4601
|
|
|
4570
|
-
Repeated calls share one shutdown result.
|
|
4571
|
-
|
|
4602
|
+
Repeated calls share one shutdown result. Generation-fenced recovery
|
|
4603
|
+
stays within the original shutdown deadline. Unfenced deregistration
|
|
4604
|
+
makes one attempt and does not retry an uncertain outcome.
|
|
4572
4605
|
"""
|
|
4573
4606
|
self._stop.set()
|
|
4574
4607
|
if self._shutdown_task is None:
|
|
@@ -4664,8 +4697,9 @@ class Worker:
|
|
|
4664
4697
|
log.warning("worker session %s close failed: %s", session.options.session_id, error)
|
|
4665
4698
|
|
|
4666
4699
|
if self._registered:
|
|
4667
|
-
await self.
|
|
4700
|
+
await self._deregister_before_deadline(deadline)
|
|
4668
4701
|
self._registered = False
|
|
4702
|
+
self._registration_token = None
|
|
4669
4703
|
log.info("worker %s deregistered", self.worker_id)
|
|
4670
4704
|
self._sticky_cache.clear()
|
|
4671
4705
|
payload_observers = getattr(self.client, "_workflow_payload_observers", None)
|
|
@@ -4676,3 +4710,62 @@ class Worker:
|
|
|
4676
4710
|
@staticmethod
|
|
4677
4711
|
def _remaining_shutdown_time(deadline: float) -> float:
|
|
4678
4712
|
return max(0.0, deadline - asyncio.get_running_loop().time())
|
|
4713
|
+
|
|
4714
|
+
def _check_deregistration_identity(self) -> None:
|
|
4715
|
+
if self._deregistration_fenced and self._registration_token is None:
|
|
4716
|
+
raise RuntimeError("worker registration token is missing; registration removal is unconfirmed")
|
|
4717
|
+
if self._registration_worker_id is not None and (
|
|
4718
|
+
self.worker_id != self._registration_worker_id
|
|
4719
|
+
or getattr(self.client, "namespace", None) != self._registration_namespace
|
|
4720
|
+
):
|
|
4721
|
+
raise RuntimeError("worker registration identity changed; registration removal is unconfirmed")
|
|
4722
|
+
|
|
4723
|
+
async def _deregistration_attempt(self, remaining: float) -> tuple[float | None, Exception] | None:
|
|
4724
|
+
kwargs: dict[str, Any] = {}
|
|
4725
|
+
if self._deregistration_fenced:
|
|
4726
|
+
kwargs = {"registration_token": self._registration_token, "timeout": remaining}
|
|
4727
|
+
if isinstance(self._registration_namespace, str):
|
|
4728
|
+
kwargs["expected_namespace"] = self._registration_namespace
|
|
4729
|
+
try:
|
|
4730
|
+
await asyncio.wait_for(
|
|
4731
|
+
self.client.deregister_worker_registration(self.worker_id, **kwargs),
|
|
4732
|
+
timeout=remaining,
|
|
4733
|
+
)
|
|
4734
|
+
except asyncio.TimeoutError as error:
|
|
4735
|
+
raise RuntimeError(
|
|
4736
|
+
"worker shutdown timed out during deregistration; registration removal is unconfirmed"
|
|
4737
|
+
) from error
|
|
4738
|
+
except ServerError as error:
|
|
4739
|
+
retry_delay = (
|
|
4740
|
+
error.worker_deregistration_retry_delay(self.worker_id, self._registration_token)
|
|
4741
|
+
if self._deregistration_fenced and self._registration_token is not None else None
|
|
4742
|
+
)
|
|
4743
|
+
if retry_delay is None:
|
|
4744
|
+
raise
|
|
4745
|
+
return float(retry_delay), error
|
|
4746
|
+
except (httpx.NetworkError, httpx.TimeoutException, httpx.RemoteProtocolError) as error:
|
|
4747
|
+
if not self._deregistration_fenced:
|
|
4748
|
+
raise
|
|
4749
|
+
return None, error
|
|
4750
|
+
return None
|
|
4751
|
+
|
|
4752
|
+
async def _deregister_before_deadline(self, deadline: float) -> None:
|
|
4753
|
+
attempt = 0
|
|
4754
|
+
while True:
|
|
4755
|
+
self._check_deregistration_identity()
|
|
4756
|
+
remaining = self._remaining_shutdown_time(deadline)
|
|
4757
|
+
if remaining <= 0:
|
|
4758
|
+
raise RuntimeError("worker shutdown deadline exhausted; registration removal is unconfirmed")
|
|
4759
|
+
retry = await self._deregistration_attempt(remaining)
|
|
4760
|
+
if retry is None:
|
|
4761
|
+
return
|
|
4762
|
+
retry_delay, failure = retry
|
|
4763
|
+
delay = min(1.0, 0.1 * 2 ** min(attempt, 4)) if retry_delay is None else retry_delay
|
|
4764
|
+
if delay >= self._remaining_shutdown_time(deadline):
|
|
4765
|
+
raise RuntimeError(
|
|
4766
|
+
"worker shutdown cannot retry within its deadline; registration removal is unconfirmed"
|
|
4767
|
+
) from failure
|
|
4768
|
+
if attempt == 0:
|
|
4769
|
+
log.warning("worker deregistration is unconfirmed; retrying the fenced request within its deadline")
|
|
4770
|
+
attempt += 1
|
|
4771
|
+
await asyncio.sleep(delay)
|
{durable_workflow-2.10.0 → durable_workflow-2.10.1}/src/durable_workflow.egg-info/SOURCES.txt
RENAMED
|
@@ -146,6 +146,7 @@ tests/test_update_signal_condition_replay.py
|
|
|
146
146
|
tests/test_updates.py
|
|
147
147
|
tests/test_wait_condition.py
|
|
148
148
|
tests/test_worker.py
|
|
149
|
+
tests/test_worker_deregistration.py
|
|
149
150
|
tests/test_worker_observation.py
|
|
150
151
|
tests/test_worker_session.py
|
|
151
152
|
tests/test_worker_telemetry_wire.py
|
{durable_workflow-2.10.0 → durable_workflow-2.10.1}/tests/test_cooperative_cancellation_worker.py
RENAMED
|
@@ -607,8 +607,12 @@ async def test_shutdown_timeout_abandons_cleanup_and_replacement_replays_origina
|
|
|
607
607
|
await asyncio.wait_for(entered.wait(), timeout=1)
|
|
608
608
|
original_history = deepcopy(server.history)
|
|
609
609
|
heartbeats_before_stop = server.client.heartbeat_workflow_task.await_count
|
|
610
|
-
|
|
611
|
-
|
|
610
|
+
stopping = worker.stop()
|
|
611
|
+
try:
|
|
612
|
+
with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
|
|
613
|
+
await asyncio.wait_for(stopping, timeout=1)
|
|
614
|
+
finally:
|
|
615
|
+
release_thread.set()
|
|
612
616
|
await asyncio.wait_for(discarded.wait(), timeout=1)
|
|
613
617
|
assert fenced_heartbeats == [True]
|
|
614
618
|
assert server.client.heartbeat_workflow_task.await_count == heartbeats_before_stop
|
|
@@ -616,7 +620,8 @@ async def test_shutdown_timeout_abandons_cleanup_and_replacement_replays_origina
|
|
|
616
620
|
assert server.history == original_history
|
|
617
621
|
server.client.complete_workflow_task.assert_not_awaited()
|
|
618
622
|
server.client.fail_workflow_task.assert_not_awaited()
|
|
619
|
-
|
|
623
|
+
assert worker._registered
|
|
624
|
+
server.client.deregister_worker_registration.assert_not_awaited()
|
|
620
625
|
|
|
621
626
|
replacement = ClaimServer(history=original_history, lease_owner="replacement-worker", workflow_task_attempt=5)
|
|
622
627
|
cleanup_ids: list[str] = []
|
|
@@ -92,8 +92,7 @@ async def application_failure(kind: str) -> None:
|
|
|
92
92
|
raise ActivityCancelled()
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def]
|
|
95
|
+
async def make_owner(monkeypatch: pytest.MonkeyPatch) -> tuple[Worker, AsyncMock]:
|
|
97
96
|
monkeypatch.delenv("DURABLE_WORKFLOW_WORKER_PROTOCOL_VERSION", raising=False)
|
|
98
97
|
client = AsyncMock(spec=Client)
|
|
99
98
|
client.get_cluster_info.return_value = compatible_cluster_info(worker_protocol={
|
|
@@ -106,6 +105,12 @@ async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def
|
|
|
106
105
|
worker = Worker(client, task_queue="queue", worker_id="owner", capabilities=["cooperative_cancellation"],
|
|
107
106
|
max_concurrent_activity_tasks=1)
|
|
108
107
|
await worker._register()
|
|
108
|
+
return worker, client
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@pytest_asyncio.fixture
|
|
112
|
+
async def owner(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def]
|
|
113
|
+
worker, client = await make_owner(monkeypatch)
|
|
109
114
|
try:
|
|
110
115
|
yield worker, client
|
|
111
116
|
finally:
|
|
@@ -152,18 +157,27 @@ async def test_authority_loss_stops_and_joins_the_actual_callback_before_reporti
|
|
|
152
157
|
client.fail_activity_task.assert_not_awaited()
|
|
153
158
|
|
|
154
159
|
|
|
155
|
-
async def test_shutdown_timeout_joins_the_remote_callback_before_returning(
|
|
156
|
-
|
|
160
|
+
async def test_shutdown_timeout_joins_the_remote_callback_before_returning(
|
|
161
|
+
monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
|
|
162
|
+
) -> None:
|
|
163
|
+
worker, client = await make_owner(monkeypatch)
|
|
157
164
|
worker._shutdown_timeout = 0.01
|
|
158
165
|
marker = tmp_path / "callback"
|
|
159
166
|
worker.activities["remote"] = blocked_async
|
|
160
167
|
execution = worker._track(worker._run_activity_task(task(str(marker))))
|
|
161
168
|
await wait_for_file(marker)
|
|
162
169
|
pid = int(marker.read_text())
|
|
163
|
-
|
|
170
|
+
stopping = worker.stop()
|
|
171
|
+
with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed") as failed:
|
|
172
|
+
await asyncio.wait_for(stopping, timeout=4.0)
|
|
164
173
|
await wait_for_exit(pid)
|
|
165
174
|
assert execution.cancelled() or execution.result() == "claim_aborted"
|
|
166
175
|
assert not worker._remote_activity_processes
|
|
176
|
+
assert worker._registered
|
|
177
|
+
client.deregister_worker_registration.assert_not_awaited()
|
|
178
|
+
with pytest.raises(RuntimeError) as repeated:
|
|
179
|
+
await worker.stop()
|
|
180
|
+
assert repeated.value is failed.value
|
|
167
181
|
client.complete_activity_task.assert_not_awaited()
|
|
168
182
|
client.fail_activity_task.assert_not_awaited()
|
|
169
183
|
|
|
@@ -25,6 +25,7 @@ from durable_workflow.workflow import (
|
|
|
25
25
|
commands_to_server_commands,
|
|
26
26
|
query_state,
|
|
27
27
|
)
|
|
28
|
+
from tests.deregistration_recovery import RecoveredWorkflow as FencedRecoveryWorkflow
|
|
28
29
|
from tests.test_condition_replay_drift import ConditionIdentityWorkflow
|
|
29
30
|
from tests.test_golden_history_replay import (
|
|
30
31
|
GoldenSagaCompensationWorkflow,
|
|
@@ -371,6 +372,7 @@ WORKFLOWS = [
|
|
|
371
372
|
RepeatedPatchWorkflow,
|
|
372
373
|
PendingOperation,
|
|
373
374
|
RecoveredWorkflow,
|
|
375
|
+
FencedRecoveryWorkflow,
|
|
374
376
|
PagedRecordedSideEffectsWorkflow,
|
|
375
377
|
PostConditionReceiversWorkflow,
|
|
376
378
|
RecordedSideEffectWorkflow,
|
|
@@ -8,6 +8,7 @@ from contextlib import contextmanager
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from types import SimpleNamespace
|
|
10
10
|
from typing import Any
|
|
11
|
+
from urllib.parse import unquote
|
|
11
12
|
|
|
12
13
|
import httpx
|
|
13
14
|
import pytest
|
|
@@ -544,7 +545,12 @@ async def test_running_worker_preserves_activity_outcome_without_reexecution(
|
|
|
544
545
|
info = compatible_cluster_info()
|
|
545
546
|
info["worker_protocol"]["server_capabilities"]["query_tasks"] = False
|
|
546
547
|
return httpx.Response(200, json=info)
|
|
547
|
-
if
|
|
548
|
+
if request.method == "DELETE":
|
|
549
|
+
return httpx.Response(200, json={
|
|
550
|
+
"worker_id": unquote(path.rsplit("/", 1)[-1]),
|
|
551
|
+
"outcome": "deregistered", "recovered_workflow_task_count": 0,
|
|
552
|
+
})
|
|
553
|
+
if path.endswith("/register"):
|
|
548
554
|
return httpx.Response(200, json={"registered": True})
|
|
549
555
|
if path.endswith("/activity-tasks/poll"):
|
|
550
556
|
claims += 1
|
|
@@ -656,7 +662,10 @@ async def test_run_until_timeout_interrupts_storage_pressure(phase: str) -> None
|
|
|
656
662
|
return httpx.Response(200, json=info)
|
|
657
663
|
if request.method == "DELETE":
|
|
658
664
|
deregistered += 1
|
|
659
|
-
return httpx.Response(200, json={
|
|
665
|
+
return httpx.Response(200, json={
|
|
666
|
+
"worker_id": unquote(path.rsplit("/", 1)[-1]),
|
|
667
|
+
"outcome": "deregistered", "recovered_workflow_task_count": 0,
|
|
668
|
+
})
|
|
660
669
|
if (
|
|
661
670
|
(phase == "registration" and path.endswith("/register"))
|
|
662
671
|
or (phase == "poll" and path.endswith("/poll"))
|
|
@@ -813,7 +822,12 @@ async def test_cold_replay_completion_retries_only_its_prepared_acknowledgement(
|
|
|
813
822
|
info = compatible_cluster_info()
|
|
814
823
|
info["worker_protocol"]["server_capabilities"]["query_tasks"] = False
|
|
815
824
|
return httpx.Response(200, json=info)
|
|
816
|
-
if
|
|
825
|
+
if request.method == "DELETE":
|
|
826
|
+
return httpx.Response(200, json={
|
|
827
|
+
"worker_id": unquote(path.rsplit("/", 1)[-1]),
|
|
828
|
+
"outcome": "deregistered", "recovered_workflow_task_count": 0,
|
|
829
|
+
})
|
|
830
|
+
if path.endswith("/register"):
|
|
817
831
|
return httpx.Response(200, json={})
|
|
818
832
|
if path.endswith("/workflows/cold-replay"):
|
|
819
833
|
return httpx.Response(200, json={"status": "completed" if completed else "running"})
|
|
@@ -3226,7 +3226,7 @@ class TestUpdateValidationTaskExecution:
|
|
|
3226
3226
|
assert worker._current_task_slots()["workflow_available"] == 1
|
|
3227
3227
|
|
|
3228
3228
|
@pytest.mark.asyncio
|
|
3229
|
-
async def
|
|
3229
|
+
async def test_shutdown_releases_validation_capacity_without_deregistering_after_deadline(
|
|
3230
3230
|
self, mock_client: AsyncMock
|
|
3231
3231
|
) -> None:
|
|
3232
3232
|
validation_started = asyncio.Event()
|
|
@@ -3279,15 +3279,16 @@ class TestUpdateValidationTaskExecution:
|
|
|
3279
3279
|
await asyncio.wait_for(validation_started.wait(), timeout=1.0)
|
|
3280
3280
|
assert worker._current_task_slots()["workflow_available"] == 0
|
|
3281
3281
|
|
|
3282
|
-
|
|
3282
|
+
with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
|
|
3283
|
+
await worker.stop()
|
|
3283
3284
|
|
|
3284
3285
|
assert validation_cancelled.is_set()
|
|
3285
3286
|
assert worker._workflow_inflight == 0
|
|
3286
3287
|
assert worker._workflow_reserved == 0
|
|
3287
3288
|
assert worker._wf_semaphore._value == 1
|
|
3288
3289
|
assert worker._current_task_slots()["workflow_available"] == 1
|
|
3289
|
-
assert worker._registered is
|
|
3290
|
-
mock_client.deregister_worker_registration.
|
|
3290
|
+
assert worker._registered is True
|
|
3291
|
+
mock_client.deregister_worker_registration.assert_not_awaited()
|
|
3291
3292
|
|
|
3292
3293
|
@pytest.mark.asyncio
|
|
3293
3294
|
async def test_cancellation_before_dispatch_starts_releases_admitted_capacity(self, mock_client: AsyncMock) -> None:
|
|
@@ -4516,7 +4517,7 @@ class TestWorkerShutdown:
|
|
|
4516
4517
|
mock_client.deregister_worker_registration.assert_awaited_once_with(worker.worker_id)
|
|
4517
4518
|
|
|
4518
4519
|
@pytest.mark.asyncio
|
|
4519
|
-
async def
|
|
4520
|
+
async def test_shutdown_deadline_cancels_work_without_sending_late_deregistration(
|
|
4520
4521
|
self, mock_client: AsyncMock
|
|
4521
4522
|
) -> None:
|
|
4522
4523
|
events: list[str] = []
|
|
@@ -4539,11 +4540,13 @@ class TestWorkerShutdown:
|
|
|
4539
4540
|
tracked_task = worker._track(accepted_work())
|
|
4540
4541
|
await work_started.wait()
|
|
4541
4542
|
|
|
4542
|
-
|
|
4543
|
+
with pytest.raises(RuntimeError, match="deadline exhausted.*registration.*unconfirmed"):
|
|
4544
|
+
await worker.stop()
|
|
4543
4545
|
|
|
4544
4546
|
assert tracked_task.cancelled()
|
|
4545
|
-
assert events == ["work_cancelled"
|
|
4546
|
-
|
|
4547
|
+
assert events == ["work_cancelled"]
|
|
4548
|
+
assert worker._registered is True
|
|
4549
|
+
mock_client.deregister_worker_registration.assert_not_awaited()
|
|
4547
4550
|
|
|
4548
4551
|
@pytest.mark.asyncio
|
|
4549
4552
|
async def test_failed_registration_does_not_deregister(self, mock_client: AsyncMock) -> None:
|