aws-durable-execution-sdk-python 2.0.0__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/PKG-INFO +1 -1
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/pyproject.toml +1 -1
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/__about__.py +1 -1
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/concurrency/executor.py +43 -15
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/exceptions.py +2 -2
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/state.py +316 -136
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/concurrency_test.py +180 -10
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/map_with_concurrent_waits_int_test.py +37 -52
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/exceptions_test.py +2 -2
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/state_test.py +762 -223
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/.gitignore +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/LICENSE +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/NOTICE +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/README.md +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/.gitignore +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/concurrency/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/concurrency/models.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/config.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/constants.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/context.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/execution.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/filesystem_serdes.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/identifier.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/lambda_service.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/logger.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/base.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/callback.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/child.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/invoke.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/map.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/parallel.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/step.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/wait.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/operation/wait_for_condition.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/plugin.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/plugin_discovery.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/preview.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/py.typed +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/retries.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/serdes.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/suspend.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/threading.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/types.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/src/aws_durable_execution_sdk_python/waits.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/config_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/context_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/durable_executions_python_language_sdk_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/checkpoint_response_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/custom_serdes_roundtrip_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/error_hierarchy_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/execution_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/filesystem_serdes_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/plugin_invocation_operations_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/plugin_invocation_payload_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/plugin_user_function_lifecycle_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/e2e/should_complete_suspend_resume_int_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/execution_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/filesystem_serdes_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/lambda_service_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/logger_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/__init__.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/base_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/callback_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/child_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/invoke_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/map_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/parallel_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/step_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/wait_for_condition_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/operation/wait_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/plugin_discovery_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/plugin_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/preview_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/retries_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/serdes_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/suspend_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/test_helpers.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/threading_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/types_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/waits_test.py +0 -0
- {aws_durable_execution_sdk_python-2.0.0 → aws_durable_execution_sdk_python-2.0.1}/tests/with_retry_test.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: aws-durable-execution-sdk-python
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: AWS Durable Execution SDK for Python
|
|
5
5
|
Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
|
|
@@ -8,6 +8,7 @@ import queue
|
|
|
8
8
|
import time
|
|
9
9
|
from collections import deque
|
|
10
10
|
from concurrent.futures import ThreadPoolExecutor
|
|
11
|
+
from dataclasses import dataclass, field
|
|
11
12
|
from typing import TYPE_CHECKING, Generic, TypeVar, cast
|
|
12
13
|
|
|
13
14
|
from aws_durable_execution_sdk_python.concurrency.models import (
|
|
@@ -56,6 +57,7 @@ if TYPE_CHECKING:
|
|
|
56
57
|
from aws_durable_execution_sdk_python.state import (
|
|
57
58
|
CheckpointedResult,
|
|
58
59
|
ExecutionState,
|
|
60
|
+
ScheduledRefresh,
|
|
59
61
|
)
|
|
60
62
|
|
|
61
63
|
|
|
@@ -86,6 +88,20 @@ def _branch_error_object(err: Exception) -> ErrorObject:
|
|
|
86
88
|
return ErrorObject.from_exception(err)
|
|
87
89
|
|
|
88
90
|
|
|
91
|
+
@dataclass
|
|
92
|
+
class ResumeWave(Generic[CallableType, ResultType]):
|
|
93
|
+
"""Branches suspended until one time, and the refresh that resumes them.
|
|
94
|
+
|
|
95
|
+
The refresh is a delayed empty checkpoint requested when the first branch
|
|
96
|
+
suspends. Its response shows the waits complete, so the wave resumes one
|
|
97
|
+
round trip after its time.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
resume_at: float
|
|
101
|
+
refresh: ScheduledRefresh
|
|
102
|
+
branches: list[Branch[CallableType, ResultType]] = field(default_factory=list)
|
|
103
|
+
|
|
104
|
+
|
|
89
105
|
class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
90
106
|
"""Execute durable operations concurrently. This contains the execution logic for Map and Parallel.
|
|
91
107
|
|
|
@@ -252,7 +268,9 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
252
268
|
|
|
253
269
|
events: queue.Queue[BranchEvent[ResultType]] = queue.Queue()
|
|
254
270
|
pending: deque[Branch[CallableType, ResultType]] = deque(self.branches)
|
|
255
|
-
|
|
271
|
+
# The heap orders resume times. The dict groups the branches under each.
|
|
272
|
+
resume_times: list[float] = []
|
|
273
|
+
waves: dict[float, ResumeWave[CallableType, ResultType]] = {}
|
|
256
274
|
branch_by_index: dict[int, Branch[CallableType, ResultType]] = {
|
|
257
275
|
branch.index: branch for branch in self.branches
|
|
258
276
|
}
|
|
@@ -317,19 +335,20 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
317
335
|
running += 1
|
|
318
336
|
needs_snapshot_rebuild = True
|
|
319
337
|
|
|
320
|
-
# Resume due
|
|
321
|
-
# refresh serves the whole due wave; a failure is terminal
|
|
338
|
+
# Resume due waves in-process. A refresh failure is terminal
|
|
322
339
|
# for the execution and propagates from this thread.
|
|
323
340
|
now: float = time.time()
|
|
324
|
-
|
|
325
|
-
while
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
for branch in
|
|
341
|
+
resumed = False
|
|
342
|
+
while resume_times and resume_times[0] <= now:
|
|
343
|
+
wave = waves.pop(heapq.heappop(resume_times))
|
|
344
|
+
wave.refresh.wait()
|
|
345
|
+
# Branches joined the wave in event order. Resume in index
|
|
346
|
+
# order so scheduling stays deterministic.
|
|
347
|
+
for branch in sorted(wave.branches, key=lambda b: b.index):
|
|
331
348
|
submit(branch)
|
|
332
349
|
running += 1
|
|
350
|
+
resumed = True
|
|
351
|
+
if resumed:
|
|
333
352
|
continue
|
|
334
353
|
|
|
335
354
|
if running == 0:
|
|
@@ -340,18 +359,18 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
340
359
|
raise retryable_error
|
|
341
360
|
# Every in-flight branch is suspended and no slot is
|
|
342
361
|
# free (or no work remains): suspend the parent.
|
|
343
|
-
if
|
|
362
|
+
if resume_times:
|
|
344
363
|
raise TimedSuspendExecution(
|
|
345
364
|
"All concurrent work complete or suspended pending retry.",
|
|
346
|
-
|
|
365
|
+
resume_times[0],
|
|
347
366
|
)
|
|
348
367
|
raise SuspendExecution(
|
|
349
368
|
"All concurrent work complete or suspended and pending external callback."
|
|
350
369
|
)
|
|
351
370
|
|
|
352
371
|
timeout: float | None = None
|
|
353
|
-
if
|
|
354
|
-
timeout = max(
|
|
372
|
+
if resume_times:
|
|
373
|
+
timeout = max(resume_times[0] - time.time(), 0)
|
|
355
374
|
try:
|
|
356
375
|
event: BranchEvent[ResultType] = events.get(timeout=timeout)
|
|
357
376
|
except queue.Empty:
|
|
@@ -383,7 +402,13 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
383
402
|
needs_snapshot_rebuild = True
|
|
384
403
|
case BranchEventKind.SUSPENDED_UNTIL if event.resume_at is not None:
|
|
385
404
|
applied.suspend_until(event.resume_at)
|
|
386
|
-
|
|
405
|
+
if event.resume_at not in waves:
|
|
406
|
+
waves[event.resume_at] = ResumeWave(
|
|
407
|
+
event.resume_at,
|
|
408
|
+
execution_state.schedule_refresh(event.resume_at),
|
|
409
|
+
)
|
|
410
|
+
heapq.heappush(resume_times, event.resume_at)
|
|
411
|
+
waves[event.resume_at].branches.append(applied)
|
|
387
412
|
running -= 1
|
|
388
413
|
needs_snapshot_rebuild = True
|
|
389
414
|
case BranchEventKind.ORPHANED:
|
|
@@ -402,6 +427,9 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
402
427
|
msg = f"Unhandled branch event: {event}"
|
|
403
428
|
raise InvalidStateError(msg)
|
|
404
429
|
finally:
|
|
430
|
+
# Nothing will wait for a refresh whose wave never resumed.
|
|
431
|
+
for wave in waves.values():
|
|
432
|
+
wave.refresh.cancel()
|
|
405
433
|
# Shutdown without waiting for running threads for early return
|
|
406
434
|
# when completion criteria are met (e.g., min_successful).
|
|
407
435
|
# Running threads continue in the background of this invocation
|
|
@@ -13,7 +13,7 @@ BAD_REQUEST_ERROR: int = 400
|
|
|
13
13
|
TOO_MANY_REQUESTS_ERROR: int = 429
|
|
14
14
|
SERVICE_ERROR: int = 500
|
|
15
15
|
INVALID_PARAMETER_VALUE_EXCEPTION: str = "InvalidParameterValueException"
|
|
16
|
-
INVALID_CHECKPOINT_TOKEN_PREFIX: str = "Invalid
|
|
16
|
+
INVALID_CHECKPOINT_TOKEN_PREFIX: str = "Invalid checkpoint token"
|
|
17
17
|
|
|
18
18
|
# Non-retryable customer error codes that arrive as non-4xx (e.g. HTTP 502) from Lambda.
|
|
19
19
|
# Unlike typical 5xx errors, these require customer intervention (e.g., fixing
|
|
@@ -162,7 +162,7 @@ class BotoClientError(InvocationError):
|
|
|
162
162
|
These arrive as HTTP 502 but require customer intervention to fix.
|
|
163
163
|
- 4xx errors → EXECUTION, except:
|
|
164
164
|
- 429 (TooManyRequests) → INVOCATION (throttling is transient)
|
|
165
|
-
- InvalidParameterValueException with "Invalid
|
|
165
|
+
- InvalidParameterValueException with "Invalid checkpoint token" (exact match) → INVOCATION
|
|
166
166
|
(stale token from a concurrent checkpoint; next invocation gets a fresh token)
|
|
167
167
|
- 5xx, network errors → INVOCATION
|
|
168
168
|
"""
|