aws-durable-execution-sdk-python 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
1
  # SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates.
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
- __version__ = "1.1.0"
4
+ __version__ = "1.1.2"
@@ -58,8 +58,9 @@ class TimerScheduler:
58
58
  self, resubmit_callback: Callable[[ExecutableWithState], None]
59
59
  ) -> None:
60
60
  self.resubmit_callback = resubmit_callback
61
- self._pending_resumes: list[tuple[float, ExecutableWithState]] = []
61
+ self._pending_resumes: list[tuple[float, int, ExecutableWithState]] = []
62
62
  self._lock = threading.Lock()
63
+ self._schedule_counter = 0
63
64
  self._shutdown = threading.Event()
64
65
  self._timer_thread = threading.Thread(target=self._timer_loop, daemon=True)
65
66
  self._timer_thread.start()
@@ -73,9 +74,18 @@ class TimerScheduler:
73
74
  def schedule_resume(
74
75
  self, exe_state: ExecutableWithState, resume_time: float
75
76
  ) -> None:
76
- """Schedule a task to resume at the specified time."""
77
+ """Schedule a task to resume at the specified time.
78
+
79
+ Uses a counter as a tie-breaker to ensure FIFO ordering when multiple
80
+ tasks have the same resume_time, preventing TypeError from comparing
81
+ ExecutableWithState objects.
82
+ """
77
83
  with self._lock:
78
- heapq.heappush(self._pending_resumes, (resume_time, exe_state))
84
+ heapq.heappush(
85
+ self._pending_resumes,
86
+ (resume_time, self._schedule_counter, exe_state),
87
+ )
88
+ self._schedule_counter += 1
79
89
 
80
90
  def shutdown(self) -> None:
81
91
  """Shutdown the timer thread and cancel all pending resumes."""
@@ -108,7 +118,7 @@ class TimerScheduler:
108
118
  self._pending_resumes
109
119
  and self._pending_resumes[0][0] <= current_time
110
120
  ):
111
- _, exe_state = heapq.heappop(self._pending_resumes)
121
+ _, _, exe_state = heapq.heappop(self._pending_resumes)
112
122
  if exe_state.can_resume:
113
123
  exe_state.reset_to_pending()
114
124
  self.resubmit_callback(exe_state)
@@ -32,7 +32,7 @@ from aws_durable_execution_sdk_python.state import ExecutionState, ReplayStatus
32
32
  if TYPE_CHECKING:
33
33
  from collections.abc import Callable, MutableMapping
34
34
 
35
- import boto3 # type: ignore
35
+ from mypy_boto3_lambda import LambdaClient as Boto3LambdaClient
36
36
 
37
37
  from aws_durable_execution_sdk_python.types import LambdaContext
38
38
 
@@ -59,6 +59,16 @@ class InitialExecutionState:
59
59
  next_marker=input_dict.get("NextMarker", ""),
60
60
  )
61
61
 
62
+ @staticmethod
63
+ def from_json_dict(input_dict: MutableMapping[str, Any]) -> InitialExecutionState:
64
+ operations = []
65
+ if input_operations := input_dict.get("Operations"):
66
+ operations = [Operation.from_json_dict(op) for op in input_operations]
67
+ return InitialExecutionState(
68
+ operations=operations,
69
+ next_marker=input_dict.get("NextMarker", ""),
70
+ )
71
+
62
72
  def get_execution_operation(self) -> Operation | None:
63
73
  if not self.operations:
64
74
  # Due to payload size limitations we may have an empty operations list.
@@ -91,6 +101,12 @@ class InitialExecutionState:
91
101
  "NextMarker": self.next_marker,
92
102
  }
93
103
 
104
+ def to_json_dict(self) -> MutableMapping[str, Any]:
105
+ return {
106
+ "Operations": [op.to_json_dict() for op in self.operations],
107
+ "NextMarker": self.next_marker,
108
+ }
109
+
94
110
 
95
111
  @dataclass(frozen=True)
96
112
  class DurableExecutionInvocationInput:
@@ -110,6 +126,18 @@ class DurableExecutionInvocationInput:
110
126
  ),
111
127
  )
112
128
 
129
+ @staticmethod
130
+ def from_json_dict(
131
+ input_dict: MutableMapping[str, Any],
132
+ ) -> DurableExecutionInvocationInput:
133
+ return DurableExecutionInvocationInput(
134
+ durable_execution_arn=input_dict["DurableExecutionArn"],
135
+ checkpoint_token=input_dict["CheckpointToken"],
136
+ initial_execution_state=InitialExecutionState.from_json_dict(
137
+ input_dict.get("InitialExecutionState", {})
138
+ ),
139
+ )
140
+
113
141
  def to_dict(self) -> MutableMapping[str, Any]:
114
142
  return {
115
143
  "DurableExecutionArn": self.durable_execution_arn,
@@ -117,6 +145,13 @@ class DurableExecutionInvocationInput:
117
145
  "InitialExecutionState": self.initial_execution_state.to_dict(),
118
146
  }
119
147
 
148
+ def to_json_dict(self) -> MutableMapping[str, Any]:
149
+ return {
150
+ "DurableExecutionArn": self.durable_execution_arn,
151
+ "CheckpointToken": self.checkpoint_token,
152
+ "InitialExecutionState": self.initial_execution_state.to_json_dict(),
153
+ }
154
+
120
155
 
121
156
  @dataclass(frozen=True)
122
157
  class DurableExecutionInvocationInputWithClient(DurableExecutionInvocationInput):
@@ -202,7 +237,7 @@ class DurableExecutionInvocationOutput:
202
237
  def durable_execution(
203
238
  func: Callable[[Any, DurableContext], Any] | None = None,
204
239
  *,
205
- boto3_client: boto3.client | None = None,
240
+ boto3_client: Boto3LambdaClient | None = None,
206
241
  ) -> Callable[[Any, LambdaContext], Any]:
207
242
  # Decorator called with parameters
208
243
  if func is None:
@@ -225,7 +260,7 @@ def durable_execution(
225
260
  logger.debug(
226
261
  "durableExecutionArn: %s", event.get("DurableExecutionArn")
227
262
  )
228
- invocation_input = DurableExecutionInvocationInput.from_dict(event)
263
+ invocation_input = DurableExecutionInvocationInput.from_json_dict(event)
229
264
  except (KeyError, TypeError, AttributeError) as e:
230
265
  msg = (
231
266
  "Unexpected payload provided to start the durable execution. "
@@ -1,13 +1,15 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import copy
3
4
  import datetime
4
5
  import logging
6
+ from collections.abc import MutableMapping
5
7
  from dataclasses import dataclass, field
6
8
  from enum import Enum
7
- from typing import TYPE_CHECKING, Any, Protocol, TypeAlias
9
+ from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, cast
8
10
 
9
- import boto3 # type: ignore
10
- from botocore.config import Config # type: ignore
11
+ import boto3
12
+ from botocore.config import Config
11
13
 
12
14
  from aws_durable_execution_sdk_python.exceptions import (
13
15
  CallableRuntimeError,
@@ -16,7 +18,11 @@ from aws_durable_execution_sdk_python.exceptions import (
16
18
  )
17
19
 
18
20
  if TYPE_CHECKING:
19
- from collections.abc import MutableMapping
21
+ from mypy_boto3_lambda import LambdaClient as Boto3LambdaClient
22
+ from mypy_boto3_lambda.type_defs import (
23
+ CheckpointDurableExecutionResponseTypeDef,
24
+ GetDurableExecutionStateResponseTypeDef,
25
+ )
20
26
 
21
27
  from aws_durable_execution_sdk_python.identifier import OperationIdentifier
22
28
 
@@ -692,6 +698,24 @@ class OperationUpdate:
692
698
  # endregion wait
693
699
 
694
700
 
701
+ class TimestampConverter:
702
+ """Converter for datetime/Unix timestamp conversions."""
703
+
704
+ @staticmethod
705
+ def to_unix_millis(dt: datetime.datetime | None) -> int | None:
706
+ """Convert datetime to Unix timestamp in milliseconds."""
707
+ return int(dt.timestamp() * 1000) if dt else None
708
+
709
+ @staticmethod
710
+ def from_unix_millis(ms: int | None) -> datetime.datetime | None:
711
+ """Convert Unix timestamp in milliseconds to datetime."""
712
+ return (
713
+ datetime.datetime.fromtimestamp(ms / 1000, tz=datetime.UTC)
714
+ if ms is not None
715
+ else None
716
+ )
717
+
718
+
695
719
  @dataclass(frozen=True)
696
720
  class Operation:
697
721
  """Represent the Operation type for GetDurableExecutionState and CheckpointDurableExecution."""
@@ -805,9 +829,11 @@ class Operation:
805
829
  step_dict["Error"] = self.step_details.error.to_dict()
806
830
  result["StepDetails"] = step_dict
807
831
  if self.wait_details:
808
- result["WaitDetails"] = {
809
- "ScheduledEndTimestamp": self.wait_details.scheduled_end_timestamp
810
- }
832
+ result["WaitDetails"] = (
833
+ {"ScheduledEndTimestamp": self.wait_details.scheduled_end_timestamp}
834
+ if self.wait_details.scheduled_end_timestamp
835
+ else {}
836
+ )
811
837
  if self.callback_details:
812
838
  callback_dict: MutableMapping[str, Any] = {
813
839
  "CallbackId": self.callback_details.callback_id
@@ -826,6 +852,79 @@ class Operation:
826
852
  result["ChainedInvokeDetails"] = invoke_dict
827
853
  return result
828
854
 
855
+ def to_json_dict(self) -> MutableMapping[str, Any]:
856
+ """Convert the Operation to a JSON-serializable dictionary.
857
+
858
+ Converts datetime objects to millisecond timestamps for JSON compatibility.
859
+
860
+ Returns:
861
+ A dictionary with JSON-serializable values
862
+ """
863
+ # Start with the regular to_dict output
864
+ result = self.to_dict()
865
+
866
+ # Convert datetime objects to millisecond timestamps
867
+ if ts := result.get("StartTimestamp"):
868
+ result["StartTimestamp"] = TimestampConverter.to_unix_millis(ts)
869
+
870
+ if ts := result.get("EndTimestamp"):
871
+ result["EndTimestamp"] = TimestampConverter.to_unix_millis(ts)
872
+
873
+ if (step_details := result.get("StepDetails")) and (
874
+ ts := step_details.get("NextAttemptTimestamp")
875
+ ):
876
+ result["StepDetails"]["NextAttemptTimestamp"] = (
877
+ TimestampConverter.to_unix_millis(ts)
878
+ )
879
+
880
+ if (wait_details := result.get("WaitDetails")) and (
881
+ ts := wait_details.get("ScheduledEndTimestamp")
882
+ ):
883
+ result["WaitDetails"]["ScheduledEndTimestamp"] = (
884
+ TimestampConverter.to_unix_millis(ts)
885
+ )
886
+
887
+ return result
888
+
889
+ @classmethod
890
+ def from_json_dict(cls, data: MutableMapping[str, Any]) -> Operation:
891
+ """Create an Operation from a JSON-serializable dictionary.
892
+
893
+ Converts millisecond timestamps back to datetime objects.
894
+
895
+ Args:
896
+ data: Dictionary with JSON-serializable values (millisecond timestamps)
897
+
898
+ Returns:
899
+ An Operation instance with datetime objects
900
+ """
901
+ # Make a copy to avoid modifying the original data
902
+ data_copy = copy.deepcopy(data)
903
+
904
+ # Convert millisecond timestamps back to datetime objects
905
+ if ms := data_copy.get("StartTimestamp"):
906
+ data_copy["StartTimestamp"] = TimestampConverter.from_unix_millis(ms)
907
+
908
+ if ms := data_copy.get("EndTimestamp"):
909
+ data_copy["EndTimestamp"] = TimestampConverter.from_unix_millis(ms)
910
+
911
+ if (step_details := data_copy.get("StepDetails")) and (
912
+ ms := step_details.get("NextAttemptTimestamp")
913
+ ):
914
+ step_details["NextAttemptTimestamp"] = TimestampConverter.from_unix_millis(
915
+ ms
916
+ )
917
+
918
+ if (wait_details := data_copy.get("WaitDetails")) and (
919
+ ms := wait_details.get("ScheduledEndTimestamp")
920
+ ):
921
+ wait_details["ScheduledEndTimestamp"] = TimestampConverter.from_unix_millis(
922
+ ms
923
+ )
924
+
925
+ # Use the existing from_dict method with the converted data
926
+ return cls.from_dict(data_copy)
927
+
829
928
 
830
929
  @dataclass(frozen=True)
831
930
  class CheckpointUpdatedExecutionState:
@@ -937,19 +1036,32 @@ class DurableServiceClient(Protocol):
937
1036
  class LambdaClient(DurableServiceClient):
938
1037
  """Persist durable operations to the Lambda Durable Function APIs."""
939
1038
 
940
- def __init__(self, client: Any) -> None:
1039
+ _cached_boto_client: Boto3LambdaClient | None = None
1040
+
1041
+ def __init__(self, client: Boto3LambdaClient) -> None:
941
1042
  self.client = client
942
1043
 
943
- @staticmethod
944
- def initialize_client() -> LambdaClient:
945
- client = boto3.client(
946
- "lambda",
947
- config=Config(
948
- connect_timeout=5,
949
- read_timeout=50,
950
- ),
951
- )
952
- return LambdaClient(client=client)
1044
+ @classmethod
1045
+ def initialize_client(cls) -> LambdaClient:
1046
+ """Initialize or return cached Lambda client.
1047
+
1048
+ Implements lazy initialization with class-level caching to optimize
1049
+ Lambda warm starts. The boto3 client is created once and reused across
1050
+ invocations, avoiding repeated credential resolution and connection
1051
+ pool setup.
1052
+
1053
+ Returns:
1054
+ LambdaClient: A new LambdaClient instance wrapping the cached boto3 client.
1055
+ """
1056
+ if cls._cached_boto_client is None:
1057
+ cls._cached_boto_client = boto3.client(
1058
+ "lambda",
1059
+ config=Config(
1060
+ connect_timeout=5,
1061
+ read_timeout=50,
1062
+ ),
1063
+ )
1064
+ return cls(client=cls._cached_boto_client)
953
1065
 
954
1066
  def checkpoint(
955
1067
  self,
@@ -959,19 +1071,20 @@ class LambdaClient(DurableServiceClient):
959
1071
  client_token: str | None,
960
1072
  ) -> CheckpointOutput:
961
1073
  try:
962
- params = {
963
- "DurableExecutionArn": durable_execution_arn,
964
- "CheckpointToken": checkpoint_token,
965
- "Updates": [o.to_dict() for o in updates],
966
- }
1074
+ optional_params: dict[str, str] = {}
967
1075
  if client_token is not None:
968
- params["ClientToken"] = client_token
969
-
970
- result: MutableMapping[str, Any] = self.client.checkpoint_durable_execution(
971
- **params
1076
+ optional_params["ClientToken"] = client_token
1077
+
1078
+ result: CheckpointDurableExecutionResponseTypeDef = (
1079
+ self.client.checkpoint_durable_execution(
1080
+ DurableExecutionArn=durable_execution_arn,
1081
+ CheckpointToken=checkpoint_token,
1082
+ Updates=cast(Any, [o.to_dict() for o in updates]),
1083
+ **optional_params, # type: ignore[arg-type]
1084
+ )
972
1085
  )
973
1086
 
974
- return CheckpointOutput.from_dict(result)
1087
+ return CheckpointOutput.from_dict(cast(MutableMapping[str, Any], result))
975
1088
  except Exception as e:
976
1089
  checkpoint_error = CheckpointError.from_exception(e)
977
1090
  logger.exception(
@@ -987,13 +1100,15 @@ class LambdaClient(DurableServiceClient):
987
1100
  max_items: int = 1000,
988
1101
  ) -> StateOutput:
989
1102
  try:
990
- result: MutableMapping[str, Any] = self.client.get_durable_execution_state(
991
- DurableExecutionArn=durable_execution_arn,
992
- CheckpointToken=checkpoint_token,
993
- Marker=next_marker,
994
- MaxItems=max_items,
1103
+ result: GetDurableExecutionStateResponseTypeDef = (
1104
+ self.client.get_durable_execution_state(
1105
+ DurableExecutionArn=durable_execution_arn,
1106
+ CheckpointToken=checkpoint_token,
1107
+ Marker=next_marker,
1108
+ MaxItems=max_items,
1109
+ )
995
1110
  )
996
- return StateOutput.from_dict(result)
1111
+ return StateOutput.from_dict(cast(MutableMapping[str, Any], result))
997
1112
  except Exception as e:
998
1113
  error = GetExecutionStateError.from_exception(e)
999
1114
  logger.exception(
@@ -314,6 +314,7 @@ class ExecutionState:
314
314
  OperationStatus.FAILED,
315
315
  OperationStatus.CANCELLED,
316
316
  OperationStatus.STOPPED,
317
+ OperationStatus.TIMED_OUT,
317
318
  }
318
319
  }
319
320
  if completed_ops.issubset(self._visited_operations):
@@ -613,20 +614,20 @@ class ExecutionState:
613
614
 
614
615
  logger.debug("Checkpoint batch processed successfully")
615
616
 
616
- # Signal completion for any synchronous operations
617
- for queued_op in batch:
618
- if queued_op.completion_event is not None:
619
- queued_op.completion_event.set()
620
-
621
617
  # Update local token for next iteration
622
618
  current_checkpoint_token = output.checkpoint_token
623
619
 
624
- # Fetch new operations from the API
620
+ # Fetch new operations from the API before unblocking sync waiters
625
621
  self.fetch_paginated_operations(
626
622
  output.new_execution_state.operations,
627
623
  output.checkpoint_token,
628
624
  output.new_execution_state.next_marker,
629
625
  )
626
+
627
+ # Signal completion for any synchronous operations
628
+ for queued_op in batch:
629
+ if queued_op.completion_event is not None:
630
+ queued_op.completion_event.set()
630
631
  except Exception as e:
631
632
  # Checkpoint failed - wake all blocked threads so they can raise error
632
633
  # Drain both queues and signal all completion events
@@ -0,0 +1,131 @@
1
+ Metadata-Version: 2.4
2
+ Name: aws-durable-execution-sdk-python
3
+ Version: 1.1.2
4
+ Summary: AWS Durable Execution SDK for Python
5
+ Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
6
+ Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
7
+ Project-URL: Source, https://github.com/aws/aws-durable-execution-sdk-python
8
+ Author-email: yaythomas <tgaigher@amazon.com>
9
+ License-Expression: Apache-2.0
10
+ License-File: LICENSE
11
+ License-File: NOTICE
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Programming Language :: Python :: Implementation :: CPython
19
+ Classifier: Programming Language :: Python :: Implementation :: PyPy
20
+ Requires-Python: >=3.11
21
+ Requires-Dist: boto3>=1.42.1
22
+ Description-Content-Type: text/markdown
23
+
24
+ # AWS Durable Execution SDK for Python
25
+
26
+ [![Build](https://github.com/aws/aws-durable-execution-sdk-python/actions/workflows/ci.yml/badge.svg)](https://github.com/aws/aws-durable-execution-sdk-python/actions/workflows/ci.yml)
27
+ [![PyPI - Version](https://img.shields.io/pypi/v/aws-durable-execution-sdk-python.svg)](https://pypi.org/project/aws-durable-execution-sdk-python)
28
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/aws-durable-execution-sdk-python.svg)](https://pypi.org/project/aws-durable-execution-sdk-python)
29
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/aws/aws-durable-execution-sdk-python/badge)](https://scorecard.dev/viewer/?uri=github.com/aws/aws-durable-execution-sdk-python)
30
+ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
31
+
32
+ -----
33
+
34
+ Build reliable, long-running AWS Lambda workflows with checkpointed steps, waits, callbacks, and parallel execution.
35
+
36
+ ## ✨ Key Features
37
+
38
+ - **Automatic checkpointing** - Resume execution after Lambda pauses or restarts
39
+ - **Durable steps** - Run work with retry strategies and deterministic replay
40
+ - **Waits and callbacks** - Pause for time or external signals without blocking Lambda
41
+ - **Parallel and map operations** - Fan out work with configurable completion criteria
42
+ - **Child contexts** - Structure complex workflows into isolated subflows
43
+ - **Replay-safe logging** - Use `context.logger` for structured, de-duplicated logs
44
+ - **Local and cloud testing** - Validate workflows with the testing SDK
45
+
46
+ ## 📦 Packages
47
+
48
+ | Package | Description | Version |
49
+ | --- | --- | --- |
50
+ | `aws-durable-execution-sdk-python` | Execution SDK for Lambda durable functions | [![PyPI - Version](https://img.shields.io/pypi/v/aws-durable-execution-sdk-python.svg)](https://pypi.org/project/aws-durable-execution-sdk-python) |
51
+ | `aws-durable-execution-sdk-python-testing` | Local/cloud test runner and pytest helpers | [![PyPI - Version](https://img.shields.io/pypi/v/aws-durable-execution-sdk-python-testing.svg)](https://pypi.org/project/aws-durable-execution-sdk-python-testing) |
52
+
53
+ ## 🚀 Quick Start
54
+
55
+ Install the execution SDK:
56
+
57
+ ```console
58
+ pip install aws-durable-execution-sdk-python
59
+ ```
60
+
61
+ Create a durable Lambda handler:
62
+
63
+ ```python
64
+ from aws_durable_execution_sdk_python import (
65
+ DurableContext,
66
+ StepContext,
67
+ durable_execution,
68
+ durable_step,
69
+ )
70
+ from aws_durable_execution_sdk_python.config import Duration
71
+
72
+ @durable_step
73
+ def validate_order(step_ctx: StepContext, order_id: str) -> dict:
74
+ step_ctx.logger.info("Validating order", extra={"order_id": order_id})
75
+ return {"order_id": order_id, "valid": True}
76
+
77
+ @durable_execution
78
+ def handler(event: dict, context: DurableContext) -> dict:
79
+ order_id = event["order_id"]
80
+ context.logger.info("Starting workflow", extra={"order_id": order_id})
81
+
82
+ validation = context.step(validate_order(order_id), name="validate_order")
83
+ if not validation["valid"]:
84
+ return {"status": "rejected", "order_id": order_id}
85
+
86
+ # simulate approval (real world: use wait_for_callback)
87
+ context.wait(duration=Duration.from_seconds(5), name="await_confirmation")
88
+
89
+ return {"status": "approved", "order_id": order_id}
90
+ ```
91
+
92
+ ## 📚 Documentation
93
+
94
+ - **[AWS Documentation](https://docs.aws.amazon.com/lambda/latest/dg/durable-functions.html)** - Official AWS Lambda durable functions guide
95
+ - **[Documentation index](docs/index.md)** - SDK Overview and navigation
96
+
97
+ **New to durable functions?**
98
+ - [Getting started guide](docs/getting-started.md) - Build your first durable function
99
+
100
+ **Core operations:**
101
+ - [Steps](docs/core/steps.md) - Execute code with automatic checkpointing and retry support
102
+ - [Wait operations](docs/core/wait.md) - Pause execution without blocking Lambda resources
103
+ - [Callbacks](docs/core/callbacks.md) - Wait for external systems to respond
104
+ - [Invoke operations](docs/core/invoke.md) - Call other durable functions and compose workflows
105
+ - [Child contexts](docs/core/child-contexts.md) - Organize complex workflows into isolated units
106
+ - [Parallel operations](docs/core/parallel.md) - Run multiple operations concurrently
107
+ - [Map operations](docs/core/map.md) - Process collections in parallel with batching
108
+ - [Logger integration](docs/core/logger.md) - Add structured logging to track execution
109
+
110
+ **Advanced topics:**
111
+ - [Error handling](docs/advanced/error-handling.md) - Handle failures and implement retry strategies
112
+ - [Testing modes](docs/advanced/testing-modes.md) - Run tests locally or against deployed Lambda functions
113
+ - [Testing patterns](docs/testing-patterns/basic-tests.md) - Practical testing examples
114
+ - [Serialization](docs/advanced/serialization.md) - Customize how data is serialized in checkpoints
115
+
116
+ **Architecture:**
117
+ - [Architecture diagrams](docs/architecture.md) - Class diagrams and concurrency flows
118
+
119
+ **API reference:**
120
+ - API reference docs are in progress. Use the core operation docs above for now.
121
+
122
+ ## 💬 Feedback & Support
123
+
124
+ - [Bug report](https://github.com/aws/aws-durable-execution-sdk-python/issues/new?template=bug_report.yml)
125
+ - [Feature request](https://github.com/aws/aws-durable-execution-sdk-python/issues/new?template=feature_request.yml)
126
+ - [Documentation feedback](https://github.com/aws/aws-durable-execution-sdk-python/issues/new?template=documentation.yml)
127
+ - [Contributing guide](CONTRIBUTING.md)
128
+
129
+ ## 📄 License
130
+
131
+ See the [LICENSE](LICENSE) file for our project's licensing.
@@ -1,23 +1,23 @@
1
1
  aws_durable_execution_sdk_python/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- aws_durable_execution_sdk_python/__about__.py,sha256=J8Ioq0hfedDOXKbDonThH6dyrixwCOoGnMoJKfbnTuQ,137
2
+ aws_durable_execution_sdk_python/__about__.py,sha256=7qzc63t8OUhHP9YvXrUIRIK7LxcEvXL60VN7Z4b_y4M,137
3
3
  aws_durable_execution_sdk_python/__init__.py,sha256=reI6AQTpEyeK3ev6iZJuu7A6PSoa0V1VJGOgSB2RpGM,1023
4
4
  aws_durable_execution_sdk_python/config.py,sha256=vd_8IZO5KZU3qWENbWDVtyLglqw3sa6SU-R8CYUgm9g,18938
5
5
  aws_durable_execution_sdk_python/context.py,sha256=efJIc5KrsTzcI_azMjxqrHiPt71PuiQy4vbdsUiTtLI,22467
6
6
  aws_durable_execution_sdk_python/exceptions.py,sha256=AvuXnBHAijXb3TepNTu5CVDb_cpR6Yw6x4W3jELVneQ,13631
7
- aws_durable_execution_sdk_python/execution.py,sha256=9-FQEZglOMcNUUMQg5ejtnigTSOgSrKsu-vpiIYFANE,17160
7
+ aws_durable_execution_sdk_python/execution.py,sha256=A20fOp6_wXdIcL1YiqJ_RGJbQ09D1lSivBQnUY0FT3Y,18571
8
8
  aws_durable_execution_sdk_python/identifier.py,sha256=0NyNTb8mTrUni7PlmFtzvJdW6lk0_XpuAaL89LNc378,324
9
- aws_durable_execution_sdk_python/lambda_service.py,sha256=FrUcHth2hlXK__h8mfEp2T5NW-WjpaF56GleTota0Ro,33416
9
+ aws_durable_execution_sdk_python/lambda_service.py,sha256=FkkcmFxDZLA4JS7mMW2YT9BfqTNHHJtEk9kSADwu5bk,37858
10
10
  aws_durable_execution_sdk_python/logger.py,sha256=nde1fhe9uK-UmXRZI7T_sIoU-_T5PnZo16boaFQFtyw,4281
11
11
  aws_durable_execution_sdk_python/py.typed,sha256=DbCDcAirv769HS1ibzZLXq8VPazYf7JCkKSzj3ctudM,58
12
12
  aws_durable_execution_sdk_python/retries.py,sha256=CNJ6msQHVhcdSjMuuQAvR0WvsaN7cFA7QgQTVI8pK2w,5829
13
13
  aws_durable_execution_sdk_python/serdes.py,sha256=-hzqPyAHi1rsOUwpi7RjiJBNiDc0Oe5_QVNMemYBfGY,16808
14
- aws_durable_execution_sdk_python/state.py,sha256=tUnN-hC_4QmAISUq8zaFZ5TF6UFkL0VPdYZM8cRNCm8,33221
14
+ aws_durable_execution_sdk_python/state.py,sha256=5kWSwtOCt0VkhhB2orhGyrXNg1nBt-HXfXm0T8VzAiE,33303
15
15
  aws_durable_execution_sdk_python/suspend.py,sha256=PIgc6w5-diZIq58-tPv51dzBSHGW7e4oR0QFam8Vgi0,3323
16
16
  aws_durable_execution_sdk_python/threading.py,sha256=cU7vIEWHYco-qqkgoAMyPHSOQ8b_Q_r_rZGG7m1g7I4,7650
17
17
  aws_durable_execution_sdk_python/types.py,sha256=8vqwa5q2EikesX_NdhR_njin7l7uHakhWPH_ntHY_PI,5152
18
18
  aws_durable_execution_sdk_python/waits.py,sha256=ZO_SWafI0G7rWxItpnL7w4o--eDp2VAjnGZlrBJfufc,3883
19
19
  aws_durable_execution_sdk_python/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
- aws_durable_execution_sdk_python/concurrency/executor.py,sha256=LqGGDE94My7ECsaOcvfZE_-yZmYVXM0Um9M0_Z042rI,17800
20
+ aws_durable_execution_sdk_python/concurrency/executor.py,sha256=NkpB0gRRykhc1eMB1xGNy7VPMawIkejMuV2-T--Vnao,18156
21
21
  aws_durable_execution_sdk_python/concurrency/models.py,sha256=wPi0PEFsCFthoXlRMkomS0Tl9hZ7IVk4d-9xWH5cIJQ,15540
22
22
  aws_durable_execution_sdk_python/operation/__init__.py,sha256=xjtSpukZb6sftgJ5oVPSc6TcgKgVQg4vXHeYIeKOPks,25
23
23
  aws_durable_execution_sdk_python/operation/base.py,sha256=5Ppis4xFJoXQK4f7-cF-RYmeF84nAVlepzJgHx_OBYc,7049
@@ -29,8 +29,8 @@ aws_durable_execution_sdk_python/operation/parallel.py,sha256=uzxa_XsTvimC63Hpad
29
29
  aws_durable_execution_sdk_python/operation/step.py,sha256=5R-Rzmfh5uYHMQdq3WoDzwXUBwNg-ToeWW5DI7_l7zw,15567
30
30
  aws_durable_execution_sdk_python/operation/wait.py,sha256=P1R7-BrQ0eadFxQiGLHL3ySkuJUqPvVv7zewAzBeVT8,4193
31
31
  aws_durable_execution_sdk_python/operation/wait_for_condition.py,sha256=s33dEf5qosed_DxZiNN0A_t9DHfpHBpcfD7HqaEfbx0,11768
32
- aws_durable_execution_sdk_python-1.1.0.dist-info/METADATA,sha256=nwHBHS--ccExnKbPm6kI_EAiBlpDCgNfD29NMtbthKE,23580
33
- aws_durable_execution_sdk_python-1.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
34
- aws_durable_execution_sdk_python-1.1.0.dist-info/licenses/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
35
- aws_durable_execution_sdk_python-1.1.0.dist-info/licenses/NOTICE,sha256=1CkO1kwu3Q_OHYTj-d-yiBJA_lNN73a4zSntavaD4oc,67
36
- aws_durable_execution_sdk_python-1.1.0.dist-info/RECORD,,
32
+ aws_durable_execution_sdk_python-1.1.2.dist-info/METADATA,sha256=hYagEtD3Dl8MjSAH41gPAODHtYKb3Dyo3CNpm4mA9xs,6424
33
+ aws_durable_execution_sdk_python-1.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
34
+ aws_durable_execution_sdk_python-1.1.2.dist-info/licenses/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
35
+ aws_durable_execution_sdk_python-1.1.2.dist-info/licenses/NOTICE,sha256=1CkO1kwu3Q_OHYTj-d-yiBJA_lNN73a4zSntavaD4oc,67
36
+ aws_durable_execution_sdk_python-1.1.2.dist-info/RECORD,,
@@ -1,681 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: aws-durable-execution-sdk-python
3
- Version: 1.1.0
4
- Summary: AWS Durable Execution SDK for Python
5
- Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
6
- Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
7
- Project-URL: Source, https://github.com/aws/aws-durable-execution-sdk-python
8
- Author-email: yaythomas <tgaigher@amazon.com>
9
- License-Expression: Apache-2.0
10
- License-File: LICENSE
11
- License-File: NOTICE
12
- Classifier: Development Status :: 4 - Beta
13
- Classifier: Programming Language :: Python
14
- Classifier: Programming Language :: Python :: 3.11
15
- Classifier: Programming Language :: Python :: 3.12
16
- Classifier: Programming Language :: Python :: 3.13
17
- Classifier: Programming Language :: Python :: 3.14
18
- Classifier: Programming Language :: Python :: Implementation :: CPython
19
- Classifier: Programming Language :: Python :: Implementation :: PyPy
20
- Requires-Python: >=3.11
21
- Requires-Dist: boto3>=1.42.1
22
- Description-Content-Type: text/markdown
23
-
24
- # AWS Durable Execution SDK for Python
25
-
26
- [![PyPI - Version](https://img.shields.io/pypi/v/aws-durable-execution-sdk-python.svg)](https://pypi.org/project/aws-durable-execution-sdk-python)
27
- [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/aws-durable-execution-sdk-python.svg)](https://pypi.org/project/aws-durable-execution-sdk-python)
28
-
29
- [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/aws/aws-durable-execution-sdk-python/badge)](https://scorecard.dev/viewer/?uri=github.com/aws/aws-durable-execution-sdk-python)
30
-
31
- -----
32
-
33
- ## Table of Contents
34
-
35
- - [Installation](#installation)
36
- - [License](#license)
37
-
38
- ## Installation
39
-
40
- ```console
41
- pip install aws-durable-execution-sdk-python
42
- ```
43
-
44
- ## Developers
45
- Please see [CONTRIBUTING.md](CONTRIBUTING.md). It contains the testing guide, sample commands and instructions
46
- for how to contribute to this package.
47
-
48
- tldr; use `hatch` and it will manage virtual envs and dependencies for you, so you don't have to do it manually.
49
-
50
- ## Core Architecture
51
- The entry-point that consumers of the SDK interact with is the DurableContext.
52
-
53
- ### DurableContext Operations
54
- - **Core Methods**: `set_logger`, `step`, `invoke`, `map`, `parallel`, `run_in_child_context`, `wait`, `create_callback`, `wait_for_callback`, `wait_for_condition`
55
- - **Thread Safety**: Uses `OrderedCounter` for generating sequential step IDs
56
- - **State Management**: Delegates to `ExecutionState` for checkpointing
57
-
58
- ### Concurrency Implementation
59
- - **Map/Parallel**: Both inherit from `ConcurrentExecutor` abstract base class
60
- - **Thread Pool**: Uses `ThreadPoolExecutor` for concurrent execution
61
- - **State Tracking**: `ExecutableWithState` manages individual task lifecycle
62
- - **Completion Logic**: `ExecutionCounters` tracks success/failure criteria
63
- - **Suspension**: `TimerScheduler` handles timed suspensions and resumptions
64
-
65
- ### Configuration System
66
- - **Modular Configs**: Separate config classes for each operation type
67
- - **Completion Control**: `CompletionConfig` defines success/failure criteria
68
- - **Serialization**: `SerDes` interface for custom serialization
69
-
70
- ### Operation Handlers
71
- - **Separation of Concerns**: Each operation has dedicated handler function
72
- - **Checkpointing**: All operations integrate with execution state checkpointing
73
- - **Error Handling**: Consistent error handling and retry logic across operations
74
-
75
-
76
- ```mermaid
77
- classDiagram
78
- class DurableContext {
79
- -ExecutionState state
80
- -Any lambda_context
81
- -str _parent_id
82
- -OrderedCounter _step_counter
83
- -LogInfo _log_info
84
- -Logger logger
85
-
86
- +set_logger(LoggerInterface new_logger)
87
- +step(Callable func, str name, StepConfig config) T
88
- +invoke(str function_name, P payload, str name, InvokeConfig config) R
89
- +map(Sequence inputs, Callable func, str name, MapConfig config) BatchResult
90
- +parallel(Sequence functions, str name, ParallelConfig config) BatchResult
91
- +run_in_child_context(Callable func, str name, ChildConfig config) T
92
- +wait(int seconds, str name)
93
- +create_callback(str name, CallbackConfig config) Callback
94
- +wait_for_callback(Callable submitter, str name, WaitForCallbackConfig config) Any
95
- +wait_for_condition(Callable check, WaitForConditionConfig config, str name) T
96
- }
97
-
98
- class DurableContextProtocol {
99
- <<interface>>
100
- +step(Callable func, str name, StepConfig config) T
101
- +run_in_child_context(Callable func, str name, ChildConfig config) T
102
- +map(Sequence inputs, Callable func, str name, MapConfig config) BatchResult
103
- +parallel(Sequence functions, str name, ParallelConfig config) BatchResult
104
- +wait(int seconds, str name)
105
- +create_callback(str name, CallbackConfig config) Callback
106
- }
107
-
108
- class OrderedCounter {
109
- -OrderedLock _lock
110
- -int _counter
111
- +increment() int
112
- +decrement() int
113
- +get_current() int
114
- }
115
-
116
- class ExecutionState {
117
- +str durable_execution_arn
118
- +get_checkpoint_result(str operation_id) CheckpointedResult
119
- +create_checkpoint(OperationUpdate operation_update)
120
- }
121
-
122
- class Logger {
123
- +LoggerInterface logger
124
- +LogInfo info
125
- +with_log_info(LogInfo info) Logger
126
- +from_log_info(LoggerInterface logger, LogInfo info) Logger
127
- }
128
-
129
- DurableContext ..|> DurableContextProtocol : implements
130
- DurableContext --> ExecutionState : uses
131
- DurableContext --> OrderedCounter : contains
132
- DurableContext --> Logger : contains
133
- ```
134
-
135
- ## Operation Handlers
136
- The `DurableContext` calls operation handlers, which contain the execution logic for each operation.
137
-
138
- ```mermaid
139
- classDiagram
140
- class DurableContext {
141
- +step(Callable func, str name, StepConfig config) T
142
- +invoke(str function_name, P payload, str name, InvokeConfig config) R
143
- +map(Sequence inputs, Callable func, str name, MapConfig config) BatchResult
144
- +parallel(Sequence functions, str name, ParallelConfig config) BatchResult
145
- +run_in_child_context(Callable func, str name, ChildConfig config) T
146
- +wait(int seconds, str name)
147
- +create_callback(str name, CallbackConfig config) Callback
148
- +wait_for_callback(Callable submitter, str name, WaitForCallbackConfig config) Any
149
- +wait_for_condition(Callable check, WaitForConditionConfig config, str name) T
150
- }
151
-
152
- class step_handler {
153
- <<function>>
154
- +step_handler(Callable func, ExecutionState state, OperationIdentifier op_id, StepConfig config, Logger logger) T
155
- }
156
-
157
- class invoke_handler {
158
- <<function>>
159
- +invoke_handler(str function_name, P payload, ExecutionState state, OperationIdentifier op_id, InvokeConfig config) R
160
- }
161
-
162
- class map_handler {
163
- <<function>>
164
- +map_handler(Sequence items, Callable func, MapConfig config, ExecutionState state, Callable run_in_child_context) BatchResult
165
- }
166
-
167
- class parallel_handler {
168
- <<function>>
169
- +parallel_handler(Sequence callables, ParallelConfig config, ExecutionState state, Callable run_in_child_context) BatchResult
170
- }
171
-
172
- class child_handler {
173
- <<function>>
174
- +child_handler(Callable func, ExecutionState state, OperationIdentifier op_id, ChildConfig config) T
175
- }
176
-
177
- class wait_handler {
178
- <<function>>
179
- +wait_handler(int seconds, ExecutionState state, OperationIdentifier op_id)
180
- }
181
-
182
- class create_callback_handler {
183
- <<function>>
184
- +create_callback_handler(ExecutionState state, OperationIdentifier op_id, CallbackConfig config) str
185
- }
186
-
187
- class wait_for_callback_handler {
188
- <<function>>
189
- +wait_for_callback_handler(DurableContext context, Callable submitter, str name, WaitForCallbackConfig config) Any
190
- }
191
-
192
- class wait_for_condition_handler {
193
- <<function>>
194
- +wait_for_condition_handler(Callable check, WaitForConditionConfig config, ExecutionState state, OperationIdentifier op_id, Logger logger) T
195
- }
196
-
197
- DurableContext --> step_handler : calls
198
- DurableContext --> invoke_handler : calls
199
- DurableContext --> map_handler : calls
200
- DurableContext --> parallel_handler : calls
201
- DurableContext --> child_handler : calls
202
- DurableContext --> wait_handler : calls
203
- DurableContext --> create_callback_handler : calls
204
- DurableContext --> wait_for_callback_handler : calls
205
- DurableContext --> wait_for_condition_handler : calls
206
- ```
207
-
208
- ## Configuration Module Classes
209
-
210
- ```mermaid
211
- classDiagram
212
- class StepConfig {
213
- +Callable retry_strategy
214
- +StepSemantics step_semantics
215
- +SerDes serdes
216
- }
217
-
218
- class InvokeConfig~P,R~ {
219
- +int timeout_seconds
220
- +SerDes~P~ serdes_payload
221
- +SerDes~R~ serdes_result
222
- }
223
-
224
- class MapConfig {
225
- +int max_concurrency
226
- +ItemBatcher item_batcher
227
- +CompletionConfig completion_config
228
- +SerDes serdes
229
- }
230
-
231
- class ParallelConfig {
232
- +int max_concurrency
233
- +CompletionConfig completion_config
234
- +SerDes serdes
235
- }
236
-
237
- class ChildConfig~T~ {
238
- +SerDes serdes
239
- +OperationSubType sub_type
240
- +Callable~T,str~ summary_generator
241
- }
242
-
243
- class CallbackConfig {
244
- +int timeout_seconds
245
- +int heartbeat_timeout_seconds
246
- +SerDes serdes
247
- }
248
-
249
- class WaitForCallbackConfig {
250
- +Callable retry_strategy
251
- }
252
-
253
- class WaitForConditionConfig~T~ {
254
- +Callable wait_strategy
255
- +T initial_state
256
- +SerDes serdes
257
- }
258
-
259
- class CompletionConfig {
260
- +int min_successful
261
- +int tolerated_failure_count
262
- +float tolerated_failure_percentage
263
- +first_successful()$ CompletionConfig
264
- +all_completed()$ CompletionConfig
265
- +all_successful()$ CompletionConfig
266
- }
267
-
268
- class ItemBatcher~T~ {
269
- +int max_items_per_batch
270
- +float max_item_bytes_per_batch
271
- +T batch_input
272
- }
273
-
274
- WaitForCallbackConfig --|> CallbackConfig : extends
275
- MapConfig --> CompletionConfig : contains
276
- MapConfig --> ItemBatcher : contains
277
- ParallelConfig --> CompletionConfig : contains
278
- ```
279
-
280
- ## Types and Protocols Module
281
-
282
- ```mermaid
283
- classDiagram
284
- class DurableContextProtocol {
285
- <<interface>>
286
- +step(Callable func, str name, StepConfig config) T
287
- +run_in_child_context(Callable func, str name, ChildConfig config) T
288
- +map(Sequence inputs, Callable func, str name, MapConfig config) BatchResult
289
- +parallel(Sequence functions, str name, ParallelConfig config) BatchResult
290
- +wait(int seconds, str name)
291
- +create_callback(str name, CallbackConfig config) Callback
292
- }
293
-
294
- class LoggerInterface {
295
- <<interface>>
296
- +debug(object msg, *args, Mapping extra)
297
- +info(object msg, *args, Mapping extra)
298
- +warning(object msg, *args, Mapping extra)
299
- +error(object msg, *args, Mapping extra)
300
- +exception(object msg, *args, Mapping extra)
301
- }
302
-
303
- class CallbackProtocol~C_co~ {
304
- <<interface>>
305
- +str callback_id
306
- +result() C_co
307
- }
308
-
309
- class BatchResultProtocol~T~ {
310
- <<interface>>
311
- +get_results() list~T~
312
- }
313
-
314
- class StepContext {
315
- +LoggerInterface logger
316
- }
317
-
318
- class WaitForConditionCheckContext {
319
- +LoggerInterface logger
320
- }
321
-
322
- class OperationContext {
323
- +LoggerInterface logger
324
- }
325
-
326
- StepContext --|> OperationContext : extends
327
- WaitForConditionCheckContext --|> OperationContext : extends
328
- ```
329
-
330
- ## SerDes Module Classes
331
-
332
- ```mermaid
333
- classDiagram
334
- class SerDes~T~ {
335
- <<abstract>>
336
- +serialize(T value, SerDesContext context) str
337
- +deserialize(str data, SerDesContext context) T
338
- }
339
-
340
- class JsonSerDes~T~ {
341
- +serialize(T value, SerDesContext context) str
342
- +deserialize(str data, SerDesContext context) T
343
- }
344
-
345
- class SerDesContext {
346
- +str operation_id
347
- +str durable_execution_arn
348
- }
349
-
350
- class serialize {
351
- <<function>>
352
- +serialize(SerDes serdes, T value, str operation_id, str durable_execution_arn) str
353
- }
354
-
355
- class deserialize {
356
- <<function>>
357
- +deserialize(SerDes serdes, str data, str operation_id, str durable_execution_arn) T
358
- }
359
-
360
- JsonSerDes ..|> SerDes : implements
361
- serialize --> SerDes : uses
362
- deserialize --> SerDes : uses
363
- SerDes --> SerDesContext : uses
364
- ```
365
-
366
- ## Concurrency Architecture - Map and Parallel Operations
367
-
368
- ```mermaid
369
- classDiagram
370
- class ConcurrentExecutor~CallableType,ResultType~ {
371
- <<abstract>>
372
- +list~Executable~ executables
373
- +int max_concurrency
374
- +CompletionConfig completion_config
375
- +ExecutionCounters counters
376
- +list~ExecutableWithState~ executables_with_state
377
- +Event _completion_event
378
- +SuspendExecution _suspend_exception
379
-
380
- +execute(ExecutionState state, Callable run_in_child_context) BatchResult~ResultType~
381
- +execute_item(DurableContext child_context, Executable executable)* ResultType
382
- +should_execution_suspend() SuspendResult
383
- -_on_task_complete(ExecutableWithState exe_state, Future future, TimerScheduler scheduler)
384
- -_create_result() BatchResult~ResultType~
385
- }
386
-
387
- class MapExecutor~T,R~ {
388
- +Sequence~T~ items
389
- +execute_item(DurableContext child_context, Executable executable) R
390
- +from_items(Sequence items, Callable func, MapConfig config)$ MapExecutor
391
- }
392
-
393
- class ParallelExecutor {
394
- +execute_item(DurableContext child_context, Executable executable) R
395
- +from_callables(Sequence callables, ParallelConfig config)$ ParallelExecutor
396
- }
397
-
398
- class Executable~CallableType~ {
399
- +int index
400
- +CallableType func
401
- }
402
-
403
- class ExecutableWithState~CallableType,ResultType~ {
404
- +Executable~CallableType~ executable
405
- -BranchStatus _status
406
- -Future _future
407
- -float _suspend_until
408
- -ResultType _result
409
- -Exception _error
410
-
411
- +run(Future future)
412
- +suspend()
413
- +suspend_with_timeout(float timestamp)
414
- +complete(ResultType result)
415
- +fail(Exception error)
416
- +reset_to_pending()
417
- +can_resume() bool
418
- +is_running() bool
419
- }
420
-
421
- class ExecutionCounters {
422
- +int total_tasks
423
- +int min_successful
424
- +int success_count
425
- +int failure_count
426
- -Lock _lock
427
-
428
- +complete_task()
429
- +fail_task()
430
- +should_complete() bool
431
- +is_all_completed() bool
432
- +is_min_successful_reached() bool
433
- +is_failure_tolerance_exceeded() bool
434
- }
435
-
436
- class TimerScheduler {
437
- +Callable resubmit_callback
438
- -list _pending_resumes
439
- -Lock _lock
440
- -Event _shutdown
441
- -Thread _timer_thread
442
-
443
- +schedule_resume(ExecutableWithState exe_state, float resume_time)
444
- +shutdown()
445
- -_timer_loop()
446
- }
447
-
448
- class BatchResult~R~ {
449
- +list~BatchItem~R~~ all
450
- +CompletionReason completion_reason
451
- +succeeded() list~BatchItem~R~~
452
- +failed() list~BatchItem~R~~
453
- +get_results() list~R~
454
- +throw_if_error()
455
- }
456
-
457
- class BatchItem~R~ {
458
- +int index
459
- +BatchItemStatus status
460
- +R result
461
- +ErrorObject error
462
- }
463
-
464
- MapExecutor --|> ConcurrentExecutor : extends
465
- ParallelExecutor --|> ConcurrentExecutor : extends
466
- ConcurrentExecutor --> ExecutableWithState : manages
467
- ConcurrentExecutor --> ExecutionCounters : uses
468
- ConcurrentExecutor --> TimerScheduler : uses
469
- ConcurrentExecutor --> BatchResult : creates
470
- ExecutableWithState --> Executable : contains
471
- BatchResult --> BatchItem : contains
472
- ```
473
-
474
- ## Concurrency Flow
475
-
476
- ```mermaid
477
- sequenceDiagram
478
- participant DC as DurableContext
479
- participant MH as map_handler
480
- participant ME as MapExecutor
481
- participant CE as ConcurrentExecutor
482
- participant TP as ThreadPoolExecutor
483
- participant TS as TimerScheduler
484
- participant EC as ExecutionCounters
485
-
486
- DC->>MH: map(inputs, func, config)
487
- MH->>ME: MapExecutor.from_items()
488
- ME->>CE: execute(state, run_in_child_context)
489
-
490
- CE->>TP: ThreadPoolExecutor(max_workers)
491
- CE->>TS: TimerScheduler(resubmitter)
492
- CE->>EC: ExecutionCounters(total, min_successful)
493
-
494
- loop For each executable
495
- CE->>TP: submit_task(executable_with_state)
496
- TP->>CE: execute_item_in_child_context()
497
- CE->>DC: run_in_child_context(child_func)
498
- DC->>ME: execute_item(child_context, executable)
499
- end
500
-
501
- par Task Completion Handling
502
- TP->>CE: on_task_complete(future)
503
- CE->>EC: complete_task() / fail_task()
504
- CE->>CE: should_execution_suspend()
505
- alt Should Complete
506
- CE->>CE: _completion_event.set()
507
- else Should Suspend
508
- CE->>TS: schedule_resume(exe_state, timestamp)
509
- end
510
- end
511
-
512
- CE->>CE: _completion_event.wait()
513
- CE->>CE: _create_result()
514
- CE->>DC: BatchResult
515
- ```
516
-
517
- ## Threading and Locking
518
-
519
- ```mermaid
520
- classDiagram
521
- class OrderedLock {
522
- -Lock _lock
523
- -deque~Event~ _waiters
524
- -bool _is_broken
525
- -Exception _exception
526
-
527
- +acquire() bool
528
- +release()
529
- +reset()
530
- +is_broken() bool
531
- +__enter__() OrderedLock
532
- +__exit__(exc_type, exc_val, exc_tb)
533
- }
534
-
535
- class OrderedCounter {
536
- -OrderedLock _lock
537
- -int _counter
538
-
539
- +increment() int
540
- +decrement() int
541
- +get_current() int
542
- }
543
-
544
- class Event {
545
- <<threading.Event>>
546
- +set()
547
- +wait()
548
- }
549
-
550
- class Lock {
551
- <<threading.Lock>>
552
- +acquire()
553
- +release()
554
- }
555
-
556
- OrderedCounter --> OrderedLock : uses
557
- OrderedLock --> Lock : contains
558
- OrderedLock --> Event : manages queue of
559
- ```
560
-
561
- ## Checkpointing System
562
-
563
- The SDK invokes the AWS Lambda checkpoint API to persist execution state. Checkpoints are batched for efficiency and can be either
564
- synchronous (blocking) or asynchronous (non-blocking). Critical checkpoints are blocking,
565
- meaning that execution will not proceed until the checkpoint call has successfully completed.
566
-
567
- ### Checkpoint Types
568
-
569
- Checkpoints are categorized by their action (START, SUCCEED, FAIL) and whether they are critical to execution correctness:
570
-
571
- | Operation Type | Action | Is Sync? | Rationale |
572
- |---------------|--------|----------|-----------|
573
- | Step (AtMostOncePerRetry) | START | Yes | Prevents duplicate execution - must wait for confirmation |
574
- | Step (AtLeastOncePerRetry) | START | No | Performance optimization - idempotent operations can retry |
575
- | Step | SUCCEED/FAIL | Yes | Ensures result persisted before returning to caller |
576
- | Callback | START | Yes | Must wait for API to generate callback ID |
577
- | Callback | SUCCEED/FAIL | Yes | Ensures callback result persisted |
578
- | Invoke | START | Yes | Ensures chained invoke recorded before proceeding |
579
- | Invoke | SUCCEED/FAIL | Yes | Ensures invoke result persisted |
580
- | Context (Child) | START | No | Fire-and-forget for performance - parent tracks completion |
581
- | Context (Child) | SUCCEED/FAIL | Yes | Ensures child result available to parent |
582
- | Wait | START | No | Observability only - no blocking needed |
583
- | Wait | SUCCEED | Yes | Ensures wait completion recorded |
584
- | Wait for Condition | START | No | Observability only - condition check is idempotent |
585
- | Wait for Condition | SUCCEED/FAIL | Yes | Ensures condition result persisted |
586
- | Empty Checkpoint | N/A | Yes (default) | Refreshes checkpoint token and operations list |
587
-
588
- ### Synchronous vs Asynchronous Checkpoints
589
-
590
- **Synchronous Checkpoints (is_sync=True, default)**:
591
- - Block the caller until the checkpoint is processed by the background thread
592
- - Ensure the checkpoint is persisted before continuing execution
593
- - Safe default for correctness
594
- - Used for critical operations where confirmation is required
595
-
596
- **Asynchronous Checkpoints (is_sync=False, opt-in)**:
597
- - Return immediately without waiting for the checkpoint to complete
598
- - Performance optimization for specific use cases
599
- - Used for observability checkpoints and fire-and-forget operations
600
- - Only safe when the operation is idempotent or non-critical
601
-
602
- ### Checkpoint Batching
603
-
604
- The SDK uses a background thread to batch multiple checkpoint operations into a single API call for efficiency. This reduces API overhead and
605
- improves throughput.
606
-
607
- ```mermaid
608
- sequenceDiagram
609
- participant MT as Main Thread
610
- participant Q as Checkpoint Queue
611
- participant BT as Background Thread
612
- participant API as Durable Functions API
613
-
614
- Note over MT,API: Synchronous Checkpoint Flow
615
- MT->>Q: Enqueue operation + completion event
616
- MT->>MT: Block on completion event
617
- BT->>Q: Collect batch (up to 1 second or 750KB)
618
- BT->>API: POST /checkpoint (batched operations)
619
- API-->>BT: New checkpoint token + operations
620
- BT->>BT: Update execution state
621
- BT->>MT: Signal completion event
622
- MT->>MT: Resume execution
623
-
624
- Note over MT,API: Asynchronous Checkpoint Flow
625
- MT->>Q: Enqueue operation (no event)
626
- MT->>MT: Continue immediately
627
- BT->>Q: Collect batch (up to 1 second or 750KB)
628
- BT->>API: POST /checkpoint (batched operations)
629
- API-->>BT: New checkpoint token + operations
630
- BT->>BT: Update execution state
631
- ```
632
-
633
- ### Batching Configuration
634
-
635
- Checkpoint batching is controlled by `CheckpointBatcherConfig`:
636
-
637
- ```python
638
- @dataclass(frozen=True)
639
- class CheckpointBatcherConfig:
640
- max_batch_size_bytes: int = 750 * 1024 # 750KB
641
- max_batch_time_seconds: float = 1.0 # 1 second
642
- max_batch_operations: int | float = float("inf") # No limit
643
- ```
644
-
645
- The background thread collects operations until one of these limits is reached:
646
- 1. Batch size exceeds 750KB
647
- 2. 1 second has elapsed since the first operation
648
- 3. Maximum operation count is reached (unlimited by default)
649
-
650
- ### Concurrency Management
651
-
652
- The checkpointing system handles concurrent operations (map/parallel) by tracking parent-child relationships:
653
-
654
- 1. When a CONTEXT operation completes (SUCCEED/FAIL), all descendant operations are marked as orphaned
655
- 2. Orphaned operations are rejected if they attempt to checkpoint
656
- 3. This prevents child operations from checkpointing after their parent has already completed
657
- 4. Uses a single lock (`_parent_done_lock`) to coordinate completion and checkpoint validation
658
-
659
- ### Error Handling
660
-
661
- When a checkpoint fails in the background thread:
662
-
663
- 1. **Error Signaling**: The background thread creates a `BackgroundThreadError` wrapping the original exception
664
- 2. **Event Notification**: All completion events (both in the current batch and queued operations) are signaled with this error
665
- 3. **Immediate Propagation**: Synchronous callers waiting on `create_checkpoint(is_sync=True)` immediately receive the `BackgroundThreadError`
666
- 4. **Future Prevention**: A failure event (`_checkpointing_failed`) is set to prevent any future checkpoint attempts
667
- 5. **Clean Termination**: The background thread exits cleanly after signaling all waiting operations
668
-
669
- For **synchronous operations** (default `is_sync=True`):
670
- - The main thread receives `BackgroundThreadError` immediately when calling `create_checkpoint()`
671
- - This prevents further execution with corrupted state
672
-
673
- For **asynchronous operations** (`is_sync=False`):
674
- - The error is detected on the next synchronous checkpoint attempt
675
- - The `_checkpointing_failed` event causes immediate failure before queuing
676
-
677
- This ensures no code continues executing after a checkpoint failure, maintaining execution state integrity.
678
-
679
- ## License
680
-
681
- This project is licensed under the [Apache-2.0 License](LICENSE).