aws-durable-execution-sdk-python 1.1.2__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aws_durable_execution_sdk_python/__about__.py +1 -1
- aws_durable_execution_sdk_python/__init__.py +7 -1
- aws_durable_execution_sdk_python/concurrency/executor.py +6 -0
- aws_durable_execution_sdk_python/concurrency/models.py +88 -17
- aws_durable_execution_sdk_python/context.py +26 -5
- aws_durable_execution_sdk_python/logger.py +1 -1
- aws_durable_execution_sdk_python/operation/step.py +3 -2
- aws_durable_execution_sdk_python/operation/wait_for_condition.py +3 -2
- aws_durable_execution_sdk_python/serdes.py +1 -5
- aws_durable_execution_sdk_python/state.py +76 -34
- {aws_durable_execution_sdk_python-1.1.2.dist-info → aws_durable_execution_sdk_python-1.4.0.dist-info}/METADATA +1 -1
- {aws_durable_execution_sdk_python-1.1.2.dist-info → aws_durable_execution_sdk_python-1.4.0.dist-info}/RECORD +15 -15
- {aws_durable_execution_sdk_python-1.1.2.dist-info → aws_durable_execution_sdk_python-1.4.0.dist-info}/WHEEL +1 -1
- {aws_durable_execution_sdk_python-1.1.2.dist-info → aws_durable_execution_sdk_python-1.4.0.dist-info}/licenses/LICENSE +0 -0
- {aws_durable_execution_sdk_python-1.1.2.dist-info → aws_durable_execution_sdk_python-1.4.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
"""AWS Lambda Durable Executions Python SDK."""
|
|
2
2
|
|
|
3
|
+
# Package metadata
|
|
4
|
+
from aws_durable_execution_sdk_python.__about__ import __version__
|
|
5
|
+
|
|
3
6
|
# Main context - used in every durable function
|
|
4
7
|
# Helper decorators - commonly used for step functions
|
|
8
|
+
# Concurrency
|
|
9
|
+
from aws_durable_execution_sdk_python.concurrency.models import BatchResult
|
|
5
10
|
from aws_durable_execution_sdk_python.context import (
|
|
6
11
|
DurableContext,
|
|
7
12
|
durable_step,
|
|
@@ -20,7 +25,7 @@ from aws_durable_execution_sdk_python.exceptions import (
|
|
|
20
25
|
from aws_durable_execution_sdk_python.execution import durable_execution
|
|
21
26
|
|
|
22
27
|
# Essential context types - passed to user functions
|
|
23
|
-
from aws_durable_execution_sdk_python.types import
|
|
28
|
+
from aws_durable_execution_sdk_python.types import StepContext
|
|
24
29
|
|
|
25
30
|
__all__ = [
|
|
26
31
|
"BatchResult",
|
|
@@ -29,6 +34,7 @@ __all__ = [
|
|
|
29
34
|
"InvocationError",
|
|
30
35
|
"StepContext",
|
|
31
36
|
"ValidationError",
|
|
37
|
+
"__version__",
|
|
32
38
|
"durable_execution",
|
|
33
39
|
"durable_step",
|
|
34
40
|
"durable_wait_for_callback",
|
|
@@ -30,6 +30,7 @@ from aws_durable_execution_sdk_python.identifier import OperationIdentifier
|
|
|
30
30
|
from aws_durable_execution_sdk_python.lambda_service import ErrorObject
|
|
31
31
|
from aws_durable_execution_sdk_python.operation.child import child_handler
|
|
32
32
|
|
|
33
|
+
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
35
|
from collections.abc import Callable
|
|
35
36
|
|
|
@@ -196,6 +197,11 @@ class ConcurrentExecutor(ABC, Generic[CallableType, ResultType]):
|
|
|
196
197
|
"▶️ Executing concurrent operation, items: %d", len(self.executables)
|
|
197
198
|
)
|
|
198
199
|
|
|
200
|
+
# Early return for empty executables
|
|
201
|
+
if not self.executables:
|
|
202
|
+
logger.debug("No items to execute, returning empty result")
|
|
203
|
+
return self._create_result()
|
|
204
|
+
|
|
199
205
|
max_workers = self.max_concurrency or len(self.executables)
|
|
200
206
|
|
|
201
207
|
self.executables_with_state = [
|
|
@@ -114,6 +114,85 @@ class BatchResult(Generic[R], BatchResultProtocol[R]): # noqa: PYI059
|
|
|
114
114
|
completion_reason = CompletionReason(completion_reason_value)
|
|
115
115
|
return cls(batch_items, completion_reason)
|
|
116
116
|
|
|
117
|
+
@staticmethod
|
|
118
|
+
def _get_completion_reason(
|
|
119
|
+
failure_count: int,
|
|
120
|
+
success_count: int,
|
|
121
|
+
completed_count: int,
|
|
122
|
+
total_count: int,
|
|
123
|
+
completion_config: CompletionConfig | None,
|
|
124
|
+
) -> CompletionReason:
|
|
125
|
+
"""
|
|
126
|
+
Determine completion reason based on completion counts.
|
|
127
|
+
|
|
128
|
+
Logic order:
|
|
129
|
+
1. Check failure tolerance FIRST (before checking if all completed)
|
|
130
|
+
2. Check if all completed
|
|
131
|
+
3. Check if minimum successful reached
|
|
132
|
+
4. Default to ALL_COMPLETED
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
failure_count: Number of failed items
|
|
136
|
+
success_count: Number of succeeded items
|
|
137
|
+
completed_count: Total completed (succeeded + failed)
|
|
138
|
+
total_count: Total number of items
|
|
139
|
+
completion_config: Optional completion configuration
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
CompletionReason enum value
|
|
143
|
+
"""
|
|
144
|
+
# STEP 1: Check tolerance first, before checking if all completed
|
|
145
|
+
|
|
146
|
+
# Handle fail-fast behavior (no completion config or empty completion config)
|
|
147
|
+
if completion_config is None:
|
|
148
|
+
if failure_count > 0:
|
|
149
|
+
return CompletionReason.FAILURE_TOLERANCE_EXCEEDED
|
|
150
|
+
else:
|
|
151
|
+
# Check if completion config has any criteria set
|
|
152
|
+
has_any_completion_criteria = (
|
|
153
|
+
completion_config.min_successful is not None
|
|
154
|
+
or completion_config.tolerated_failure_count is not None
|
|
155
|
+
or completion_config.tolerated_failure_percentage is not None
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
if not has_any_completion_criteria:
|
|
159
|
+
# Empty completion config - fail fast on any failure
|
|
160
|
+
if failure_count > 0:
|
|
161
|
+
return CompletionReason.FAILURE_TOLERANCE_EXCEEDED
|
|
162
|
+
else:
|
|
163
|
+
# Check specific tolerance thresholds
|
|
164
|
+
if (
|
|
165
|
+
completion_config.tolerated_failure_count is not None
|
|
166
|
+
and failure_count > completion_config.tolerated_failure_count
|
|
167
|
+
):
|
|
168
|
+
return CompletionReason.FAILURE_TOLERANCE_EXCEEDED
|
|
169
|
+
|
|
170
|
+
if (
|
|
171
|
+
completion_config.tolerated_failure_percentage is not None
|
|
172
|
+
and total_count > 0
|
|
173
|
+
):
|
|
174
|
+
failure_percentage = (failure_count / total_count) * 100
|
|
175
|
+
if (
|
|
176
|
+
failure_percentage
|
|
177
|
+
> completion_config.tolerated_failure_percentage
|
|
178
|
+
):
|
|
179
|
+
return CompletionReason.FAILURE_TOLERANCE_EXCEEDED
|
|
180
|
+
|
|
181
|
+
# STEP 2: Check if all completed
|
|
182
|
+
if completed_count == total_count:
|
|
183
|
+
return CompletionReason.ALL_COMPLETED
|
|
184
|
+
|
|
185
|
+
# STEP 3: Check if minimum successful reached
|
|
186
|
+
if (
|
|
187
|
+
completion_config is not None
|
|
188
|
+
and completion_config.min_successful is not None
|
|
189
|
+
and success_count >= completion_config.min_successful
|
|
190
|
+
):
|
|
191
|
+
return CompletionReason.MIN_SUCCESSFUL_REACHED
|
|
192
|
+
|
|
193
|
+
# STEP 4: Default
|
|
194
|
+
return CompletionReason.ALL_COMPLETED
|
|
195
|
+
|
|
117
196
|
@classmethod
|
|
118
197
|
def from_items(
|
|
119
198
|
cls,
|
|
@@ -123,12 +202,8 @@ class BatchResult(Generic[R], BatchResultProtocol[R]): # noqa: PYI059
|
|
|
123
202
|
"""
|
|
124
203
|
Infer completion reason based on batch item statuses and completion config.
|
|
125
204
|
|
|
126
|
-
This follows the same logic as the TypeScript implementation
|
|
127
|
-
- If all items completed: ALL_COMPLETED
|
|
128
|
-
- If minSuccessful threshold met and not all completed: MIN_SUCCESSFUL_REACHED
|
|
129
|
-
- Otherwise: FAILURE_TOLERANCE_EXCEEDED
|
|
205
|
+
This follows the same logic as the TypeScript implementation.
|
|
130
206
|
"""
|
|
131
|
-
|
|
132
207
|
statuses = (item.status for item in items)
|
|
133
208
|
counts = Counter(statuses)
|
|
134
209
|
succeeded_count = counts.get(BatchItemStatus.SUCCEEDED, 0)
|
|
@@ -138,18 +213,14 @@ class BatchResult(Generic[R], BatchResultProtocol[R]): # noqa: PYI059
|
|
|
138
213
|
completed_count = succeeded_count + failed_count
|
|
139
214
|
total_count = started_count + completed_count
|
|
140
215
|
|
|
141
|
-
#
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
)
|
|
149
|
-
completion_reason = CompletionReason.MIN_SUCCESSFUL_REACHED
|
|
150
|
-
else:
|
|
151
|
-
# Otherwise, assume failure tolerance was exceeded
|
|
152
|
-
completion_reason = CompletionReason.FAILURE_TOLERANCE_EXCEEDED
|
|
216
|
+
# Determine completion reason using the same logic as JavaScript SDK
|
|
217
|
+
completion_reason = cls._get_completion_reason(
|
|
218
|
+
failure_count=failed_count,
|
|
219
|
+
success_count=succeeded_count,
|
|
220
|
+
completed_count=completed_count,
|
|
221
|
+
total_count=total_count,
|
|
222
|
+
completion_config=completion_config,
|
|
223
|
+
)
|
|
153
224
|
|
|
154
225
|
return cls(items, completion_reason)
|
|
155
226
|
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import logging
|
|
5
|
+
from dataclasses import dataclass
|
|
5
6
|
from typing import TYPE_CHECKING, Any, Concatenate, Generic, ParamSpec, TypeVar
|
|
6
7
|
|
|
7
8
|
from aws_durable_execution_sdk_python.config import (
|
|
@@ -43,21 +44,21 @@ from aws_durable_execution_sdk_python.serdes import (
|
|
|
43
44
|
)
|
|
44
45
|
from aws_durable_execution_sdk_python.state import ExecutionState # noqa: TCH001
|
|
45
46
|
from aws_durable_execution_sdk_python.threading import OrderedCounter
|
|
47
|
+
from aws_durable_execution_sdk_python.types import Callback as CallbackProtocol
|
|
48
|
+
from aws_durable_execution_sdk_python.types import (
|
|
49
|
+
DurableContext as DurableContextProtocol,
|
|
50
|
+
)
|
|
46
51
|
from aws_durable_execution_sdk_python.types import (
|
|
47
|
-
BatchResult,
|
|
48
52
|
LoggerInterface,
|
|
49
53
|
StepContext,
|
|
50
54
|
WaitForCallbackContext,
|
|
51
55
|
WaitForConditionCheckContext,
|
|
52
56
|
)
|
|
53
|
-
from aws_durable_execution_sdk_python.types import Callback as CallbackProtocol
|
|
54
|
-
from aws_durable_execution_sdk_python.types import (
|
|
55
|
-
DurableContext as DurableContextProtocol,
|
|
56
|
-
)
|
|
57
57
|
|
|
58
58
|
if TYPE_CHECKING:
|
|
59
59
|
from collections.abc import Callable, Sequence
|
|
60
60
|
|
|
61
|
+
from aws_durable_execution_sdk_python.concurrency.models import BatchResult
|
|
61
62
|
from aws_durable_execution_sdk_python.state import CheckpointedResult
|
|
62
63
|
from aws_durable_execution_sdk_python.types import LambdaContext
|
|
63
64
|
from aws_durable_execution_sdk_python.waits import WaitForConditionConfig
|
|
@@ -74,6 +75,20 @@ logger = logging.getLogger(__name__)
|
|
|
74
75
|
PASS_THROUGH_SERDES: SerDes[Any] = PassThroughSerDes()
|
|
75
76
|
|
|
76
77
|
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class ExecutionContext:
|
|
80
|
+
"""Readonly metadata about the current durable execution context.
|
|
81
|
+
|
|
82
|
+
This class provides immutable access to execution-level metadata.
|
|
83
|
+
|
|
84
|
+
Attributes:
|
|
85
|
+
durable_execution_arn: The Amazon Resource Name (ARN) of the current
|
|
86
|
+
durable execution.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
durable_execution_arn: str
|
|
90
|
+
|
|
91
|
+
|
|
77
92
|
def durable_step(
|
|
78
93
|
func: Callable[Concatenate[StepContext, Params], T],
|
|
79
94
|
) -> Callable[Params, Callable[[StepContext], T]]:
|
|
@@ -218,11 +233,13 @@ class DurableContext(DurableContextProtocol):
|
|
|
218
233
|
def __init__(
|
|
219
234
|
self,
|
|
220
235
|
state: ExecutionState,
|
|
236
|
+
execution_context: ExecutionContext,
|
|
221
237
|
lambda_context: LambdaContext | None = None,
|
|
222
238
|
parent_id: str | None = None,
|
|
223
239
|
logger: Logger | None = None,
|
|
224
240
|
) -> None:
|
|
225
241
|
self.state: ExecutionState = state
|
|
242
|
+
self.execution_context: ExecutionContext = execution_context
|
|
226
243
|
self.lambda_context = lambda_context
|
|
227
244
|
self._parent_id: str | None = parent_id
|
|
228
245
|
self._step_counter: OrderedCounter = OrderedCounter()
|
|
@@ -245,6 +262,9 @@ class DurableContext(DurableContextProtocol):
|
|
|
245
262
|
):
|
|
246
263
|
return DurableContext(
|
|
247
264
|
state=state,
|
|
265
|
+
execution_context=ExecutionContext(
|
|
266
|
+
durable_execution_arn=state.durable_execution_arn
|
|
267
|
+
),
|
|
248
268
|
lambda_context=lambda_context,
|
|
249
269
|
parent_id=None,
|
|
250
270
|
)
|
|
@@ -254,6 +274,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
254
274
|
logger.debug("Creating child context for parent %s", parent_id)
|
|
255
275
|
return DurableContext(
|
|
256
276
|
state=self.state,
|
|
277
|
+
execution_context=self.execution_context,
|
|
257
278
|
lambda_context=self.lambda_context,
|
|
258
279
|
parent_id=parent_id,
|
|
259
280
|
logger=self.logger.with_log_info(
|
|
@@ -72,7 +72,7 @@ class Logger(LoggerInterface):
|
|
|
72
72
|
# Use 'operation_name' instead of 'name' as key because the stdlib LogRecord internally reserved 'name' parameter
|
|
73
73
|
extra["operationName"] = info.name
|
|
74
74
|
if info.attempt is not None:
|
|
75
|
-
extra["attempt"] = info.attempt
|
|
75
|
+
extra["attempt"] = info.attempt
|
|
76
76
|
if info.operation_id:
|
|
77
77
|
extra["operationId"] = info.operation_id
|
|
78
78
|
return cls(
|
|
@@ -202,9 +202,10 @@ class StepOperationExecutor(OperationExecutor[T]):
|
|
|
202
202
|
ExecutionError: For fatal errors that should not be retried
|
|
203
203
|
May raise other exceptions that will be handled by retry_handler
|
|
204
204
|
"""
|
|
205
|
-
attempt
|
|
205
|
+
# Get current attempt - checkpointed attempts + 1
|
|
206
|
+
attempt: int = 1
|
|
206
207
|
if checkpointed_result.operation and checkpointed_result.operation.step_details:
|
|
207
|
-
attempt = checkpointed_result.operation.step_details.attempt
|
|
208
|
+
attempt = checkpointed_result.operation.step_details.attempt + 1
|
|
208
209
|
|
|
209
210
|
step_context: StepContext = StepContext(
|
|
210
211
|
logger=self.context_logger.with_log_info(
|
|
@@ -170,10 +170,11 @@ class WaitForConditionOperationExecutor(OperationExecutor[T]):
|
|
|
170
170
|
else:
|
|
171
171
|
current_state = self.config.initial_state
|
|
172
172
|
|
|
173
|
-
# Get attempt number
|
|
173
|
+
# Get attempt number - current attempt is checkpointed attempts + 1
|
|
174
|
+
# The checkpoint stores completed attempts, so the current attempt being executed is one more
|
|
174
175
|
attempt: int = 1
|
|
175
176
|
if checkpointed_result.operation and checkpointed_result.operation.step_details:
|
|
176
|
-
attempt = checkpointed_result.operation.step_details.attempt
|
|
177
|
+
attempt = checkpointed_result.operation.step_details.attempt + 1
|
|
177
178
|
|
|
178
179
|
try:
|
|
179
180
|
# Execute the check function with the injected logger
|
|
@@ -316,11 +316,7 @@ class TypeCodec(Codec):
|
|
|
316
316
|
def decode(self, tag: TypeTag, value: Any) -> Any:
|
|
317
317
|
match tag:
|
|
318
318
|
case (
|
|
319
|
-
TypeTag.NONE
|
|
320
|
-
| TypeTag.STR
|
|
321
|
-
| TypeTag.BOOL
|
|
322
|
-
| TypeTag.INT
|
|
323
|
-
| TypeTag.FLOAT
|
|
319
|
+
TypeTag.NONE | TypeTag.STR | TypeTag.BOOL | TypeTag.INT | TypeTag.FLOAT
|
|
324
320
|
):
|
|
325
321
|
return self.primitive_codec.decode(tag, value)
|
|
326
322
|
case TypeTag.BYTES:
|
|
@@ -45,12 +45,12 @@ class CheckpointBatcherConfig:
|
|
|
45
45
|
Attributes:
|
|
46
46
|
max_batch_size_bytes: Maximum batch size in bytes (default: 750KB)
|
|
47
47
|
max_batch_time_seconds: Maximum time to wait before flushing batch (default: 1.0 second)
|
|
48
|
-
max_batch_operations: Maximum number of operations per batch (default:
|
|
48
|
+
max_batch_operations: Maximum number of operations per batch (default: 250)
|
|
49
49
|
"""
|
|
50
50
|
|
|
51
|
-
max_batch_size_bytes: int = 750 * 1024 # 750KB
|
|
52
|
-
max_batch_time_seconds: float = 1.0
|
|
53
|
-
max_batch_operations: int
|
|
51
|
+
max_batch_size_bytes: int = 750 * 1024 # 750KB
|
|
52
|
+
max_batch_time_seconds: float = 1.0
|
|
53
|
+
max_batch_operations: int = 250
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
@dataclass(frozen=True)
|
|
@@ -592,15 +592,21 @@ class ExecutionState:
|
|
|
592
592
|
batch: list[QueuedOperation] = self._collect_checkpoint_batch()
|
|
593
593
|
|
|
594
594
|
if batch:
|
|
595
|
-
# Extract OperationUpdates
|
|
596
|
-
updates: list[OperationUpdate] = [
|
|
597
|
-
|
|
598
|
-
|
|
595
|
+
# Extract OperationUpdates, excluding empty checkpoints from API call
|
|
596
|
+
updates: list[OperationUpdate] = []
|
|
597
|
+
empty_count = 0
|
|
598
|
+
|
|
599
|
+
for q in batch:
|
|
600
|
+
if q.operation_update is not None:
|
|
601
|
+
updates.append(q.operation_update)
|
|
602
|
+
else:
|
|
603
|
+
empty_count += 1
|
|
599
604
|
|
|
600
605
|
logger.debug(
|
|
601
|
-
"
|
|
602
|
-
len(batch),
|
|
606
|
+
"Sending %d OperationUpdates out of %d operations, excluding %d empty checkpoints",
|
|
603
607
|
len(updates),
|
|
608
|
+
len(batch),
|
|
609
|
+
empty_count,
|
|
604
610
|
)
|
|
605
611
|
|
|
606
612
|
try:
|
|
@@ -687,26 +693,43 @@ class ExecutionState:
|
|
|
687
693
|
operation if queues are empty, then collects additional operations within the time
|
|
688
694
|
window.
|
|
689
695
|
|
|
696
|
+
Empty checkpoints (operation_update=None) are coalesced: the first empty checkpoint
|
|
697
|
+
counts toward the batch operation limit, but subsequent empty checkpoints do not.
|
|
698
|
+
All empty checkpoints remain in the batch so their completion events are signaled.
|
|
699
|
+
This avoids unnecessary batches when many concurrent map/parallel branches resume
|
|
700
|
+
simultaneously and each queues an empty checkpoint.
|
|
701
|
+
|
|
690
702
|
Returns:
|
|
691
703
|
List of QueuedOperation objects ready for batch processing. Returns empty list
|
|
692
704
|
if no operations are available.
|
|
693
705
|
"""
|
|
694
706
|
batch: list[QueuedOperation] = []
|
|
707
|
+
has_empty_checkpoint = False
|
|
695
708
|
total_size = 0
|
|
709
|
+
effective_operation_count = 0 # Operations that count toward batch limit
|
|
696
710
|
|
|
697
711
|
# First, drain overflow queue (FIFO order preserved)
|
|
698
712
|
try:
|
|
699
|
-
while
|
|
713
|
+
while effective_operation_count < self._batcher_config.max_batch_operations:
|
|
700
714
|
overflow_op = self._overflow_queue.get_nowait()
|
|
701
|
-
op_size = self._calculate_operation_size(overflow_op)
|
|
702
|
-
|
|
703
|
-
if total_size + op_size > self._batcher_config.max_batch_size_bytes:
|
|
704
|
-
# Put back and stop
|
|
705
|
-
self._overflow_queue.put(overflow_op)
|
|
706
|
-
break
|
|
707
715
|
|
|
708
|
-
|
|
709
|
-
|
|
716
|
+
if overflow_op.operation_update is None: # Empty checkpoint
|
|
717
|
+
batch.append(overflow_op)
|
|
718
|
+
if not has_empty_checkpoint:
|
|
719
|
+
effective_operation_count += (
|
|
720
|
+
1 # First empty counts toward limit
|
|
721
|
+
)
|
|
722
|
+
has_empty_checkpoint = True
|
|
723
|
+
# Subsequent empties don't count toward limit
|
|
724
|
+
else:
|
|
725
|
+
op_size = self._calculate_operation_size(overflow_op)
|
|
726
|
+
if total_size + op_size > self._batcher_config.max_batch_size_bytes:
|
|
727
|
+
# Put back and stop
|
|
728
|
+
self._overflow_queue.put(overflow_op)
|
|
729
|
+
break
|
|
730
|
+
batch.append(overflow_op)
|
|
731
|
+
total_size += op_size
|
|
732
|
+
effective_operation_count += 1
|
|
710
733
|
except queue.Empty:
|
|
711
734
|
pass
|
|
712
735
|
|
|
@@ -720,7 +743,13 @@ class ExecutionState:
|
|
|
720
743
|
) # Check stop signal every 100ms
|
|
721
744
|
self._checkpoint_queue.task_done()
|
|
722
745
|
batch.append(first_op)
|
|
723
|
-
|
|
746
|
+
|
|
747
|
+
if first_op.operation_update is None:
|
|
748
|
+
has_empty_checkpoint = True
|
|
749
|
+
else:
|
|
750
|
+
total_size += self._calculate_operation_size(first_op)
|
|
751
|
+
|
|
752
|
+
effective_operation_count = 1
|
|
724
753
|
break
|
|
725
754
|
except queue.Empty:
|
|
726
755
|
continue
|
|
@@ -735,7 +764,7 @@ class ExecutionState:
|
|
|
735
764
|
# Collect additional operations within the time window
|
|
736
765
|
while (
|
|
737
766
|
time.time() < batch_deadline
|
|
738
|
-
and
|
|
767
|
+
and effective_operation_count < self._batcher_config.max_batch_operations
|
|
739
768
|
and not self._checkpointing_stopped.is_set()
|
|
740
769
|
):
|
|
741
770
|
remaining_time = min(
|
|
@@ -749,26 +778,39 @@ class ExecutionState:
|
|
|
749
778
|
try:
|
|
750
779
|
additional_op = self._checkpoint_queue.get(timeout=remaining_time)
|
|
751
780
|
self._checkpoint_queue.task_done()
|
|
752
|
-
op_size = self._calculate_operation_size(additional_op)
|
|
753
|
-
|
|
754
|
-
# Check if adding this operation would exceed size limit
|
|
755
|
-
if total_size + op_size > self._batcher_config.max_batch_size_bytes:
|
|
756
|
-
# Put in overflow queue for next batch
|
|
757
|
-
self._overflow_queue.put(additional_op)
|
|
758
|
-
logger.debug(
|
|
759
|
-
"Batch size limit reached, moving operation to overflow queue"
|
|
760
|
-
)
|
|
761
|
-
break
|
|
762
781
|
|
|
763
|
-
|
|
764
|
-
|
|
782
|
+
if additional_op.operation_update is None: # Empty checkpoint
|
|
783
|
+
batch.append(additional_op)
|
|
784
|
+
if not has_empty_checkpoint:
|
|
785
|
+
effective_operation_count += (
|
|
786
|
+
1 # First empty counts toward limit
|
|
787
|
+
)
|
|
788
|
+
has_empty_checkpoint = True
|
|
789
|
+
# Subsequent empties don't count toward limit
|
|
790
|
+
else:
|
|
791
|
+
op_size = self._calculate_operation_size(additional_op)
|
|
792
|
+
# Check if adding this operation would exceed size limit
|
|
793
|
+
if total_size + op_size > self._batcher_config.max_batch_size_bytes:
|
|
794
|
+
# Put in overflow queue for next batch
|
|
795
|
+
self._overflow_queue.put(additional_op)
|
|
796
|
+
logger.debug(
|
|
797
|
+
"Batch size limit reached, moving operation to overflow queue"
|
|
798
|
+
)
|
|
799
|
+
break
|
|
800
|
+
batch.append(additional_op)
|
|
801
|
+
total_size += op_size
|
|
802
|
+
effective_operation_count += 1
|
|
765
803
|
|
|
766
804
|
except queue.Empty:
|
|
767
805
|
break
|
|
768
806
|
|
|
807
|
+
empty_count = sum(1 for q in batch if q.operation_update is None)
|
|
769
808
|
logger.debug(
|
|
770
|
-
"Collected batch of %d operations, total size: %d bytes",
|
|
809
|
+
"Collected batch of %d operations (%d effective, %d non-empty, %d empty), total size: %d bytes",
|
|
771
810
|
len(batch),
|
|
811
|
+
effective_operation_count,
|
|
812
|
+
len(batch) - empty_count,
|
|
813
|
+
empty_count,
|
|
772
814
|
total_size,
|
|
773
815
|
)
|
|
774
816
|
return batch
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aws-durable-execution-sdk-python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: AWS Durable Execution SDK for Python
|
|
5
5
|
Project-URL: Documentation, https://github.com/aws/aws-durable-execution-sdk-python#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/aws/aws-durable-execution-sdk-python/issues
|
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
aws_durable_execution_sdk_python/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
aws_durable_execution_sdk_python/__about__.py,sha256=
|
|
3
|
-
aws_durable_execution_sdk_python/__init__.py,sha256=
|
|
2
|
+
aws_durable_execution_sdk_python/__about__.py,sha256=ZjNnP948vlNHMpNCtGcD-ZfYMICduDyvWTwaUejUdXc,137
|
|
3
|
+
aws_durable_execution_sdk_python/__init__.py,sha256=LMxy0GN8uCDxLkPznYyng-6frbD_0Js7rHeSR5O8oKQ,1206
|
|
4
4
|
aws_durable_execution_sdk_python/config.py,sha256=vd_8IZO5KZU3qWENbWDVtyLglqw3sa6SU-R8CYUgm9g,18938
|
|
5
|
-
aws_durable_execution_sdk_python/context.py,sha256=
|
|
5
|
+
aws_durable_execution_sdk_python/context.py,sha256=U0ox6-QzW5dOzZb0tL-R_MITF9O5iSeH9JmnZ5pS_UQ,23217
|
|
6
6
|
aws_durable_execution_sdk_python/exceptions.py,sha256=AvuXnBHAijXb3TepNTu5CVDb_cpR6Yw6x4W3jELVneQ,13631
|
|
7
7
|
aws_durable_execution_sdk_python/execution.py,sha256=A20fOp6_wXdIcL1YiqJ_RGJbQ09D1lSivBQnUY0FT3Y,18571
|
|
8
8
|
aws_durable_execution_sdk_python/identifier.py,sha256=0NyNTb8mTrUni7PlmFtzvJdW6lk0_XpuAaL89LNc378,324
|
|
9
9
|
aws_durable_execution_sdk_python/lambda_service.py,sha256=FkkcmFxDZLA4JS7mMW2YT9BfqTNHHJtEk9kSADwu5bk,37858
|
|
10
|
-
aws_durable_execution_sdk_python/logger.py,sha256=
|
|
10
|
+
aws_durable_execution_sdk_python/logger.py,sha256=DtPNfJS9PFUfeJqHTl34rSxYRizvVVWAtblV-5zpuY4,4277
|
|
11
11
|
aws_durable_execution_sdk_python/py.typed,sha256=DbCDcAirv769HS1ibzZLXq8VPazYf7JCkKSzj3ctudM,58
|
|
12
12
|
aws_durable_execution_sdk_python/retries.py,sha256=CNJ6msQHVhcdSjMuuQAvR0WvsaN7cFA7QgQTVI8pK2w,5829
|
|
13
|
-
aws_durable_execution_sdk_python/serdes.py,sha256
|
|
14
|
-
aws_durable_execution_sdk_python/state.py,sha256=
|
|
13
|
+
aws_durable_execution_sdk_python/serdes.py,sha256=AgsvqpOLjFR1reUO3utuKt3Q52cPUbZ9_CEjWps40qk,16744
|
|
14
|
+
aws_durable_execution_sdk_python/state.py,sha256=YFpi73RoGqz-cV8eWNHzxwjBAza8zhAm_8KCY_V0PxY,35464
|
|
15
15
|
aws_durable_execution_sdk_python/suspend.py,sha256=PIgc6w5-diZIq58-tPv51dzBSHGW7e4oR0QFam8Vgi0,3323
|
|
16
16
|
aws_durable_execution_sdk_python/threading.py,sha256=cU7vIEWHYco-qqkgoAMyPHSOQ8b_Q_r_rZGG7m1g7I4,7650
|
|
17
17
|
aws_durable_execution_sdk_python/types.py,sha256=8vqwa5q2EikesX_NdhR_njin7l7uHakhWPH_ntHY_PI,5152
|
|
18
18
|
aws_durable_execution_sdk_python/waits.py,sha256=ZO_SWafI0G7rWxItpnL7w4o--eDp2VAjnGZlrBJfufc,3883
|
|
19
19
|
aws_durable_execution_sdk_python/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
-
aws_durable_execution_sdk_python/concurrency/executor.py,sha256=
|
|
21
|
-
aws_durable_execution_sdk_python/concurrency/models.py,sha256=
|
|
20
|
+
aws_durable_execution_sdk_python/concurrency/executor.py,sha256=zaJsunB9YPyMZe9fAur3sZ1btm365RkyET6wCRsUL7s,18349
|
|
21
|
+
aws_durable_execution_sdk_python/concurrency/models.py,sha256=5YGCr6cqgts6d_pvLe_J7PxXFWcECFHnVaJZPIXmVcQ,18158
|
|
22
22
|
aws_durable_execution_sdk_python/operation/__init__.py,sha256=xjtSpukZb6sftgJ5oVPSc6TcgKgVQg4vXHeYIeKOPks,25
|
|
23
23
|
aws_durable_execution_sdk_python/operation/base.py,sha256=5Ppis4xFJoXQK4f7-cF-RYmeF84nAVlepzJgHx_OBYc,7049
|
|
24
24
|
aws_durable_execution_sdk_python/operation/callback.py,sha256=wSSdD7R9cOZuRxVeGqqMGirrEmKaZp8iN34uDc7avH0,6685
|
|
@@ -26,11 +26,11 @@ aws_durable_execution_sdk_python/operation/child.py,sha256=L2RMHhTt_xewAS20T1iZN
|
|
|
26
26
|
aws_durable_execution_sdk_python/operation/invoke.py,sha256=R-31JLlazDWECe8Q242LMqiwuyg1_GzycDNKYJEoz_M,6801
|
|
27
27
|
aws_durable_execution_sdk_python/operation/map.py,sha256=NMp-U3dvv2OK9kiUF1PQfkll1JKzpL7Sh_cxyda_Kyk,5024
|
|
28
28
|
aws_durable_execution_sdk_python/operation/parallel.py,sha256=uzxa_XsTvimC63HpadwCjNhGoUcqxM8P4bvQLVHQTq0,4715
|
|
29
|
-
aws_durable_execution_sdk_python/operation/step.py,sha256=
|
|
29
|
+
aws_durable_execution_sdk_python/operation/step.py,sha256=hnwiLtXNLAOl71FPduxyMYaUggQsrelLeP1W5kRXXlM,15629
|
|
30
30
|
aws_durable_execution_sdk_python/operation/wait.py,sha256=P1R7-BrQ0eadFxQiGLHL3ySkuJUqPvVv7zewAzBeVT8,4193
|
|
31
|
-
aws_durable_execution_sdk_python/operation/wait_for_condition.py,sha256=
|
|
32
|
-
aws_durable_execution_sdk_python-1.
|
|
33
|
-
aws_durable_execution_sdk_python-1.
|
|
34
|
-
aws_durable_execution_sdk_python-1.
|
|
35
|
-
aws_durable_execution_sdk_python-1.
|
|
36
|
-
aws_durable_execution_sdk_python-1.
|
|
31
|
+
aws_durable_execution_sdk_python/operation/wait_for_condition.py,sha256=9t-XsfKQfg66Ys57L61VWA5_JYPvVNnGD6ddFq-h4Xg,11921
|
|
32
|
+
aws_durable_execution_sdk_python-1.4.0.dist-info/METADATA,sha256=nKFN7Mr7GvfCdIfAFBLIZ1i358avvac6j-qqV4hT8ek,6424
|
|
33
|
+
aws_durable_execution_sdk_python-1.4.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
34
|
+
aws_durable_execution_sdk_python-1.4.0.dist-info/licenses/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
|
|
35
|
+
aws_durable_execution_sdk_python-1.4.0.dist-info/licenses/NOTICE,sha256=1CkO1kwu3Q_OHYTj-d-yiBJA_lNN73a4zSntavaD4oc,67
|
|
36
|
+
aws_durable_execution_sdk_python-1.4.0.dist-info/RECORD,,
|
|
File without changes
|