aws-durable-execution-sdk-python 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aws_durable_execution_sdk_python/__about__.py +1 -1
- aws_durable_execution_sdk_python/concurrency/executor.py +22 -8
- aws_durable_execution_sdk_python/context.py +31 -13
- aws_durable_execution_sdk_python/execution.py +230 -125
- aws_durable_execution_sdk_python/identifier.py +4 -4
- aws_durable_execution_sdk_python/lambda_service.py +10 -27
- aws_durable_execution_sdk_python/operation/child.py +30 -0
- aws_durable_execution_sdk_python/operation/invoke.py +6 -0
- aws_durable_execution_sdk_python/plugin.py +169 -35
- aws_durable_execution_sdk_python/plugin_discovery.py +146 -2
- aws_durable_execution_sdk_python/state.py +141 -58
- aws_durable_execution_sdk_python-2.1.0.dist-info/METADATA +261 -0
- {aws_durable_execution_sdk_python-2.0.1.dist-info → aws_durable_execution_sdk_python-2.1.0.dist-info}/RECORD +14 -14
- {aws_durable_execution_sdk_python-2.0.1.dist-info → aws_durable_execution_sdk_python-2.1.0.dist-info}/WHEEL +1 -1
- aws_durable_execution_sdk_python-2.0.1.dist-info/METADATA +0 -156
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextvars
|
|
5
6
|
import heapq
|
|
6
7
|
import logging
|
|
7
8
|
import queue
|
|
@@ -44,7 +45,7 @@ from aws_durable_execution_sdk_python.identifier import (
|
|
|
44
45
|
OperationIdentifier,
|
|
45
46
|
OperationIdNamespace,
|
|
46
47
|
)
|
|
47
|
-
from aws_durable_execution_sdk_python.lambda_service import ErrorObject
|
|
48
|
+
from aws_durable_execution_sdk_python.lambda_service import ErrorObject, OperationType
|
|
48
49
|
from aws_durable_execution_sdk_python.operation.child import child_handler
|
|
49
50
|
|
|
50
51
|
|
|
@@ -183,6 +184,7 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
183
184
|
executable.index
|
|
184
185
|
),
|
|
185
186
|
sub_type=self.sub_type_iteration,
|
|
187
|
+
operation_type=OperationType.CONTEXT,
|
|
186
188
|
parent_id=executor_context._parent_id, # noqa: SLF001
|
|
187
189
|
name=self.get_iteration_name(executable.index),
|
|
188
190
|
)
|
|
@@ -291,13 +293,25 @@ class ConcurrentExecutor(Generic[CallableType, ResultType]):
|
|
|
291
293
|
|
|
292
294
|
def submit(branch: Branch[CallableType, ResultType]) -> None:
|
|
293
295
|
branch.start()
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
296
|
+
if execution_state._plugin_executor._plugins: # noqa: SLF001
|
|
297
|
+
# Every admission/resume gets its own Context. Branch bindings
|
|
298
|
+
# cannot leak into siblings, the coordinator, or reused workers.
|
|
299
|
+
pool.submit(
|
|
300
|
+
contextvars.copy_context().run,
|
|
301
|
+
self._branch_worker,
|
|
302
|
+
execution_state,
|
|
303
|
+
executor_context,
|
|
304
|
+
events,
|
|
305
|
+
branch.executable,
|
|
306
|
+
)
|
|
307
|
+
else:
|
|
308
|
+
pool.submit(
|
|
309
|
+
self._branch_worker,
|
|
310
|
+
execution_state,
|
|
311
|
+
executor_context,
|
|
312
|
+
events,
|
|
313
|
+
branch.executable,
|
|
314
|
+
)
|
|
301
315
|
|
|
302
316
|
try:
|
|
303
317
|
# Only rebuild the items snapshot after a terminal event changes
|
|
@@ -51,7 +51,10 @@ from aws_durable_execution_sdk_python.operation.callback import (
|
|
|
51
51
|
CallbackOperationExecutor,
|
|
52
52
|
wait_for_callback_handler,
|
|
53
53
|
)
|
|
54
|
-
from aws_durable_execution_sdk_python.operation.child import
|
|
54
|
+
from aws_durable_execution_sdk_python.operation.child import (
|
|
55
|
+
ChildOperationExecutor,
|
|
56
|
+
child_handler,
|
|
57
|
+
)
|
|
55
58
|
from aws_durable_execution_sdk_python.operation.invoke import InvokeOperationExecutor
|
|
56
59
|
from aws_durable_execution_sdk_python.operation.map import map_handler
|
|
57
60
|
from aws_durable_execution_sdk_python.operation.parallel import parallel_handler
|
|
@@ -515,12 +518,18 @@ class DurableContext(DurableContextProtocol):
|
|
|
515
518
|
|
|
516
519
|
@contextmanager
|
|
517
520
|
def _replay_aware(
|
|
518
|
-
self,
|
|
521
|
+
self,
|
|
522
|
+
operation_identifier: OperationIdentifier | None = None,
|
|
523
|
+
*,
|
|
524
|
+
is_virtual: bool = False,
|
|
519
525
|
) -> Iterator[None]:
|
|
520
526
|
"""Wrap a single operation with replay-boundary detection.
|
|
521
527
|
|
|
522
528
|
The operation kind is inferred from its own checkpoint (when one
|
|
523
|
-
exists)
|
|
529
|
+
exists). The one per-call-site input is `is_virtual`: a virtual child
|
|
530
|
+
context never checkpoints itself, so with `is_virtual=True` any
|
|
531
|
+
checkpoint at its id is rejected right after the identity check,
|
|
532
|
+
before replay and update hooks run.
|
|
524
533
|
|
|
525
534
|
The boundary has these parts:
|
|
526
535
|
|
|
@@ -555,6 +564,10 @@ class DurableContext(DurableContextProtocol):
|
|
|
555
564
|
if was_replaying and next_operation is not None:
|
|
556
565
|
if operation_identifier is not None:
|
|
557
566
|
operation_identifier.validate_checkpoint(next_operation)
|
|
567
|
+
if is_virtual:
|
|
568
|
+
ChildOperationExecutor.validate_virtual_checkpoint(
|
|
569
|
+
operation_identifier, next_operation
|
|
570
|
+
)
|
|
558
571
|
|
|
559
572
|
# While replaying, an operation that already has a checkpoint was
|
|
560
573
|
# observed in a prior invocation. If the backend says the operation
|
|
@@ -599,7 +612,9 @@ class DurableContext(DurableContextProtocol):
|
|
|
599
612
|
self,
|
|
600
613
|
sub_type: OperationSubType,
|
|
601
614
|
name: str | None = None,
|
|
602
|
-
|
|
615
|
+
*,
|
|
616
|
+
operation_type: OperationType,
|
|
617
|
+
is_virtual: bool = False,
|
|
603
618
|
) -> Iterator[OperationIdentifier]:
|
|
604
619
|
"""Allocate an operation and validate its replay identity before hooks."""
|
|
605
620
|
operation_identifier = OperationIdentifier(
|
|
@@ -609,7 +624,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
609
624
|
name=name,
|
|
610
625
|
operation_type=operation_type,
|
|
611
626
|
)
|
|
612
|
-
with self._replay_aware(operation_identifier):
|
|
627
|
+
with self._replay_aware(operation_identifier, is_virtual=is_virtual):
|
|
613
628
|
yield operation_identifier
|
|
614
629
|
|
|
615
630
|
# endregion replay status
|
|
@@ -636,7 +651,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
636
651
|
if not config:
|
|
637
652
|
config = CallbackConfig()
|
|
638
653
|
with self._operation_replay_aware(
|
|
639
|
-
OperationSubType.CALLBACK, name
|
|
654
|
+
OperationSubType.CALLBACK, name, operation_type=OperationType.CALLBACK
|
|
640
655
|
) as operation_identifier:
|
|
641
656
|
executor: CallbackOperationExecutor = CallbackOperationExecutor(
|
|
642
657
|
state=self.state,
|
|
@@ -672,7 +687,9 @@ class DurableContext(DurableContextProtocol):
|
|
|
672
687
|
if not config:
|
|
673
688
|
config = InvokeConfig[P, R]()
|
|
674
689
|
with self._operation_replay_aware(
|
|
675
|
-
OperationSubType.CHAINED_INVOKE,
|
|
690
|
+
OperationSubType.CHAINED_INVOKE,
|
|
691
|
+
name,
|
|
692
|
+
operation_type=OperationType.CHAINED_INVOKE,
|
|
676
693
|
) as operation_identifier:
|
|
677
694
|
executor: InvokeOperationExecutor[R] = InvokeOperationExecutor(
|
|
678
695
|
function_name=function_name,
|
|
@@ -700,7 +717,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
700
717
|
config.completion_config._validate_for_total(len(inputs))
|
|
701
718
|
|
|
702
719
|
with self._operation_replay_aware(
|
|
703
|
-
OperationSubType.MAP, map_name
|
|
720
|
+
OperationSubType.MAP, map_name, operation_type=OperationType.CONTEXT
|
|
704
721
|
) as operation_identifier:
|
|
705
722
|
operation_id = operation_identifier.operation_id
|
|
706
723
|
map_context = self.create_child_context(operation_id=operation_id)
|
|
@@ -751,7 +768,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
751
768
|
config.completion_config._validate_for_total(len(functions))
|
|
752
769
|
|
|
753
770
|
with self._operation_replay_aware(
|
|
754
|
-
OperationSubType.PARALLEL, name
|
|
771
|
+
OperationSubType.PARALLEL, name, operation_type=OperationType.CONTEXT
|
|
755
772
|
) as operation_identifier:
|
|
756
773
|
operation_id = operation_identifier.operation_id
|
|
757
774
|
parallel_context = self.create_child_context(operation_id=operation_id)
|
|
@@ -811,14 +828,15 @@ class DurableContext(DurableContextProtocol):
|
|
|
811
828
|
if config and config.sub_type
|
|
812
829
|
else OperationSubType.RUN_IN_CHILD_CONTEXT
|
|
813
830
|
)
|
|
831
|
+
is_virtual: bool = config.is_virtual if config else False
|
|
814
832
|
|
|
815
833
|
with self._operation_replay_aware(
|
|
816
834
|
sub_type,
|
|
817
835
|
step_name,
|
|
818
836
|
operation_type=OperationType.CONTEXT,
|
|
837
|
+
is_virtual=is_virtual,
|
|
819
838
|
) as operation_identifier:
|
|
820
839
|
operation_id = operation_identifier.operation_id
|
|
821
|
-
is_virtual: bool = config.is_virtual if config else False
|
|
822
840
|
|
|
823
841
|
def callable_with_child_context():
|
|
824
842
|
return func(
|
|
@@ -845,7 +863,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
845
863
|
if not config:
|
|
846
864
|
config = StepConfig()
|
|
847
865
|
with self._operation_replay_aware(
|
|
848
|
-
OperationSubType.STEP, step_name
|
|
866
|
+
OperationSubType.STEP, step_name, operation_type=OperationType.STEP
|
|
849
867
|
) as operation_identifier:
|
|
850
868
|
executor: StepOperationExecutor[T] = StepOperationExecutor(
|
|
851
869
|
func=func,
|
|
@@ -868,7 +886,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
868
886
|
msg = "duration must be at least 1 second"
|
|
869
887
|
raise ValidationError(msg)
|
|
870
888
|
with self._operation_replay_aware(
|
|
871
|
-
OperationSubType.WAIT, name
|
|
889
|
+
OperationSubType.WAIT, name, operation_type=OperationType.WAIT
|
|
872
890
|
) as operation_identifier:
|
|
873
891
|
wait_seconds = duration.seconds
|
|
874
892
|
executor: WaitOperationExecutor = WaitOperationExecutor(
|
|
@@ -938,7 +956,7 @@ class DurableContext(DurableContextProtocol):
|
|
|
938
956
|
raise ValidationError(msg)
|
|
939
957
|
|
|
940
958
|
with self._operation_replay_aware(
|
|
941
|
-
OperationSubType.WAIT_FOR_CONDITION, name
|
|
959
|
+
OperationSubType.WAIT_FOR_CONDITION, name, operation_type=OperationType.STEP
|
|
942
960
|
) as operation_identifier:
|
|
943
961
|
executor: WaitForConditionOperationExecutor[T] = (
|
|
944
962
|
WaitForConditionOperationExecutor(
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import contextlib
|
|
4
|
+
import contextvars
|
|
4
5
|
import functools
|
|
5
6
|
import json
|
|
6
7
|
import logging
|
|
7
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
8
|
+
from concurrent.futures import Future, ThreadPoolExecutor, wait
|
|
8
9
|
from dataclasses import dataclass, field
|
|
9
10
|
from typing import TYPE_CHECKING, Any
|
|
10
11
|
|
|
@@ -188,9 +189,9 @@ def durable_execution(
|
|
|
188
189
|
|
|
189
190
|
logger.debug("Starting durable execution handler...")
|
|
190
191
|
|
|
191
|
-
|
|
192
|
+
configured_plugins = load_configured_plugins(plugins)
|
|
193
|
+
plugin_executor = PluginExecutor(configured_plugins)
|
|
192
194
|
|
|
193
|
-
@plugin_executor.handle_durable_output
|
|
194
195
|
def wrapper(event: Any, context: LambdaContext) -> MutableMapping[str, Any]:
|
|
195
196
|
invocation_input: DurableExecutionInvocationInput
|
|
196
197
|
service_client: DurableServiceClient
|
|
@@ -279,88 +280,75 @@ def durable_execution(
|
|
|
279
280
|
),
|
|
280
281
|
)
|
|
281
282
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
ThreadPoolExecutor(
|
|
285
|
-
max_workers=2, thread_name_prefix="dex-handler"
|
|
286
|
-
) as executor,
|
|
287
|
-
contextlib.closing(execution_state) as execution_state,
|
|
288
|
-
):
|
|
289
|
-
execution_operation = execution_state.get_execution_operation()
|
|
290
|
-
|
|
291
|
-
# execute the plugins
|
|
292
|
-
plugin_executor.on_invocation_start(
|
|
293
|
-
execution_arn=invocation_input.durable_execution_arn,
|
|
294
|
-
lambda_context=context,
|
|
295
|
-
execution_start_time=(
|
|
296
|
-
execution_operation.start_timestamp
|
|
297
|
-
if execution_operation is not None
|
|
298
|
-
else None
|
|
299
|
-
),
|
|
300
|
-
is_first_invocation=not has_prior_operations,
|
|
301
|
-
execution_input=input_event,
|
|
302
|
-
# Read the map through a callable rather than snapshotting it
|
|
303
|
-
# here: the invocation-end hook needs the state as of the end of
|
|
304
|
-
# the invocation, and neither hook pays for the conversion until
|
|
305
|
-
# a plugin actually reads it.
|
|
306
|
-
operations_provider=lambda: execution_state.operations,
|
|
307
|
-
updated_operation_ids=invocation_input.updated_operation_ids,
|
|
308
|
-
)
|
|
309
|
-
# Thread 1: Run background checkpoint processing
|
|
310
|
-
executor.submit(execution_state.checkpoint_batches_forever)
|
|
283
|
+
def invoke_handler() -> MutableMapping[str, Any]:
|
|
284
|
+
output: MutableMapping[str, Any]
|
|
311
285
|
|
|
312
|
-
# Thread 2: Execute user function
|
|
313
286
|
logger.debug(
|
|
314
287
|
"%s entering user-space...", invocation_input.durable_execution_arn
|
|
315
288
|
)
|
|
316
|
-
user_future = executor.submit(func, input_event, durable_context)
|
|
317
|
-
|
|
318
|
-
logger.debug(
|
|
319
|
-
"%s waiting for user code completion...",
|
|
320
|
-
invocation_input.durable_execution_arn,
|
|
321
|
-
)
|
|
322
|
-
|
|
323
289
|
try:
|
|
324
290
|
# Background checkpointing errors will propagate through CompletionEvent.wait() as BackgroundThreadError
|
|
325
|
-
result =
|
|
291
|
+
result = func(input_event, durable_context)
|
|
326
292
|
|
|
327
293
|
# done with userland
|
|
328
294
|
logger.debug(
|
|
329
295
|
"%s exiting user-space...",
|
|
330
296
|
invocation_input.durable_execution_arn,
|
|
331
297
|
)
|
|
332
|
-
|
|
333
|
-
#
|
|
334
|
-
#
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
298
|
+
|
|
299
|
+
# The handler may finish while background checkpoint work is
|
|
300
|
+
# still running for async operations it did not await, such as
|
|
301
|
+
# early-completing map or parallel branches. If one of those
|
|
302
|
+
# checkpoint responses omitted the next checkpoint token, this
|
|
303
|
+
# invocation can no longer record anything else with the
|
|
304
|
+
# service, including the handler's final result.
|
|
305
|
+
#
|
|
306
|
+
# In that case, return PENDING instead of SUCCEEDED. The next
|
|
307
|
+
# invocation will replay from the last durable checkpoint and
|
|
308
|
+
# produce the result again. This check catches revocation
|
|
309
|
+
# already observed before we build the terminal response; the
|
|
310
|
+
# post-with check below catches revocation discovered during
|
|
311
|
+
# cleanup of in-flight checkpoint work.
|
|
312
|
+
if execution_state.is_checkpoint_token_revoked:
|
|
339
313
|
logger.debug(
|
|
340
|
-
"
|
|
341
|
-
|
|
342
|
-
LAMBDA_RESPONSE_SIZE_LIMIT,
|
|
343
|
-
)
|
|
344
|
-
success_operation = OperationUpdate.create_execution_succeed(
|
|
345
|
-
payload=serialized_result
|
|
314
|
+
"Checkpoint token revoked; ending invocation with "
|
|
315
|
+
"PENDING instead of SUCCEEDED."
|
|
346
316
|
)
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
#
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
317
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
318
|
+
else:
|
|
319
|
+
serialized_result = json.dumps(result)
|
|
320
|
+
# large response handling here. Remember if checkpointing to complete, NOT to include
|
|
321
|
+
# payload in response
|
|
322
|
+
if (
|
|
323
|
+
serialized_result
|
|
324
|
+
and len(serialized_result) > LAMBDA_RESPONSE_SIZE_LIMIT
|
|
325
|
+
):
|
|
326
|
+
logger.debug(
|
|
327
|
+
"Response size (%s bytes) exceeds Lambda limit (%s) bytes). Checkpointing result.",
|
|
328
|
+
len(serialized_result),
|
|
329
|
+
LAMBDA_RESPONSE_SIZE_LIMIT,
|
|
354
330
|
)
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
result=
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
331
|
+
success_operation = OperationUpdate.create_execution_succeed(
|
|
332
|
+
payload=serialized_result
|
|
333
|
+
)
|
|
334
|
+
# Checkpoint large result with blocking (is_sync=True, default).
|
|
335
|
+
# Must ensure the result is persisted before returning to Lambda.
|
|
336
|
+
# Large results exceed Lambda response limits and must be stored durably
|
|
337
|
+
# before the execution completes.
|
|
338
|
+
try:
|
|
339
|
+
execution_state.create_checkpoint(
|
|
340
|
+
success_operation, is_sync=True
|
|
341
|
+
)
|
|
342
|
+
except CheckpointError as e:
|
|
343
|
+
output = handle_checkpoint_error(e).to_dict()
|
|
344
|
+
else:
|
|
345
|
+
output = DurableExecutionInvocationOutput.create_succeeded(
|
|
346
|
+
result=""
|
|
347
|
+
).to_dict()
|
|
348
|
+
else:
|
|
349
|
+
output = DurableExecutionInvocationOutput.create_succeeded(
|
|
350
|
+
result=serialized_result
|
|
351
|
+
).to_dict()
|
|
364
352
|
|
|
365
353
|
except BackgroundThreadError as bg_error:
|
|
366
354
|
# Background checkpoint system failed - propagated through CompletionEvent
|
|
@@ -378,85 +366,202 @@ def durable_execution(
|
|
|
378
366
|
"without retry.",
|
|
379
367
|
extra=bg_error.source_exception.build_logger_extras(),
|
|
380
368
|
)
|
|
381
|
-
|
|
369
|
+
output = DurableExecutionInvocationOutput(
|
|
382
370
|
status=InvocationStatus.FAILED,
|
|
383
371
|
error=ErrorObject.from_exception(bg_error.source_exception),
|
|
384
372
|
).to_dict()
|
|
373
|
+
else:
|
|
374
|
+
raise bg_error.source_exception from bg_error
|
|
385
375
|
else:
|
|
386
376
|
logger.exception("Checkpoint processing failed")
|
|
387
|
-
|
|
377
|
+
raise bg_error.source_exception from bg_error
|
|
388
378
|
|
|
389
379
|
except SuspendExecution:
|
|
390
380
|
# User code suspended - stop background checkpointing thread
|
|
391
381
|
logger.debug("Suspending execution...")
|
|
392
|
-
|
|
393
|
-
status=InvocationStatus.PENDING
|
|
394
|
-
).to_dict()
|
|
382
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
395
383
|
|
|
396
384
|
except CheckpointError as e:
|
|
397
|
-
# Checkpoint system is broken - stop background thread and exit immediately
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
385
|
+
# Checkpoint system is broken - stop background thread and exit immediately.
|
|
386
|
+
if execution_state.is_checkpoint_token_revoked:
|
|
387
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
388
|
+
else:
|
|
389
|
+
logger.exception(
|
|
390
|
+
"Checkpoint system failed",
|
|
391
|
+
extra=e.build_logger_extras(),
|
|
392
|
+
)
|
|
393
|
+
output = handle_checkpoint_error(e).to_dict()
|
|
403
394
|
except InvocationError as e:
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
395
|
+
if execution_state.is_checkpoint_token_revoked:
|
|
396
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
397
|
+
else:
|
|
398
|
+
# Non-retryable Durable API errors (e.g., customer configuration issues,
|
|
399
|
+
# 4xx client errors) will never succeed on retry — fail the execution immediately.
|
|
400
|
+
if not e.is_retryable():
|
|
401
|
+
logger.exception(
|
|
402
|
+
"Non-retryable Durable API error. Must fail execution without retry.",
|
|
403
|
+
extra=e.build_logger_extras(), # type: ignore[attr-defined]
|
|
404
|
+
)
|
|
405
|
+
output = DurableExecutionInvocationOutput(
|
|
406
|
+
status=InvocationStatus.FAILED,
|
|
407
|
+
error=ErrorObject.from_exception(e),
|
|
408
|
+
).to_dict()
|
|
409
|
+
else:
|
|
410
|
+
logger.exception("Invocation error. Must terminate.")
|
|
411
|
+
# Throw the error to trigger Lambda retry
|
|
412
|
+
raise
|
|
413
|
+
except ExecutionError as e:
|
|
414
|
+
if execution_state.is_checkpoint_token_revoked:
|
|
415
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
416
|
+
else:
|
|
407
417
|
logger.exception(
|
|
408
|
-
"
|
|
409
|
-
extra=e.build_logger_extras(), # type: ignore[attr-defined]
|
|
418
|
+
"Execution error. Must fail execution without retry."
|
|
410
419
|
)
|
|
411
|
-
|
|
420
|
+
output = DurableExecutionInvocationOutput(
|
|
412
421
|
status=InvocationStatus.FAILED,
|
|
413
422
|
error=ErrorObject.from_exception(e),
|
|
414
423
|
).to_dict()
|
|
415
|
-
logger.exception("Invocation error. Must terminate.")
|
|
416
|
-
# Throw the error to trigger Lambda retry
|
|
417
|
-
raise
|
|
418
|
-
except ExecutionError as e:
|
|
419
|
-
logger.exception("Execution error. Must fail execution without retry.")
|
|
420
|
-
return DurableExecutionInvocationOutput(
|
|
421
|
-
status=InvocationStatus.FAILED,
|
|
422
|
-
error=ErrorObject.from_exception(e),
|
|
423
|
-
).to_dict()
|
|
424
424
|
except Exception as e:
|
|
425
425
|
# all user-space errors go here
|
|
426
|
-
|
|
426
|
+
if execution_state.is_checkpoint_token_revoked:
|
|
427
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
428
|
+
else:
|
|
429
|
+
logger.exception("Execution failed")
|
|
427
430
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
+
result = DurableExecutionInvocationOutput(
|
|
432
|
+
status=InvocationStatus.FAILED,
|
|
433
|
+
error=ErrorObject.from_exception(e),
|
|
434
|
+
).to_dict()
|
|
431
435
|
|
|
432
|
-
|
|
436
|
+
serialized_result = json.dumps(result)
|
|
433
437
|
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
438
|
+
if (
|
|
439
|
+
serialized_result
|
|
440
|
+
and len(serialized_result) > LAMBDA_RESPONSE_SIZE_LIMIT
|
|
441
|
+
):
|
|
442
|
+
logger.debug(
|
|
443
|
+
"Response size (%s bytes) exceeds Lambda limit (%s) bytes). Checkpointing result.",
|
|
444
|
+
len(serialized_result),
|
|
445
|
+
LAMBDA_RESPONSE_SIZE_LIMIT,
|
|
446
|
+
)
|
|
447
|
+
failed_operation = OperationUpdate.create_execution_fail(
|
|
448
|
+
error=ErrorObject.from_exception(e)
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
# Checkpoint large result with blocking (is_sync=True, default).
|
|
452
|
+
# Must ensure the result is persisted before returning to Lambda.
|
|
453
|
+
# Large results exceed Lambda response limits and must be stored durably
|
|
454
|
+
# before the execution completes.
|
|
455
|
+
try:
|
|
456
|
+
execution_state.create_checkpoint_sync(failed_operation)
|
|
457
|
+
except SuspendExecution:
|
|
458
|
+
# Revocation can occur after the check at the top of this handler and
|
|
459
|
+
# before this checkpoint. In that case create_checkpoint_sync raises
|
|
460
|
+
# SuspendExecution. So return PENDING instead of FAILED.
|
|
461
|
+
output = DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
462
|
+
except CheckpointError as checkpoint_error:
|
|
463
|
+
output = handle_checkpoint_error(checkpoint_error).to_dict()
|
|
464
|
+
else:
|
|
465
|
+
output = DurableExecutionInvocationOutput(
|
|
466
|
+
status=InvocationStatus.FAILED
|
|
467
|
+
).to_dict()
|
|
468
|
+
else:
|
|
469
|
+
output = result
|
|
470
|
+
|
|
471
|
+
return output
|
|
472
|
+
|
|
473
|
+
def invoke_worker() -> MutableMapping[str, Any]:
|
|
474
|
+
# This task owns Start, user cleanup, output preparation, resource
|
|
475
|
+
# cleanup and End. The caller owns the executor that runs this task.
|
|
476
|
+
with plugin_executor.run():
|
|
477
|
+
checkpoint_future: Future[None] | None = None
|
|
478
|
+
|
|
479
|
+
def wait_for_checkpoint() -> None:
|
|
480
|
+
if checkpoint_future is not None:
|
|
481
|
+
# Match the former executor join: observed checkpoint
|
|
482
|
+
# errors retain their existing classification paths.
|
|
483
|
+
wait((checkpoint_future,))
|
|
484
|
+
|
|
485
|
+
try:
|
|
486
|
+
with contextlib.ExitStack() as resources:
|
|
487
|
+
# LIFO cleanup: branches join with checkpointing alive;
|
|
488
|
+
# then close stops checkpointing and the wait completes.
|
|
489
|
+
resources.callback(wait_for_checkpoint)
|
|
490
|
+
resources.callback(
|
|
491
|
+
plugin_executor._run_in_invocation_context,
|
|
492
|
+
execution_state.close,
|
|
493
|
+
)
|
|
494
|
+
execution_operation = execution_state.get_execution_operation()
|
|
495
|
+
|
|
496
|
+
# execute the plugins
|
|
497
|
+
plugin_executor.on_invocation_start(
|
|
498
|
+
execution_arn=invocation_input.durable_execution_arn,
|
|
499
|
+
lambda_context=context,
|
|
500
|
+
execution_start_time=(
|
|
501
|
+
execution_operation.start_timestamp
|
|
502
|
+
if execution_operation is not None
|
|
503
|
+
else None
|
|
504
|
+
),
|
|
505
|
+
is_first_invocation=not has_prior_operations,
|
|
506
|
+
execution_input=input_event,
|
|
507
|
+
# Read the map through a callable rather than snapshotting it
|
|
508
|
+
# here: the invocation-end hook needs the state as of the end of
|
|
509
|
+
# the invocation, and neither hook pays for the conversion until
|
|
510
|
+
# a plugin actually reads it.
|
|
511
|
+
operations_provider=lambda: execution_state.operations,
|
|
512
|
+
updated_operation_ids=invocation_input.updated_operation_ids,
|
|
513
|
+
)
|
|
514
|
+
# No checkpoint work starts until all Start hooks finish.
|
|
515
|
+
checkpoint_future = executor.submit(
|
|
516
|
+
execution_state.checkpoint_batches_forever
|
|
517
|
+
)
|
|
518
|
+
output = plugin_executor._run_in_invocation_context(
|
|
519
|
+
invoke_handler
|
|
520
|
+
)
|
|
521
|
+
# The ExitStack has now closed execution_state and joined the
|
|
522
|
+
# checkpoint thread. So any revocation observed while draining
|
|
523
|
+
# in-flight checkpoint work is visible here. A terminal status
|
|
524
|
+
# built before that point can no longer be recorded with the
|
|
525
|
+
# service. So report PENDING instead; the next invocation
|
|
526
|
+
# replays from the last durable checkpoint.
|
|
527
|
+
if (
|
|
528
|
+
output.get("Status")
|
|
529
|
+
in {
|
|
530
|
+
InvocationStatus.SUCCEEDED.value,
|
|
531
|
+
InvocationStatus.FAILED.value,
|
|
532
|
+
}
|
|
533
|
+
and execution_state.is_checkpoint_token_revoked
|
|
534
|
+
):
|
|
535
|
+
logger.debug(
|
|
536
|
+
"Checkpoint token revoked during invocation cleanup; ending "
|
|
537
|
+
"invocation with PENDING instead of terminal status."
|
|
538
|
+
)
|
|
539
|
+
output = (
|
|
540
|
+
DurableExecutionInvocationOutput.create_pending().to_dict()
|
|
541
|
+
)
|
|
542
|
+
plugin_executor.on_invocation_end(
|
|
543
|
+
DurableExecutionInvocationOutput.from_dict(output)
|
|
442
544
|
)
|
|
443
|
-
|
|
444
|
-
|
|
545
|
+
return output
|
|
546
|
+
except Exception as error:
|
|
547
|
+
plugin_executor.on_invocation_end(
|
|
548
|
+
DurableExecutionInvocationOutput.create_retry(
|
|
549
|
+
ErrorObject.from_exception(error)
|
|
550
|
+
)
|
|
445
551
|
)
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
return result
|
|
552
|
+
raise
|
|
553
|
+
|
|
554
|
+
with ThreadPoolExecutor(
|
|
555
|
+
max_workers=2, thread_name_prefix="dex-handler"
|
|
556
|
+
) as executor:
|
|
557
|
+
if configured_plugins:
|
|
558
|
+
invocation_future = executor.submit(
|
|
559
|
+
contextvars.copy_context().run, invoke_worker
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
# Preserve the original fresh-worker context without plugins.
|
|
563
|
+
invocation_future = executor.submit(invoke_worker)
|
|
564
|
+
return invocation_future.result()
|
|
460
565
|
|
|
461
566
|
return wrapper
|
|
462
567
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import hashlib
|
|
6
|
-
from dataclasses import dataclass
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
7
|
|
|
8
8
|
from aws_durable_execution_sdk_python.exceptions import (
|
|
9
9
|
NonDeterministicExecutionError,
|
|
@@ -34,17 +34,17 @@ class OperationIdNamespace:
|
|
|
34
34
|
|
|
35
35
|
@dataclass(frozen=True)
|
|
36
36
|
class OperationIdentifier:
|
|
37
|
-
"""
|
|
37
|
+
"""Operation identity with an explicit type independent of its subtype."""
|
|
38
38
|
|
|
39
39
|
operation_id: str
|
|
40
40
|
sub_type: OperationSubType
|
|
41
41
|
parent_id: str | None = None
|
|
42
42
|
name: str | None = None
|
|
43
|
-
operation_type: OperationType
|
|
43
|
+
operation_type: OperationType = field(kw_only=True)
|
|
44
44
|
|
|
45
45
|
@property
|
|
46
46
|
def type(self) -> OperationType:
|
|
47
|
-
return self.operation_type
|
|
47
|
+
return self.operation_type
|
|
48
48
|
|
|
49
49
|
def validate_checkpoint(self, checkpoint: Operation | None) -> None:
|
|
50
50
|
"""Ensure replay history belongs to this operation before it is consumed."""
|