aws-durable-execution-sdk-python-testing 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aws_durable_execution_sdk_python_testing/__about__.py +1 -1
- aws_durable_execution_sdk_python_testing/checkpoint/core.py +171 -0
- aws_durable_execution_sdk_python_testing/checkpoint/effects.py +61 -0
- aws_durable_execution_sdk_python_testing/checkpoint/processor.py +124 -45
- aws_durable_execution_sdk_python_testing/checkpoint/processors/base.py +40 -14
- aws_durable_execution_sdk_python_testing/checkpoint/processors/callback.py +5 -2
- aws_durable_execution_sdk_python_testing/checkpoint/processors/context.py +6 -0
- aws_durable_execution_sdk_python_testing/checkpoint/processors/execution.py +4 -9
- aws_durable_execution_sdk_python_testing/checkpoint/processors/invoke.py +139 -0
- aws_durable_execution_sdk_python_testing/checkpoint/processors/step.py +15 -21
- aws_durable_execution_sdk_python_testing/checkpoint/processors/wait.py +10 -16
- aws_durable_execution_sdk_python_testing/checkpoint/transformer.py +145 -48
- aws_durable_execution_sdk_python_testing/checkpoint/validators/checkpoint.py +65 -8
- aws_durable_execution_sdk_python_testing/checkpoint/validators/operations/invoke.py +79 -18
- aws_durable_execution_sdk_python_testing/child_dispatcher.py +828 -0
- aws_durable_execution_sdk_python_testing/cli.py +48 -1
- aws_durable_execution_sdk_python_testing/client.py +21 -10
- aws_durable_execution_sdk_python_testing/clock.py +69 -0
- aws_durable_execution_sdk_python_testing/exceptions.py +14 -0
- aws_durable_execution_sdk_python_testing/execution.py +583 -49
- aws_durable_execution_sdk_python_testing/executor.py +1813 -349
- aws_durable_execution_sdk_python_testing/invoker.py +356 -85
- aws_durable_execution_sdk_python_testing/model.py +81 -73
- aws_durable_execution_sdk_python_testing/observer.py +100 -88
- aws_durable_execution_sdk_python_testing/runner.py +407 -30
- aws_durable_execution_sdk_python_testing/scheduler.py +38 -13
- aws_durable_execution_sdk_python_testing/stores/filesystem.py +10 -2
- aws_durable_execution_sdk_python_testing/stores/sqlite.py +18 -6
- aws_durable_execution_sdk_python_testing/threads.py +157 -0
- aws_durable_execution_sdk_python_testing/token.py +11 -2
- aws_durable_execution_sdk_python_testing/web/handlers.py +33 -7
- aws_durable_execution_sdk_python_testing/web/serialization.py +1 -2
- aws_durable_execution_sdk_python_testing/web/server.py +1 -1
- aws_durable_execution_sdk_python_testing/worker/__init__.py +9 -0
- aws_durable_execution_sdk_python_testing/worker/checkpoint_tasks.py +119 -0
- aws_durable_execution_sdk_python_testing/worker/execution_worker.py +117 -0
- aws_durable_execution_sdk_python_testing/worker/lane.py +118 -0
- aws_durable_execution_sdk_python_testing/worker/registry.py +124 -0
- aws_durable_execution_sdk_python_testing/worker/status.py +27 -0
- aws_durable_execution_sdk_python_testing/worker/task.py +57 -0
- {aws_durable_execution_sdk_python_testing-1.2.1.dist-info → aws_durable_execution_sdk_python_testing-1.3.0.dist-info}/METADATA +50 -58
- aws_durable_execution_sdk_python_testing-1.3.0.dist-info/RECORD +64 -0
- {aws_durable_execution_sdk_python_testing-1.2.1.dist-info → aws_durable_execution_sdk_python_testing-1.3.0.dist-info}/WHEEL +1 -1
- aws_durable_execution_sdk_python_testing/checkpoint/validators/transitions.py +0 -66
- aws_durable_execution_sdk_python_testing-1.2.1.dist-info/RECORD +0 -52
- {aws_durable_execution_sdk_python_testing-1.2.1.dist-info/licenses → aws_durable_execution_sdk_python_testing}/LICENSE +0 -0
- {aws_durable_execution_sdk_python_testing-1.2.1.dist-info/licenses → aws_durable_execution_sdk_python_testing}/NOTICE +0 -0
- {aws_durable_execution_sdk_python_testing-1.2.1.dist-info → aws_durable_execution_sdk_python_testing-1.3.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
from dataclasses import dataclass
|
|
4
|
+
from dataclasses import dataclass, replace
|
|
5
5
|
from threading import Lock
|
|
6
6
|
from typing import TYPE_CHECKING, Any, Protocol
|
|
7
7
|
from uuid import uuid4
|
|
@@ -15,13 +15,22 @@ from aws_durable_execution_sdk_python.execution import (
|
|
|
15
15
|
DurableExecutionInvocationOutput,
|
|
16
16
|
InitialExecutionState,
|
|
17
17
|
)
|
|
18
|
+
from aws_durable_execution_sdk_python.lambda_service import ErrorObject
|
|
18
19
|
|
|
20
|
+
from aws_durable_execution_sdk_python_testing.checkpoint.processor import (
|
|
21
|
+
DEFAULT_MAX_INVOCATION_PAGE_BYTES,
|
|
22
|
+
)
|
|
19
23
|
from aws_durable_execution_sdk_python_testing.exceptions import (
|
|
20
24
|
DurableFunctionsTestError,
|
|
25
|
+
InvalidInvocationOutputError,
|
|
21
26
|
InvalidParameterValueException,
|
|
22
27
|
ResourceNotFoundException,
|
|
23
28
|
)
|
|
24
|
-
from aws_durable_execution_sdk_python_testing.
|
|
29
|
+
from aws_durable_execution_sdk_python_testing.execution import OperationPaginatorState
|
|
30
|
+
from aws_durable_execution_sdk_python_testing.model import (
|
|
31
|
+
LambdaContext,
|
|
32
|
+
executed_version,
|
|
33
|
+
)
|
|
25
34
|
|
|
26
35
|
|
|
27
36
|
if TYPE_CHECKING:
|
|
@@ -31,24 +40,66 @@ if TYPE_CHECKING:
|
|
|
31
40
|
from aws_durable_execution_sdk_python_testing.execution import Execution
|
|
32
41
|
|
|
33
42
|
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
# Every Invoke the runner sends is one Lambda invocation: a handler
|
|
44
|
+
# invocation, or a chained invoke of a non-durable target. The client
|
|
45
|
+
# waits for it to return, so the read timeout exceeds the emulated
|
|
46
|
+
# function timeout (``--invocation-timeout``, default 900 s) by a fixed
|
|
47
|
+
# headroom for the network round-trip and RIE startup.
|
|
48
|
+
DEFAULT_INVOCATION_TIMEOUT_SECONDS = 900
|
|
49
|
+
LAMBDA_READ_TIMEOUT_HEADROOM_SECONDS = 60
|
|
50
|
+
|
|
41
51
|
|
|
52
|
+
def read_timeout_for(invocation_timeout_seconds: int) -> int:
|
|
53
|
+
"""Client read timeout that outlasts one invocation of ``invocation_timeout_seconds``."""
|
|
54
|
+
return invocation_timeout_seconds + LAMBDA_READ_TIMEOUT_HEADROOM_SECONDS
|
|
42
55
|
|
|
43
|
-
def create_lambda_client(endpoint_url: str | None, region_name: str) -> Any:
|
|
44
|
-
"""Create a boto3 Lambda client configured for durable function invocations."""
|
|
45
56
|
|
|
46
|
-
|
|
57
|
+
DEFAULT_LAMBDA_READ_TIMEOUT_SECONDS = read_timeout_for(
|
|
58
|
+
DEFAULT_INVOCATION_TIMEOUT_SECONDS
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Request header on every handler invocation. To a Lambda-compatible
|
|
62
|
+
# endpoint, a caller's Invoke of a durable function starts a new
|
|
63
|
+
# execution. A handler invocation is an Invoke of that function whose
|
|
64
|
+
# payload is a durable invocation input; the endpoint must run the
|
|
65
|
+
# handler once with it and return the handler's output. This header is
|
|
66
|
+
# how the runner says so. Endpoints that never start executions ignore it.
|
|
67
|
+
INVOCATION_MARKER_HEADER = "X-Dex-Handler-Invoke"
|
|
68
|
+
INVOCATION_MARKER_VALUE = "true"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _add_invocation_marker(params: dict[str, Any], **_kwargs: Any) -> None:
|
|
72
|
+
"""botocore ``before-call`` hook: mark the request as a handler invocation."""
|
|
73
|
+
params.setdefault("headers", {})[INVOCATION_MARKER_HEADER] = INVOCATION_MARKER_VALUE
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def create_lambda_client(
|
|
77
|
+
endpoint_url: str | None,
|
|
78
|
+
region_name: str,
|
|
79
|
+
read_timeout_seconds: int = DEFAULT_LAMBDA_READ_TIMEOUT_SECONDS,
|
|
80
|
+
*,
|
|
81
|
+
mark_invocations: bool = True,
|
|
82
|
+
) -> Any:
|
|
83
|
+
"""Create a boto3 Lambda client for the runner's Invoke calls.
|
|
84
|
+
|
|
85
|
+
``read_timeout_seconds`` bounds one Invoke. With ``mark_invocations``
|
|
86
|
+
every Invoke carries :data:`INVOCATION_MARKER_HEADER`; the client
|
|
87
|
+
that invokes non-durable chained targets passes ``False`` so those
|
|
88
|
+
look like any caller's Invoke.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
client: Any = boto3.client(
|
|
47
92
|
"lambda",
|
|
48
93
|
endpoint_url=endpoint_url,
|
|
49
94
|
region_name=region_name,
|
|
50
|
-
config=
|
|
95
|
+
config=Config(
|
|
96
|
+
read_timeout=read_timeout_seconds,
|
|
97
|
+
retries={"max_attempts": 0},
|
|
98
|
+
),
|
|
51
99
|
)
|
|
100
|
+
if mark_invocations:
|
|
101
|
+
client.meta.events.register("before-call.lambda.Invoke", _add_invocation_marker)
|
|
102
|
+
return client
|
|
52
103
|
|
|
53
104
|
|
|
54
105
|
@dataclass(frozen=True)
|
|
@@ -58,8 +109,74 @@ class InvokeResponse:
|
|
|
58
109
|
invocation_output: DurableExecutionInvocationOutput
|
|
59
110
|
request_id: str
|
|
60
111
|
|
|
112
|
+
@classmethod
|
|
113
|
+
def from_output(cls, output: Any, request_id: str) -> InvokeResponse:
|
|
114
|
+
"""Create from the value a handler returned.
|
|
61
115
|
|
|
62
|
-
|
|
116
|
+
Raises:
|
|
117
|
+
InvalidInvocationOutputError: If the value is not an object with
|
|
118
|
+
a known ``Status`` and, if present, an object ``Error``.
|
|
119
|
+
"""
|
|
120
|
+
try:
|
|
121
|
+
invocation_output = DurableExecutionInvocationOutput.from_dict(output)
|
|
122
|
+
# from_dict treats any falsey Error as absent, but only a missing
|
|
123
|
+
# or null Error is. An empty object is kept, so SUCCEEDED with it
|
|
124
|
+
# fails validation; a value that is not an object raises here.
|
|
125
|
+
error: Any = output.get("Error")
|
|
126
|
+
if error is not None and invocation_output.error is None:
|
|
127
|
+
invocation_output = replace(
|
|
128
|
+
invocation_output, error=ErrorObject.from_dict(error)
|
|
129
|
+
)
|
|
130
|
+
except (AttributeError, TypeError, ValueError) as e:
|
|
131
|
+
msg: str = f"Invalid invocation output: {e}"
|
|
132
|
+
raise InvalidInvocationOutputError(msg, request_id) from e
|
|
133
|
+
return cls(invocation_output=invocation_output, request_id=request_id)
|
|
134
|
+
|
|
135
|
+
@classmethod
|
|
136
|
+
def from_payload(cls, payload: bytes, request_id: str) -> InvokeResponse:
|
|
137
|
+
"""Create from the payload a Lambda invocation returned.
|
|
138
|
+
|
|
139
|
+
Raises:
|
|
140
|
+
InvalidInvocationOutputError: If the payload is not UTF-8 JSON or
|
|
141
|
+
not a valid invocation output.
|
|
142
|
+
"""
|
|
143
|
+
try:
|
|
144
|
+
# UnicodeDecodeError is a ValueError.
|
|
145
|
+
output: Any = json.loads(payload.decode("utf-8"))
|
|
146
|
+
except ValueError as e:
|
|
147
|
+
msg: str = f"Invalid invocation output: {e}"
|
|
148
|
+
raise InvalidInvocationOutputError(msg, request_id) from e
|
|
149
|
+
return cls.from_output(output, request_id)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Values the in-process Lambda context reports when the caller gives none.
|
|
153
|
+
DEFAULT_TEST_REGION = "us-west-2"
|
|
154
|
+
DEFAULT_TEST_ACCOUNT_ID = "123456789012"
|
|
155
|
+
DEFAULT_TEST_FUNCTION_NAME = "test-function"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def create_test_lambda_context(
|
|
159
|
+
*,
|
|
160
|
+
region: str = DEFAULT_TEST_REGION,
|
|
161
|
+
account_id: str = DEFAULT_TEST_ACCOUNT_ID,
|
|
162
|
+
function_name: str = DEFAULT_TEST_FUNCTION_NAME,
|
|
163
|
+
tenant_id: str | None = None,
|
|
164
|
+
) -> LambdaContext:
|
|
165
|
+
"""Build the Lambda context handed to an in-process handler.
|
|
166
|
+
|
|
167
|
+
``function_name`` is the identifier the function is invoked by:
|
|
168
|
+
``name`` or ``name:qualifier``. Lambda fills ``function_name`` with
|
|
169
|
+
the bare name, ``function_version`` with the version that runs, and
|
|
170
|
+
``invoked_function_arn`` with the ARN as invoked, so target code that
|
|
171
|
+
reads them sees the same values here. The runner keeps no versions:
|
|
172
|
+
a numeric qualifier is reported as the version, anything else
|
|
173
|
+
(no qualifier, ``$LATEST``, an alias) as ``$LATEST``.
|
|
174
|
+
|
|
175
|
+
``tenant_id`` is the execution's or invoke's tenant; ``None`` means
|
|
176
|
+
the invocation had none, as in Lambda.
|
|
177
|
+
"""
|
|
178
|
+
bare_name, _, qualifier = function_name.partition(":")
|
|
179
|
+
function_version: str = executed_version(qualifier)
|
|
63
180
|
# Create client context as a dictionary, not as objects
|
|
64
181
|
# LambdaContext.__init__ expects dictionaries and will create the objects internally
|
|
65
182
|
client_context_dict = {
|
|
@@ -83,8 +200,12 @@ def create_test_lambda_context() -> LambdaContext:
|
|
|
83
200
|
aws_request_id="test-invoke-12345",
|
|
84
201
|
client_context=client_context_dict,
|
|
85
202
|
identity=cognito_identity_dict,
|
|
86
|
-
|
|
87
|
-
|
|
203
|
+
function_name=bare_name,
|
|
204
|
+
function_version=function_version,
|
|
205
|
+
invoked_function_arn=(
|
|
206
|
+
f"arn:aws:lambda:{region}:{account_id}:function:{function_name}"
|
|
207
|
+
),
|
|
208
|
+
tenant_id=tenant_id,
|
|
88
209
|
)
|
|
89
210
|
|
|
90
211
|
|
|
@@ -98,81 +219,214 @@ class Invoker(Protocol):
|
|
|
98
219
|
function_name: str,
|
|
99
220
|
input: DurableExecutionInvocationInput,
|
|
100
221
|
endpoint_url: str | None = None,
|
|
222
|
+
tenant_id: str | None = None,
|
|
223
|
+
account_id: str | None = None,
|
|
224
|
+
region_name: str | None = None,
|
|
101
225
|
) -> InvokeResponse: ... # pragma: no cover
|
|
102
226
|
|
|
103
227
|
def update_endpoint(
|
|
104
228
|
self, endpoint_url: str, region_name: str
|
|
105
229
|
) -> None: ... # pragma: no cover
|
|
106
230
|
|
|
231
|
+
def inherit_endpoint(
|
|
232
|
+
self, child_execution_arn: str, parent_execution_arn: str
|
|
233
|
+
) -> None: ... # pragma: no cover
|
|
234
|
+
|
|
107
235
|
|
|
108
236
|
class InProcessInvoker(Invoker):
|
|
109
|
-
def __init__(
|
|
237
|
+
def __init__(
|
|
238
|
+
self,
|
|
239
|
+
handler: Callable,
|
|
240
|
+
service_client: InMemoryServiceClient,
|
|
241
|
+
max_page_bytes: int = DEFAULT_MAX_INVOCATION_PAGE_BYTES,
|
|
242
|
+
region: str = DEFAULT_TEST_REGION,
|
|
243
|
+
):
|
|
110
244
|
self.handler = handler
|
|
245
|
+
self._region = region
|
|
111
246
|
self.service_client = service_client
|
|
247
|
+
self._max_page_bytes = max_page_bytes
|
|
248
|
+
# Named handlers for chained-invoke targets. The root handler
|
|
249
|
+
# remains the fallback for the execution under test.
|
|
250
|
+
self._handlers: dict[str, Callable] = {}
|
|
251
|
+
|
|
252
|
+
def register(self, function_name: str, handler: Callable) -> None:
|
|
253
|
+
"""Register ``handler`` to be resolved by ``function_name``."""
|
|
254
|
+
self._handlers[function_name] = handler
|
|
255
|
+
|
|
256
|
+
def _resolve_handler(self, function_name: str) -> Callable:
|
|
257
|
+
"""The handler for ``function_name``, which may carry a qualifier.
|
|
258
|
+
|
|
259
|
+
A registration under the qualified identifier wins; otherwise
|
|
260
|
+
the bare name's registration serves every qualifier; otherwise
|
|
261
|
+
the runner's own handler.
|
|
262
|
+
"""
|
|
263
|
+
handler: Callable | None = self._handlers.get(function_name)
|
|
264
|
+
if handler is None:
|
|
265
|
+
handler = self._handlers.get(function_name.split(":", 1)[0])
|
|
266
|
+
return handler if handler is not None else self.handler
|
|
112
267
|
|
|
113
268
|
def create_invocation_input(
|
|
114
269
|
self, execution: Execution
|
|
115
270
|
) -> DurableExecutionInvocationInput:
|
|
271
|
+
paginator = OperationPaginatorState.pin(execution)
|
|
272
|
+
page_operations, next_marker = paginator.page(None, self._max_page_bytes)
|
|
116
273
|
return DurableExecutionInvocationInputWithClient(
|
|
117
274
|
durable_execution_arn=execution.durable_execution_arn,
|
|
118
275
|
# TODO: this needs better logic - use existing if not used yet, vs create new
|
|
119
276
|
checkpoint_token=execution.get_new_checkpoint_token(),
|
|
120
277
|
initial_execution_state=InitialExecutionState(
|
|
121
|
-
operations=
|
|
122
|
-
next_marker="",
|
|
278
|
+
operations=page_operations,
|
|
279
|
+
next_marker=next_marker or "",
|
|
123
280
|
),
|
|
281
|
+
updated_operation_ids=list(execution.updated_operation_ids),
|
|
124
282
|
service_client=self.service_client,
|
|
125
283
|
)
|
|
126
284
|
|
|
127
285
|
def invoke(
|
|
128
286
|
self,
|
|
129
|
-
function_name: str,
|
|
287
|
+
function_name: str,
|
|
130
288
|
input: DurableExecutionInvocationInput,
|
|
131
289
|
endpoint_url: str | None = None, # noqa: ARG002
|
|
290
|
+
tenant_id: str | None = None,
|
|
291
|
+
account_id: str | None = None,
|
|
292
|
+
region_name: str | None = None, # noqa: ARG002 — the context reports the runner's
|
|
132
293
|
) -> InvokeResponse:
|
|
133
|
-
# TODO: reasses if function_name will be used in future
|
|
134
294
|
input_with_client = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input(
|
|
135
295
|
input, self.service_client
|
|
136
296
|
)
|
|
137
|
-
context = create_test_lambda_context(
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
297
|
+
context = create_test_lambda_context(
|
|
298
|
+
region=self._region,
|
|
299
|
+
account_id=account_id or DEFAULT_TEST_ACCOUNT_ID,
|
|
300
|
+
function_name=function_name,
|
|
301
|
+
tenant_id=tenant_id,
|
|
142
302
|
)
|
|
303
|
+
handler: Callable = self._resolve_handler(function_name)
|
|
304
|
+
response_dict = handler(input_with_client, context)
|
|
305
|
+
return InvokeResponse.from_output(response_dict, context.aws_request_id)
|
|
143
306
|
|
|
144
307
|
def update_endpoint(self, endpoint_url: str, region_name: str) -> None:
|
|
145
308
|
"""No-op for in-process invoker."""
|
|
146
309
|
|
|
310
|
+
def inherit_endpoint(
|
|
311
|
+
self, child_execution_arn: str, parent_execution_arn: str
|
|
312
|
+
) -> None:
|
|
313
|
+
"""No-op for in-process invoker: there is no endpoint to inherit."""
|
|
314
|
+
|
|
147
315
|
|
|
148
316
|
class LambdaInvoker(Invoker):
|
|
149
|
-
def __init__(
|
|
317
|
+
def __init__(
|
|
318
|
+
self,
|
|
319
|
+
lambda_client: Any,
|
|
320
|
+
max_page_bytes: int = DEFAULT_MAX_INVOCATION_PAGE_BYTES,
|
|
321
|
+
read_timeout_seconds: int = DEFAULT_LAMBDA_READ_TIMEOUT_SECONDS,
|
|
322
|
+
endpoint_url: str = "",
|
|
323
|
+
region_name: str = "",
|
|
324
|
+
) -> None:
|
|
325
|
+
"""``endpoint_url`` and ``region_name`` describe ``lambda_client``."""
|
|
150
326
|
self.lambda_client = lambda_client
|
|
151
|
-
|
|
152
|
-
#
|
|
153
|
-
self.
|
|
154
|
-
|
|
155
|
-
|
|
327
|
+
self._max_page_bytes = max_page_bytes
|
|
328
|
+
# Applied to every client this invoker creates for another endpoint.
|
|
329
|
+
self._read_timeout_seconds = read_timeout_seconds
|
|
330
|
+
# Clients are keyed by (endpoint URL, region): the region is the
|
|
331
|
+
# signing region, so the same URL in another region is another
|
|
332
|
+
# client. Marked clients invoke handlers; unmarked clients invoke
|
|
333
|
+
# non-durable chained targets.
|
|
334
|
+
self._endpoint_clients: dict[tuple[str, str], Any] = {}
|
|
335
|
+
self._unmarked_clients: dict[tuple[str, str], Any] = {}
|
|
336
|
+
# An execution without its own endpoint is pinned, at its first
|
|
337
|
+
# invocation, to the endpoint current at that moment. Every later
|
|
338
|
+
# handler invocation and every chained target it dispatches use
|
|
339
|
+
# the pinned endpoint, so an update_endpoint call while it runs
|
|
340
|
+
# does not split it across endpoints. A child it starts inherits
|
|
341
|
+
# the pin (inherit_endpoint). An execution with its own endpoint
|
|
342
|
+
# is pinned at its first chained dispatch instead: the pin then
|
|
343
|
+
# covers only its chained targets and children, and its own
|
|
344
|
+
# endpoint is signed in the execution's region.
|
|
345
|
+
self._execution_endpoints: dict[str, tuple[str, str]] = {}
|
|
346
|
+
# Endpoint and region for executions not yet pinned.
|
|
347
|
+
self._current: tuple[str, str] = (endpoint_url, region_name)
|
|
348
|
+
if endpoint_url:
|
|
349
|
+
self._endpoint_clients[self._current] = lambda_client
|
|
156
350
|
self._lock = Lock()
|
|
157
351
|
|
|
158
352
|
@staticmethod
|
|
159
|
-
def create(
|
|
353
|
+
def create(
|
|
354
|
+
endpoint_url: str,
|
|
355
|
+
region_name: str,
|
|
356
|
+
max_page_bytes: int = DEFAULT_MAX_INVOCATION_PAGE_BYTES,
|
|
357
|
+
read_timeout_seconds: int = DEFAULT_LAMBDA_READ_TIMEOUT_SECONDS,
|
|
358
|
+
) -> LambdaInvoker:
|
|
160
359
|
"""Create with the boto lambda client."""
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
360
|
+
return LambdaInvoker(
|
|
361
|
+
create_lambda_client(endpoint_url, region_name, read_timeout_seconds),
|
|
362
|
+
max_page_bytes=max_page_bytes,
|
|
363
|
+
read_timeout_seconds=read_timeout_seconds,
|
|
364
|
+
endpoint_url=endpoint_url,
|
|
365
|
+
region_name=region_name,
|
|
366
|
+
)
|
|
165
367
|
|
|
166
368
|
def update_endpoint(self, endpoint_url: str, region_name: str) -> None:
|
|
167
|
-
"""Update the Lambda
|
|
168
|
-
|
|
369
|
+
"""Update the Lambda endpoint and region for executions not yet pinned."""
|
|
370
|
+
key: tuple[str, str] = (endpoint_url, region_name)
|
|
169
371
|
with self._lock:
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
372
|
+
self.lambda_client = self._marked_client(key)
|
|
373
|
+
self._current = key
|
|
374
|
+
|
|
375
|
+
def _marked_client(self, key: tuple[str, str]) -> Any:
|
|
376
|
+
"""Client that marks its invokes as handler invocations. Caller holds the lock."""
|
|
377
|
+
client: Any = self._endpoint_clients.get(key)
|
|
378
|
+
if client is None:
|
|
379
|
+
client = create_lambda_client(
|
|
380
|
+
key[0] or None, key[1], self._read_timeout_seconds
|
|
381
|
+
)
|
|
382
|
+
self._endpoint_clients[key] = client
|
|
383
|
+
return client
|
|
384
|
+
|
|
385
|
+
def _pinned_endpoint(self, durable_execution_arn: str) -> tuple[str, str]:
|
|
386
|
+
"""The (endpoint, region) ``durable_execution_arn`` is pinned to, pinning it now if needed."""
|
|
387
|
+
with self._lock:
|
|
388
|
+
pinned: tuple[str, str] | None = self._execution_endpoints.get(
|
|
389
|
+
durable_execution_arn
|
|
390
|
+
)
|
|
391
|
+
if pinned is None:
|
|
392
|
+
pinned = self._current
|
|
393
|
+
self._execution_endpoints[durable_execution_arn] = pinned
|
|
394
|
+
return pinned
|
|
395
|
+
|
|
396
|
+
def inherit_endpoint(
|
|
397
|
+
self, child_execution_arn: str, parent_execution_arn: str
|
|
398
|
+
) -> None:
|
|
399
|
+
"""Pin a chained child to the endpoint its parent's chained targets go to.
|
|
400
|
+
|
|
401
|
+
The parent is pinned now if it has no pin yet. So the parent's
|
|
402
|
+
plain targets, the child's handler invocations, and the child's
|
|
403
|
+
own chained dispatches all use one endpoint and region, whatever
|
|
404
|
+
update_endpoint sets for executions started later.
|
|
405
|
+
"""
|
|
406
|
+
pinned: tuple[str, str] = self._pinned_endpoint(parent_execution_arn)
|
|
407
|
+
with self._lock:
|
|
408
|
+
self._execution_endpoints[child_execution_arn] = pinned
|
|
409
|
+
|
|
410
|
+
def unmarked_client_for(self, durable_execution_arn: str) -> Any:
|
|
411
|
+
"""Client for invoking the non-durable chained targets of one execution.
|
|
412
|
+
|
|
413
|
+
The client targets the endpoint and region the execution is
|
|
414
|
+
pinned to, so a target goes where the execution's own handler
|
|
415
|
+
invocations go. It carries no invocation marker, so the endpoint
|
|
416
|
+
runs the target as any caller's Invoke.
|
|
417
|
+
"""
|
|
418
|
+
key: tuple[str, str] = self._pinned_endpoint(durable_execution_arn)
|
|
419
|
+
with self._lock:
|
|
420
|
+
client: Any = self._unmarked_clients.get(key)
|
|
421
|
+
if client is None:
|
|
422
|
+
client = create_lambda_client(
|
|
423
|
+
key[0] or None,
|
|
424
|
+
key[1],
|
|
425
|
+
self._read_timeout_seconds,
|
|
426
|
+
mark_invocations=False,
|
|
173
427
|
)
|
|
174
|
-
|
|
175
|
-
|
|
428
|
+
self._unmarked_clients[key] = client
|
|
429
|
+
return client
|
|
176
430
|
|
|
177
431
|
def _get_client_for_execution(
|
|
178
432
|
self,
|
|
@@ -180,41 +434,42 @@ class LambdaInvoker(Invoker):
|
|
|
180
434
|
lambda_endpoint: str | None = None,
|
|
181
435
|
region_name: str | None = None,
|
|
182
436
|
) -> Any:
|
|
183
|
-
"""Get the appropriate client for this execution.
|
|
184
|
-
|
|
437
|
+
"""Get the appropriate client for this execution.
|
|
438
|
+
|
|
439
|
+
An execution with its own ``lambda_endpoint`` is invoked there,
|
|
440
|
+
signed in ``region_name``, the execution's region. That endpoint
|
|
441
|
+
serves the execution's function alone (under sam it is the
|
|
442
|
+
function's own container), so it is never recorded as the
|
|
443
|
+
execution's pin: the execution's chained targets are other
|
|
444
|
+
functions and go to the pinned endpoint, which routes by name.
|
|
445
|
+
Any other execution is invoked at the endpoint it is pinned to.
|
|
446
|
+
"""
|
|
185
447
|
if lambda_endpoint:
|
|
186
|
-
if lambda_endpoint not in self._endpoint_clients:
|
|
187
|
-
self._endpoint_clients[lambda_endpoint] = create_lambda_client(
|
|
188
|
-
lambda_endpoint, region_name or "us-east-1"
|
|
189
|
-
)
|
|
190
|
-
return self._endpoint_clients[lambda_endpoint]
|
|
191
|
-
|
|
192
|
-
# Fallback to cached endpoint
|
|
193
|
-
if durable_execution_arn not in self._execution_endpoints:
|
|
194
448
|
with self._lock:
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
endpoint = self._execution_endpoints[durable_execution_arn]
|
|
449
|
+
return self._marked_client(
|
|
450
|
+
(lambda_endpoint, region_name or "us-east-1")
|
|
451
|
+
)
|
|
201
452
|
|
|
202
|
-
|
|
203
|
-
if not
|
|
453
|
+
key: tuple[str, str] = self._pinned_endpoint(durable_execution_arn)
|
|
454
|
+
if not key[0]:
|
|
455
|
+
# Built with a client and no endpoint: nothing else to pick.
|
|
204
456
|
return self.lambda_client
|
|
205
|
-
|
|
206
|
-
|
|
457
|
+
with self._lock:
|
|
458
|
+
return self._marked_client(key)
|
|
207
459
|
|
|
208
460
|
def create_invocation_input(
|
|
209
461
|
self, execution: Execution
|
|
210
462
|
) -> DurableExecutionInvocationInput:
|
|
463
|
+
paginator = OperationPaginatorState.pin(execution)
|
|
464
|
+
page_operations, next_marker = paginator.page(None, self._max_page_bytes)
|
|
211
465
|
return DurableExecutionInvocationInput(
|
|
212
466
|
durable_execution_arn=execution.durable_execution_arn,
|
|
213
467
|
checkpoint_token=execution.get_new_checkpoint_token(),
|
|
214
468
|
initial_execution_state=InitialExecutionState(
|
|
215
|
-
operations=
|
|
216
|
-
next_marker="",
|
|
469
|
+
operations=page_operations,
|
|
470
|
+
next_marker=next_marker or "",
|
|
217
471
|
),
|
|
472
|
+
updated_operation_ids=list(execution.updated_operation_ids),
|
|
218
473
|
)
|
|
219
474
|
|
|
220
475
|
def invoke(
|
|
@@ -222,13 +477,19 @@ class LambdaInvoker(Invoker):
|
|
|
222
477
|
function_name: str,
|
|
223
478
|
input: DurableExecutionInvocationInput,
|
|
224
479
|
endpoint_url: str | None = None,
|
|
480
|
+
tenant_id: str | None = None,
|
|
481
|
+
account_id: str | None = None, # noqa: ARG002 — identity is the endpoint's
|
|
482
|
+
region_name: str | None = None,
|
|
225
483
|
) -> InvokeResponse:
|
|
226
484
|
"""Invoke AWS Lambda function and return durable execution result.
|
|
227
485
|
|
|
228
486
|
Args:
|
|
229
487
|
function_name: Name of the Lambda function to invoke
|
|
230
488
|
input: Durable execution invocation input
|
|
231
|
-
endpoint_url: Lambda endpoint
|
|
489
|
+
endpoint_url: The execution's own Lambda endpoint, if it has one
|
|
490
|
+
tenant_id: The execution's tenant, sent as the Invoke TenantId
|
|
491
|
+
account_id: The execution's account; unused, the endpoint owns identity
|
|
492
|
+
region_name: The execution's region; signs an Invoke at ``endpoint_url``
|
|
232
493
|
|
|
233
494
|
Returns:
|
|
234
495
|
InvokeResponse: Response containing invocation output and request ID
|
|
@@ -236,6 +497,7 @@ class LambdaInvoker(Invoker):
|
|
|
236
497
|
Raises:
|
|
237
498
|
ResourceNotFoundException: If function does not exist
|
|
238
499
|
InvalidParameterValueException: If parameters are invalid
|
|
500
|
+
InvalidInvocationOutputError: If the function's output cannot be parsed
|
|
239
501
|
DurableFunctionsTestError: For other invocation failures
|
|
240
502
|
"""
|
|
241
503
|
|
|
@@ -246,16 +508,20 @@ class LambdaInvoker(Invoker):
|
|
|
246
508
|
|
|
247
509
|
# Get the client for this execution
|
|
248
510
|
client = self._get_client_for_execution(
|
|
249
|
-
input.durable_execution_arn, endpoint_url
|
|
511
|
+
input.durable_execution_arn, endpoint_url, region_name
|
|
250
512
|
)
|
|
251
513
|
|
|
514
|
+
invoke_kwargs: dict[str, Any] = {
|
|
515
|
+
"FunctionName": function_name,
|
|
516
|
+
"InvocationType": "RequestResponse", # Synchronous invocation
|
|
517
|
+
"Payload": json.dumps(input.to_json_dict()),
|
|
518
|
+
}
|
|
519
|
+
if tenant_id is not None:
|
|
520
|
+
invoke_kwargs["TenantId"] = tenant_id
|
|
521
|
+
|
|
252
522
|
try:
|
|
253
523
|
# Invoke AWS Lambda function using standard invoke method
|
|
254
|
-
response = client.invoke(
|
|
255
|
-
FunctionName=function_name,
|
|
256
|
-
InvocationType="RequestResponse", # Synchronous invocation
|
|
257
|
-
Payload=json.dumps(input.to_json_dict()),
|
|
258
|
-
)
|
|
524
|
+
response = client.invoke(**invoke_kwargs)
|
|
259
525
|
|
|
260
526
|
# Check HTTP status code
|
|
261
527
|
status_code = response.get("StatusCode")
|
|
@@ -269,22 +535,23 @@ class LambdaInvoker(Invoker):
|
|
|
269
535
|
msg = f"Lambda invocation failed with status {status_code}: {error_payload}"
|
|
270
536
|
raise DurableFunctionsTestError(msg)
|
|
271
537
|
|
|
272
|
-
#
|
|
273
|
-
response_payload = response["Payload"].read()
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
#
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
538
|
+
# Read response payload; from_payload decodes it, below
|
|
539
|
+
response_payload: bytes = response["Payload"].read()
|
|
540
|
+
|
|
541
|
+
# botocore puts the request ID in ResponseMetadata and keeps the
|
|
542
|
+
# response headers with lowercased names.
|
|
543
|
+
metadata: dict[str, Any] = response.get("ResponseMetadata", {})
|
|
544
|
+
headers: dict[str, str] = {
|
|
545
|
+
name.lower(): value
|
|
546
|
+
for name, value in metadata.get("HTTPHeaders", {}).items()
|
|
547
|
+
}
|
|
548
|
+
request_id: str = (
|
|
549
|
+
metadata.get("RequestId")
|
|
550
|
+
or headers.get("x-amzn-requestid")
|
|
280
551
|
or headers.get("x-amzn-request-id")
|
|
281
552
|
or f"local-{uuid4()}"
|
|
282
553
|
)
|
|
283
554
|
|
|
284
|
-
# Convert to DurableExecutionInvocationOutput
|
|
285
|
-
output = DurableExecutionInvocationOutput.from_dict(response_dict)
|
|
286
|
-
return InvokeResponse(invocation_output=output, request_id=request_id)
|
|
287
|
-
|
|
288
555
|
except client.exceptions.ResourceNotFoundException as e:
|
|
289
556
|
msg = f"Function not found: {function_name}"
|
|
290
557
|
raise ResourceNotFoundException(msg) from e
|
|
@@ -338,3 +605,7 @@ class LambdaInvoker(Invoker):
|
|
|
338
605
|
raise DurableFunctionsTestError(msg) from e
|
|
339
606
|
msg = f"Unexpected error during Lambda invocation: {e}"
|
|
340
607
|
raise DurableFunctionsTestError(msg) from e
|
|
608
|
+
|
|
609
|
+
# Parsed outside the try: an output the service cannot parse fails
|
|
610
|
+
# the execution, so it must not be reported as a failed invocation.
|
|
611
|
+
return InvokeResponse.from_payload(response_payload, request_id)
|