aws-durable-execution-sdk-python-testing 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

aws_durable_execution_sdk_python_testing/runner.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import json
 import logging
-import os
 import time
+import warnings
 from dataclasses import dataclass, field
 from typing import (
     TYPE_CHECKING,
@@ -16,7 +16,6 @@ from typing import (
     cast,
 )
 
-import aws_durable_execution_sdk_python
 import boto3  # type: ignore
 from botocore.exceptions import ClientError  # type: ignore
 from aws_durable_execution_sdk_python.execution import (
@@ -34,19 +33,35 @@ from aws_durable_execution_sdk_python.lambda_service import Operation as SvcOper
 
 from aws_durable_execution_sdk_python_testing.checkpoint.processor import (
     CheckpointProcessor,
+    DEFAULT_MAX_INVOCATION_PAGE_BYTES,
 )
 from aws_durable_execution_sdk_python_testing.client import InMemoryServiceClient
+from aws_durable_execution_sdk_python_testing.clock import Clock, RealClock, SkipClock
+from aws_durable_execution_sdk_python_testing.worker.registry import ExecutionRegistry
 from aws_durable_execution_sdk_python_testing.exceptions import (
     DurableFunctionsLocalRunnerError,
     DurableFunctionsTestError,
     InvalidParameterValueException,
     ResourceNotFoundException,
 )
+from aws_durable_execution_sdk_python_testing.child_dispatcher import (
+    DEFAULT_CHILD_EXECUTION_TIMEOUT_SECONDS,
+    DEFAULT_CHILD_RETENTION_PERIOD_DAYS,
+    ChildDispatcher,
+    EndpointChildDispatcher,
+    FunctionConfigs,
+    FunctionRegistry,
+    InProcessChildDispatcher,
+    UnconfiguredChildDispatcher,
+)
 from aws_durable_execution_sdk_python_testing.executor import Executor
+from aws_durable_execution_sdk_python_testing.execution import ExecutionStatus
 from aws_durable_execution_sdk_python_testing.invoker import (
     InProcessInvoker,
     LambdaInvoker,
     create_lambda_client,
+    create_test_lambda_context,
+    read_timeout_for,
 )
 from aws_durable_execution_sdk_python_testing.model import (
     GetDurableExecutionHistoryResponse,
@@ -107,6 +122,28 @@ class WebRunnerConfig:
     store_type: StoreType = StoreType.MEMORY
     store_path: str | None = None  # Path for filesystem store
 
+    # Emulated Lambda function timeout, applied to each handler
+    # invocation. Also bounds how long the runner waits on any Invoke it
+    # sends (see invoker.read_timeout_for).
+    invocation_timeout_seconds: int = 900
+
+    # Skip durable timer wall-clock waits (waits and step retries complete
+    # on the next scheduler turn while history keeps the real modeled
+    # durations). Defaults False so the server emulates real cloud timing;
+    # set True to opt into fast skip.
+    skip_time: bool = False
+
+    # Invocation-input pagination cap (bytes). Handler receives pages
+    # up to this size; larger state is served via
+    # GetDurableExecutionState. None falls back to Executor default
+    # (5 MB).
+    max_invocation_page_bytes: int | None = None
+    # The functions a durable function may invoke, by name. Required for
+    # chained invokes: the runner cannot learn a target's durability from
+    # the Lambda endpoint. See FunctionConfigs.from_value for the input.
+    # Last, so positional construction from before it exists is unchanged.
+    function_configs: FunctionConfigs | None = None
+
 
 @dataclass(frozen=True)
 class Operation:
@@ -398,6 +435,16 @@ def create_operation(
     return operation_class.from_svc_operation(svc_operation, all_operations)
 
 
+def _is_callback_ready_for_response(
+    events: list[Event], callback_started_event: Event
+) -> bool:
+    """Return True once the invocation that created the callback has settled."""
+    for event in events[events.index(callback_started_event) + 1 :]:
+        if event.event_type == "InvocationCompleted":
+            return True
+    return False
+
+
 def _get_callback_id_from_events(
     events: list[Event], name: str | None = None
 ) -> str | None:
@@ -409,7 +456,8 @@ def _get_callback_id_from_events(
         name: Optional callback name to search for. If not provided, returns the latest callback.
 
     Returns:
-        The callback ID string for a non-completed callback, or None if not found.
+        The callback ID string for a non-completed callback whose creating
+        invocation has completed, or None if not found.
 
     Raises:
         DurableFunctionsTestError: If the named callback has already succeeded/failed/timed out.
@@ -436,6 +484,8 @@ def _get_callback_id_from_events(
                     raise DurableFunctionsTestError(
                         f"Callback {name} has already completed (succeeded/failed/timed out)"
                     )
+                if not _is_callback_ready_for_response(events, event):
+                    return None
                 return (
                     event.callback_started_details.callback_id
                     if event.callback_started_details
@@ -448,6 +498,7 @@ def _get_callback_id_from_events(
         event
         for event in callback_started_events
         if event.event_id not in completed_callback_ids
+        and _is_callback_ready_for_response(events, event)
     ]
 
     if not active_callbacks:
@@ -467,6 +518,7 @@ class DurableFunctionTestResult:
     operations: list[Operation]
     result: OperationPayload | None = None
     error: ErrorObject | None = None
+    execution_status: ExecutionStatus | None = None
 
     @classmethod
     def create(cls, execution: Execution) -> DurableFunctionTestResult:
@@ -482,12 +534,16 @@ class DurableFunctionTestResult:
         if execution.result is None:
             msg: str = "Execution result must exist to create test result."
             raise DurableFunctionsTestError(msg)
+        if execution.close_status is None:
+            msg_status: str = "Execution close status must exist to create test result."
+            raise DurableFunctionsTestError(msg_status)
 
         return cls(
             status=execution.result.status,
             operations=operations,
             result=execution.result.result,
             error=execution.result.error,
+            execution_status=execution.close_status,
         )
 
     @classmethod
@@ -510,6 +566,16 @@ class DurableFunctionTestResult:
             )
             status = InvocationStatus.FAILED
 
+        # Map overall execution status string separately from invocation status.
+        try:
+            execution_status = ExecutionStatus[execution_response.status]
+        except KeyError:
+            logger.warning(
+                "Unknown execution status: %s, defaulting to FAILED",
+                execution_response.status,
+            )
+            execution_status = ExecutionStatus.FAILED
+
         # Convert Events to Operations - group by operation_id and merge
         try:
             svc_operations = events_to_operations(history_response.events)
@@ -530,6 +596,7 @@ class DurableFunctionTestResult:
             operations=operations,
             result=execution_response.result,
             error=execution_response.error,
+            execution_status=execution_status,
         )
 
     def get_operation_by_name(self, name: str) -> Operation:
@@ -571,25 +638,117 @@ class DurableFunctionTestResult:
 
 
 class DurableFunctionTestRunner:
-    def __init__(self, handler: Callable, poll_interval: float = 1.0):
+    def __init__(
+        self,
+        handler: Callable,
+        poll_interval: float = 1.0,
+        max_invocation_page_bytes: int | None = None,
+        execution_timeout: int = 300,
+        invocation_timeout: int = 900,
+        store: ExecutionStore | None = None,
+        skip_time: bool = True,  # noqa: FBT001, FBT002
+        region: str = "us-west-2",
+    ):
+        self._execution_timeout = execution_timeout
+        self._invocation_timeout = invocation_timeout
+        # Region the runner stands in for: reported in Lambda contexts and
+        # used to reject chained-invoke targets in another region.
+        self._region = region
+        self._max_invocation_page_bytes = (
+            max_invocation_page_bytes
+            if max_invocation_page_bytes is not None
+            else DEFAULT_MAX_INVOCATION_PAGE_BYTES
+        )
         self._scheduler: Scheduler = Scheduler()
         self._scheduler.start()
-        self._store = InMemoryExecutionStore()
+        # Defaults to the in-memory store. Pass a filesystem or sqlite
+        # store to run the in-process lifecycle on a disk-backed store.
+        self._store: ExecutionStore = (
+            store if store is not None else InMemoryExecutionStore()
+        )
         self.poll_interval = poll_interval
+        self._clock: Clock = SkipClock() if skip_time else RealClock()
+        self._registry = ExecutionRegistry(self._store, self._scheduler)
         self._checkpoint_processor = CheckpointProcessor(
-            store=self._store, scheduler=self._scheduler
+            store=self._store,
+            scheduler=self._scheduler,
+            clock=self._clock,
         )
-        self._service_client = InMemoryServiceClient(self._checkpoint_processor)
-        self._invoker = InProcessInvoker(handler, self._service_client)
+        self._service_client = InMemoryServiceClient(
+            self._checkpoint_processor, self._registry
+        )
+        self._invoker = InProcessInvoker(
+            handler,
+            self._service_client,
+            max_page_bytes=self._max_invocation_page_bytes,
+            region=region,
+        )
+        self._function_registry = FunctionRegistry()
         self._executor = Executor(
             store=self._store,
             scheduler=self._scheduler,
             invoker=self._invoker,
             checkpoint_processor=self._checkpoint_processor,
+            max_invocation_page_bytes=self._max_invocation_page_bytes,
+            invocation_timeout_seconds=invocation_timeout,
+            registry=self._registry,
+            clock=self._clock,
+            child_dispatcher=InProcessChildDispatcher(
+                self._function_registry,
+                lambda request: create_test_lambda_context(
+                    region=region,
+                    account_id=request.account_id,
+                    function_name=request.invoke_identifier(),
+                    tenant_id=request.tenant_id,
+                ),
+                invocation_timeout_seconds=invocation_timeout,
+                clock=self._clock,
+            ),
+            region=region,
         )
 
         # Wire up observer pattern - CheckpointProcessor uses this to notify executor of state changes
         self._checkpoint_processor.add_execution_observer(self._executor)
+        # A chained-invoke target the runner cannot resolve fails in the
+        # checkpoint response, as it does at the service.
+        self._checkpoint_processor.set_chained_invoke_preflight(
+            self._executor.preflight_chained_invoke
+        )
+
+    def register_durable_function(
+        self,
+        function_name: str,
+        handler: Callable,
+        execution_timeout: int = DEFAULT_CHILD_EXECUTION_TIMEOUT_SECONDS,
+        retention_period_days: int = DEFAULT_CHILD_RETENTION_PERIOD_DAYS,
+    ) -> DurableFunctionTestRunner:
+        """Register a durable function as a chained-invoke target.
+
+        A ``ctx.invoke`` of ``function_name`` runs ``handler`` as its
+        own durable execution with the given timeout and retention.
+        Returns the runner for chaining.
+        """
+        self._invoker.register(function_name, handler)
+        self._function_registry.register(
+            function_name,
+            handler,
+            is_durable=True,
+            execution_timeout_seconds=execution_timeout,
+            retention_period_days=retention_period_days,
+        )
+        return self
+
+    def register_function(
+        self, function_name: str, handler: Callable
+    ) -> DurableFunctionTestRunner:
+        """Register a non-durable function as a chained-invoke target.
+
+        A ``ctx.invoke`` of ``function_name`` calls ``handler`` once
+        with the deserialized payload and a test Lambda context.
+        Returns the runner for chaining.
+        """
+        self._function_registry.register(function_name, handler, is_durable=False)
+        return self
 
     def __enter__(self):
         return self
@@ -598,25 +757,44 @@ class DurableFunctionTestRunner:
         self.close()
 
     def close(self):
+        self._executor.shutdown()
+        self._registry.shutdown()
         self._scheduler.stop()
 
     def run(
         self,
         input: str | None = None,  # noqa: A002
-        timeout: int = 900,
+        execution_timeout: int | None = None,
+        timeout: int | None = None,
         function_name: str = "test-function",
         execution_name: str = "execution-name",
         account_id: str = "123456789012",
+        tenant_id: str | None = None,
     ) -> DurableFunctionTestResult:
+        if timeout is not None and execution_timeout is None:
+            warnings.warn(
+                "timeout on run() is deprecated. Use execution_timeout instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            execution_timeout = timeout
+        effective_timeout: int = (
+            execution_timeout
+            if execution_timeout is not None
+            else self._execution_timeout
+        )
         execution_arn = self.run_async(
             input=input,
-            timeout=timeout,
+            execution_timeout=effective_timeout,
             function_name=function_name,
             execution_name=execution_name,
             account_id=account_id,
+            tenant_id=tenant_id,
         )
 
-        return self.wait_for_result(execution_arn=execution_arn, timeout=timeout)
+        return self.wait_for_result(
+            execution_arn=execution_arn, timeout=effective_timeout
+        )
 
     def send_callback_success(
         self, callback_id: str, result: bytes | None = None
@@ -631,24 +809,48 @@ class DurableFunctionTestRunner:
     def send_callback_heartbeat(self, callback_id: str) -> None:
         self._executor.send_callback_heartbeat(callback_id=callback_id)
 
+    def get_execution_history(
+        self,
+        execution_arn: str,
+        include_execution_data: bool = False,  # noqa: FBT001, FBT002
+    ) -> GetDurableExecutionHistoryResponse:
+        """Return the event history for an execution.
+
+        Works for the execution under test and for any chained child
+        execution (the child's ARN is on the parent's
+        ChainedInvokeStarted event).
+        """
+        return self._executor.get_execution_history(
+            execution_arn, include_execution_data=include_execution_data
+        )
+
     def run_async(
         self,
         input: str | None = None,  # noqa: A002
-        timeout: int = 900,
+        execution_timeout: int | None = None,
+        timeout: int | None = None,
         function_name: str = "test-function",
         execution_name: str = "execution-name",
         account_id: str = "123456789012",
+        tenant_id: str | None = None,
     ) -> str:
+        if timeout is not None and execution_timeout is None:
+            execution_timeout = timeout
+        effective_timeout: int = (
+            execution_timeout
+            if execution_timeout is not None
+            else self._execution_timeout
+        )
         start_input = StartDurableExecutionInput(
             account_id=account_id,
             function_name=function_name,
             function_qualifier="$LATEST",
             execution_name=execution_name,
-            execution_timeout_seconds=timeout,
+            execution_timeout_seconds=effective_timeout,
             execution_retention_period_days=7,
             invocation_id="inv-12345678-1234-1234-1234-123456789012",
             trace_fields={"trace_id": "abc123", "span_id": "def456"},
-            tenant_id="tenant-001",
+            tenant_id=tenant_id,
             input=input,
         )
 
@@ -661,6 +863,53 @@ class DurableFunctionTestRunner:
             raise DurableFunctionsTestError(msg_arn)
         return output.execution_arn
 
+    def pause_execution(self, execution_arn: str, timeout: float = 60) -> None:
+        """Experimental; may change or be removed in a future release.
+
+        Pause an execution and wait for its current invocation to stop.
+
+        If a running invocation sends another checkpoint after the pause is
+        applied, its updates are accepted and saved. The response omits the
+        checkpoint token, causing the SDK to end the invocation with PENDING.
+        Further checkpoints from that invocation are rejected.
+
+        Returns once the pause request is applied and no invocation is running.
+        A running step is not interrupted.
+
+        Repeated calls leave an existing pause unchanged and wait again for
+        the running invocation to stop. Completed executions are unchanged.
+
+        If this call times out, the pause request is not cancelled.
+
+        Args:
+            execution_arn: Execution to pause.
+            timeout: Maximum wait in real seconds. Must be finite and positive.
+
+        Raises:
+            InvalidParameterValueException: If the ARN is blank.
+            ResourceNotFoundException: If the execution does not exist.
+            ValueError: If timeout is not finite and positive.
+            TimeoutError: If the pause wait times out.
+        """
+        self._executor.pause_execution(execution_arn, timeout=timeout)
+
+    def resume_execution(self, execution_arn: str) -> None:
+        """Experimental; may change or be removed in a future release.
+
+        Make the local checkpoint server answer this execution's
+        checkpoints with a token again.
+
+        Starts the invocation pause_execution(arn) held back, if any.
+
+        Repeated calls leave an execution that is not paused unchanged.
+        Completed executions are unchanged.
+
+        Raises:
+            InvalidParameterValueException: If the ARN is blank.
+            ResourceNotFoundException: If the execution does not exist.
+        """
+        self._executor.resume_execution(execution_arn)
+
     def wait_for_result(
         self, execution_arn: str, timeout: int = 60
     ) -> DurableFunctionTestResult:
@@ -688,7 +937,8 @@ class DurableFunctionTestRunner:
                 )
                 if callback_id:
                     return callback_id
-            except ResourceNotFoundException as e:
+            except ResourceNotFoundException:
+                # Execution may not be visible yet during async invoke.
                 pass
             except Exception as e:
                 msg = f"Failed to fetch execution history: {e}"
@@ -699,7 +949,7 @@ class DurableFunctionTestRunner:
 
         # Timeout reached
         elapsed = time.time() - start_time
-        msg = f"Callback did not available within {timeout}s (elapsed: {elapsed:.1f}s."
+        msg = f"Callback did not become available within {timeout}s (elapsed: {elapsed:.1f}s)."
         raise TimeoutError(msg)
 
 
@@ -710,6 +960,7 @@ class DurableChildContextTestRunner(DurableFunctionTestRunner):
         self,
         context_function: Callable[Concatenate[DurableContext, P], Any],
         *args,
+        skip_time: bool = True,
         **kwargs,
     ):
         # wrap the durable context around a durable execution handler as a convenience to run directly
@@ -717,11 +968,26 @@ class DurableChildContextTestRunner(DurableFunctionTestRunner):
         def handler(event: Any, context: DurableContext):  # noqa: ARG001
             return context_function(*args, **kwargs)(context)
 
-        super().__init__(handler)
+        super().__init__(handler, skip_time=skip_time)
 
 
 class WebRunner:
-    """Web server runner for durable functions testing with HTTP API endpoints."""
+    """Web server runner for durable functions testing with HTTP API endpoints.
+
+    Handlers run at the Lambda endpoint (``WebRunnerConfig.lambda_endpoint``);
+    the runner drives each execution by invoking its handler there.
+
+    Chained invokes (``context.invoke``) need ``WebRunnerConfig.function_configs``,
+    the targets by name (:class:`FunctionConfigs`). Without it every
+    chained invoke fails with an error naming the option. A durable
+    target runs as a child execution the runner drives; a plain target is
+    one RequestResponse Invoke at the endpoint, bounded by the invocation
+    timeout, whose function error fails the operation. Every handler
+    invocation the runner sends carries ``X-Dex-Handler-Invoke: true``
+    (:data:`invoker.INVOCATION_MARKER_HEADER`), so an endpoint that starts
+    an execution on Invoke of a durable function runs the handler once
+    instead; plain targets are sent without it.
+    """
 
     def __init__(self, config: WebRunnerConfig) -> None:
         """Initialize WebRunner with configuration.
@@ -735,6 +1001,7 @@ class WebRunner:
         self._store: ExecutionStore | None = None
         self._invoker: LambdaInvoker | None = None
         self._executor: Executor | None = None
+        self._registry: ExecutionRegistry | None = None
 
     def __enter__(self) -> Self:
         """Context manager entry point.
@@ -779,10 +1046,45 @@ class WebRunner:
         else:
             self._store = InMemoryExecutionStore()
         self._scheduler = Scheduler()
-        self._invoker = LambdaInvoker(self._create_boto3_client())
+        resolved_max_page_bytes: int = (
+            self._config.max_invocation_page_bytes
+            if self._config.max_invocation_page_bytes is not None
+            else DEFAULT_MAX_INVOCATION_PAGE_BYTES
+        )
+        read_timeout_seconds: int = read_timeout_for(
+            self._config.invocation_timeout_seconds
+        )
+        lambda_client: Any = self._create_boto3_client(read_timeout_seconds)
+        self._invoker = LambdaInvoker(
+            lambda_client,
+            max_page_bytes=resolved_max_page_bytes,
+            read_timeout_seconds=read_timeout_seconds,
+            endpoint_url=self._config.lambda_endpoint,
+            region_name=self._config.local_runner_region,
+        )
 
         # Create shared CheckpointProcessor
-        checkpoint_processor = CheckpointProcessor(self._store, self._scheduler)
+        clock: Clock = SkipClock() if self._config.skip_time else RealClock()
+        self._clock = clock
+        self._registry = ExecutionRegistry(self._store, self._scheduler)
+        checkpoint_processor = CheckpointProcessor(
+            self._store,
+            self._scheduler,
+            clock=clock,
+        )
+
+        # Non-durable targets are invoked with the invoker's unmarked client
+        # for the parent's endpoint, so the endpoint runs them as any
+        # caller's Invoke and they go where the parent's invocations go.
+        child_dispatcher: ChildDispatcher
+        if self._config.function_configs is not None:
+            child_dispatcher = EndpointChildDispatcher(
+                self._config.function_configs,
+                self._invoker.unmarked_client_for,
+                invocation_timeout_seconds=self._config.invocation_timeout_seconds,
+            )
+        else:
+            child_dispatcher = UnconfiguredChildDispatcher()
 
         # Create executor with all dependencies including checkpoint processor
         self._executor = Executor(
@@ -790,10 +1092,19 @@ class WebRunner:
             scheduler=self._scheduler,
             invoker=self._invoker,
             checkpoint_processor=checkpoint_processor,
+            max_invocation_page_bytes=resolved_max_page_bytes,
+            invocation_timeout_seconds=self._config.invocation_timeout_seconds,
+            registry=self._registry,
+            clock=clock,
+            child_dispatcher=child_dispatcher,
+            region=self._config.local_runner_region,
         )
 
         # Add executor as observer to the checkpoint processor
         checkpoint_processor.add_execution_observer(self._executor)
+        checkpoint_processor.set_chained_invoke_preflight(
+            self._executor.preflight_chained_invoke
+        )
 
         # Start the scheduler
         self._scheduler.start()
@@ -836,6 +1147,19 @@ class WebRunner:
 
             self._server = None
 
+        if self._executor is not None:
+            try:
+                self._executor.shutdown()
+            except Exception:
+                logger.exception("error shutting down executor")
+
+        if self._registry is not None:
+            try:
+                self._registry.shutdown()
+            except Exception:
+                logger.exception("error stopping execution registry")
+            self._registry = None
+
         if self._scheduler is not None:
             try:
                 self._scheduler.stop()
@@ -847,10 +1171,12 @@ class WebRunner:
         self._invoker = None
         self._executor = None
 
-    def _create_boto3_client(self) -> Any:
+    def _create_boto3_client(self, read_timeout_seconds: int) -> Any:
         """Create boto3 client for Lambda service.
 
-        Creates a boto3 client with the local runner endpoint and region from configuration.
+        Creates a boto3 client with the local runner endpoint and region from
+        configuration. ``read_timeout_seconds`` is passed through to
+        :func:`create_lambda_client`.
 
         Returns:
             Configured boto3 client for Lambda service
@@ -862,11 +1188,12 @@ class WebRunner:
         return create_lambda_client(
             endpoint_url=self._config.lambda_endpoint,
             region_name=self._config.local_runner_region,
+            read_timeout_seconds=read_timeout_seconds,
         )
 
 
 class DurableFunctionCloudTestRunner:
-    """Test runner that executes durable functions against actual AWS Lambda backend.
+    """Test runner that executes durable functions against the deployed service.
 
     This runner invokes deployed Lambda functions and polls for execution completion,
     providing the same interface as DurableFunctionTestRunner for seamless test
@@ -984,6 +1311,28 @@ class DurableFunctionCloudTestRunner:
 
         return response.get("DurableExecutionArn")
 
+    def pause_execution(self, execution_arn: str, timeout: float = 60) -> None:
+        """Experimental; may change or be removed in a future release.
+
+        Cloud pause is not implemented; use DurableFunctionTestRunner instead.
+
+        Raises:
+            NotImplementedError: Always. No service request is sent.
+        """
+        msg = "pause_execution() is not implemented for DurableFunctionCloudTestRunner"
+        raise NotImplementedError(msg)
+
+    def resume_execution(self, execution_arn: str) -> None:
+        """Experimental; may change or be removed in a future release.
+
+        Cloud resume is not implemented; use DurableFunctionTestRunner instead.
+
+        Raises:
+            NotImplementedError: Always. No service request is sent.
+        """
+        msg = "resume_execution() is not implemented for DurableFunctionCloudTestRunner"
+        raise NotImplementedError(msg)
+
     def send_callback_success(
         self, callback_id: str, result: bytes | None = None
     ) -> None:
@@ -1040,6 +1389,17 @@ class DurableFunctionCloudTestRunner:
                     DurableExecutionArn=execution_arn
                 )
                 execution = GetDurableExecutionResponse.from_dict(execution_dict)
+            except ClientError as e:
+                error_code = e.response.get("Error", {}).get("Code")
+                # Retry while the asynchronous invoke's durable execution record is
+                # still becoming visible to GetDurableExecution.
+                if error_code == "ResourceNotFoundException":
+                    logger.info("Execution status not available yet; retrying")
+                    time.sleep(self.poll_interval)
+                    continue
+
+                msg = f"Failed to get execution status: {e}"
+                raise DurableFunctionsTestError(msg) from e
             except Exception as e:
                 msg = f"Failed to get execution status: {e}"
                 raise DurableFunctionsTestError(msg) from e
@@ -1056,7 +1416,7 @@ class DurableFunctionCloudTestRunner:
             if execution.status == "FAILED":
                 logger.warning("Execution failed")
                 return execution
-            if execution.status in ["TIMED_OUT", "ABORTED"]:
+            if execution.status in ["TIMED_OUT", "STOPPED"]:
                 logger.warning("Execution terminated: %s", execution.status)
                 return execution
 
@@ -1139,7 +1499,7 @@ class DurableFunctionCloudTestRunner:
 
         # Timeout reached
         elapsed = time.time() - start_time
-        msg = f"Callback did not available within {timeout}s (elapsed: {elapsed:.1f}s."
+        msg = f"Callback did not become available within {timeout}s (elapsed: {elapsed:.1f}s)."
         raise TimeoutError(msg)
 
     def _fetch_execution_history(
@@ -1156,11 +1516,28 @@ class DurableFunctionCloudTestRunner:
         Raises:
             ClientError: If lambda client encounter error
         """
-        history_dict = self.lambda_client.get_durable_execution_history(
-            DurableExecutionArn=execution_arn,
-            IncludeExecutionData=True,
-        )
-        history_response = GetDurableExecutionHistoryResponse.from_dict(history_dict)
+        events: list[Event] = []
+        next_marker: str | None = None
+
+        while True:
+            request = {
+                "DurableExecutionArn": execution_arn,
+                "IncludeExecutionData": True,
+            }
+            if next_marker:
+                request["Marker"] = next_marker
+
+            history_dict = self.lambda_client.get_durable_execution_history(**request)
+            history_response = GetDurableExecutionHistoryResponse.from_dict(
+                history_dict
+            )
+            events.extend(history_response.events)
+
+            next_marker = history_response.next_marker
+            if not next_marker:
+                break
+
+        history_response = GetDurableExecutionHistoryResponse(events=events)
 
         logger.info("Retrieved %d events from history", len(history_response.events))