localstack-core 4.7.1.dev15__tar.gz → 4.7.1.dev17__tar.gz

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localstack-core
-Version: 4.7.1.dev15
+Version: 4.7.1.dev17
 Summary: The core library and runtime of LocalStack
 Author-email: LocalStack Contributors <info@localstack.cloud>
 License-Expression: Apache-2.0

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/localstack-core/localstack/services/cloudformation/engine/v2/change_set_model.py

@@ -8,6 +8,7 @@ from typing import Any, Final, Generator, Optional, TypedDict, Union, cast
 from typing_extensions import TypeVar
 
 from localstack.aws.api.cloudformation import ChangeAction
+from localstack.services.cloudformation.resource_provider import ResourceProviderExecutor
 from localstack.utils.strings import camel_to_snake_case
 
 T = TypeVar("T")
@@ -350,6 +351,7 @@ class NodeResource(ChangeSetNode):
     properties: Final[NodeProperties]
     condition_reference: Final[Maybe[TerminalValue]]
     depends_on: Final[Maybe[NodeDependsOn]]
+    requires_replacement: Final[bool]
 
     def __init__(
         self,
@@ -360,6 +362,7 @@ class NodeResource(ChangeSetNode):
         properties: NodeProperties,
         condition_reference: Maybe[TerminalValue],
         depends_on: Maybe[NodeDependsOn],
+        requires_replacement: bool,
     ):
         super().__init__(scope=scope, change_type=change_type)
         self.name = name
@@ -367,6 +370,7 @@ class NodeResource(ChangeSetNode):
         self.properties = properties
         self.condition_reference = condition_reference
         self.depends_on = depends_on
+        self.requires_replacement = requires_replacement
 
 
 class NodeProperties(ChangeSetNode):
@@ -722,6 +726,30 @@ class ChangeSetModel:
         change_type = parent_change_type_of([node_condition, *arguments[1:]])
         return change_type
 
+    def _resolve_requires_replacement(
+        self, node_properties: NodeProperties, resource_type: TerminalValue
+    ) -> bool:
+        # a bit hacky but we have to load the resource provider executor _and_ resource provider to get the schema
+        # Note: we don't log the attempt to load the resource provider, we need to make sure this is only done once and we already do this in the executor
+        resource_provider = ResourceProviderExecutor.try_load_resource_provider(resource_type.value)
+        if not resource_provider:
+            # if we don't support a resource, assume an in-place update for simplicity
+            return False
+
+        create_only_properties: list[str] = resource_provider.SCHEMA.get("createOnlyProperties", [])
+        # TODO: also hacky: strip the leading `/properties/` string from the definition
+        #       ideally we should use a jsonpath or similar
+        create_only_properties = [
+            property.replace("/properties/", "", 1) for property in create_only_properties
+        ]
+        for node_property in node_properties.properties:
+            if (
+                node_property.change_type == ChangeType.MODIFIED
+                and node_property.name in create_only_properties
+            ):
+                return True
+        return False
+
     def _visit_array(
         self, scope: Scope, before_array: Maybe[list], after_array: Maybe[list]
     ) -> NodeArray:
@@ -917,6 +945,9 @@ class ChangeSetModel:
         change_type = change_type_of(
             before_resource, after_resource, [properties, condition_reference, depends_on]
         )
+        requires_replacement = self._resolve_requires_replacement(
+            node_properties=properties, resource_type=terminal_value_type
+        )
         node_resource = NodeResource(
             scope=scope,
             change_type=change_type,
@@ -925,6 +956,7 @@ class ChangeSetModel:
             properties=properties,
             condition_reference=condition_reference,
             depends_on=depends_on,
+            requires_replacement=requires_replacement,
         )
         self._visited_scopes[scope] = node_resource
         return node_resource

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_describer.py

@@ -4,6 +4,7 @@ import json
 from typing import Final, Optional
 
 import localstack.aws.api.cloudformation as cfn_api
+from localstack.aws.api.cloudformation import Replacement
 from localstack.services.cloudformation.engine.v2.change_set_model import (
     NodeIntrinsicFunction,
     NodeProperty,
@@ -118,6 +119,8 @@ class ChangeSetModelDescriber(ChangeSetModelPreproc):
         physical_id: Optional[str],
         before_properties: Optional[PreprocProperties],
         after_properties: Optional[PreprocProperties],
+        # TODO: remove default
+        requires_replacement: bool = False,
     ) -> None:
         action = cfn_api.ChangeAction.Modify
         if before_properties is None:
@@ -131,14 +134,23 @@ class ChangeSetModelDescriber(ChangeSetModelPreproc):
         resource_change["ResourceType"] = type_
         if physical_id:
             resource_change["PhysicalResourceId"] = physical_id
-        if self._include_property_values and before_properties is not None:
-            before_context_properties = {PropertiesKey: before_properties.properties}
-            before_context_properties_json_str = json.dumps(before_context_properties)
-            resource_change["BeforeContext"] = before_context_properties_json_str
-        if self._include_property_values and after_properties is not None:
-            after_context_properties = {PropertiesKey: after_properties.properties}
-            after_context_properties_json_str = json.dumps(after_context_properties)
-            resource_change["AfterContext"] = after_context_properties_json_str
+        if self._include_property_values:
+            if before_properties is not None:
+                before_context_properties = {PropertiesKey: before_properties.properties}
+                before_context_properties_json_str = json.dumps(before_context_properties)
+                resource_change["BeforeContext"] = before_context_properties_json_str
+
+            if after_properties is not None:
+                after_context_properties = {PropertiesKey: after_properties.properties}
+                after_context_properties_json_str = json.dumps(after_context_properties)
+                resource_change["AfterContext"] = after_context_properties_json_str
+
+        if action == cfn_api.ChangeAction.Modify:
+            # TODO: handle "Conditional" case
+            resource_change["Replacement"] = (
+                Replacement.True_ if requires_replacement else Replacement.False_
+            )
+
         self._changes.append(
             cfn_api.Change(Type=cfn_api.ChangeType.Resource, ResourceChange=resource_change)
         )
@@ -159,6 +171,7 @@ class ChangeSetModelDescriber(ChangeSetModelPreproc):
                     type_=before.resource_type,
                     before_properties=before.properties,
                     after_properties=after.properties,
+                    requires_replacement=after.requires_replacement,
                 )
             # Case: type migration.
             # TODO: Add test to assert that on type change the resources are replaced.

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_executor.py

@@ -3,7 +3,7 @@ import logging
 import uuid
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Final, Optional
+from typing import Final, Optional, Protocol
 
 from localstack import config
 from localstack.aws.api.cloudformation import (
@@ -15,11 +15,15 @@ from localstack.constants import INTERNAL_AWS_SECRET_ACCESS_KEY
 from localstack.services.cloudformation.analytics import track_resource_operation
 from localstack.services.cloudformation.deployment_utils import log_not_available_message
 from localstack.services.cloudformation.engine.parameters import resolve_ssm_parameter
+from localstack.services.cloudformation.engine.template_deployer import REGEX_OUTPUT_APIGATEWAY
 from localstack.services.cloudformation.engine.v2.change_set_model import (
     NodeDependsOn,
     NodeOutput,
     NodeParameter,
     NodeResource,
+    TerminalValueCreated,
+    TerminalValueModified,
+    TerminalValueUnchanged,
     is_nothing,
 )
 from localstack.services.cloudformation.engine.v2.change_set_model_preproc import (
@@ -38,6 +42,7 @@ from localstack.services.cloudformation.resource_provider import (
     ResourceProviderPayload,
 )
 from localstack.services.cloudformation.v2.entities import ChangeSet, ResolvedResource
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -52,12 +57,17 @@ class ChangeSetModelExecutorResult:
     exports: dict
 
 
+class DeferredAction(Protocol):
+    def __call__(self) -> None: ...
+
+
 class ChangeSetModelExecutor(ChangeSetModelPreproc):
     # TODO: add typing for resolved resources and parameters.
     resources: Final[dict[str, ResolvedResource]]
     outputs: Final[dict]
     resolved_parameters: Final[dict]
     exports: Final[dict]
+    _deferred_actions: list[DeferredAction]
 
     def __init__(self, change_set: ChangeSet):
         super().__init__(change_set=change_set)
@@ -65,10 +75,25 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
         self.outputs = dict()
         self.resolved_parameters = dict()
         self.exports = dict()
+        self._deferred_actions = list()
+        self.resource_provider_executor = ResourceProviderExecutor(
+            stack_name=change_set.stack.stack_name,
+            stack_id=change_set.stack.stack_id,
+        )
 
-    # TODO: use a structured type for the return value
     def execute(self) -> ChangeSetModelExecutorResult:
+        # constructive process
         self.process()
+
+        if self._deferred_actions:
+            self._change_set.stack.set_stack_status(StackStatus.UPDATE_COMPLETE_CLEANUP_IN_PROGRESS)
+
+            # perform all deferred actions such as deletions. These must happen in reverse from their
+            # defined order so that resource dependencies are honoured
+            # TODO: errors will stop all rollbacks; get parity on this behaviour
+            for action in self._deferred_actions[::-1]:
+                action()
+
         return ChangeSetModelExecutorResult(
             resources=self.resources,
             parameters=self.resolved_parameters,
@@ -76,6 +101,9 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
             exports=self.exports,
         )
 
+    def _defer_action(self, action: DeferredAction):
+        self._deferred_actions.append(action)
+
     def visit_node_parameter(self, node_parameter: NodeParameter) -> PreprocEntityDelta:
         delta = super().visit_node_parameter(node_parameter)
 
@@ -248,20 +276,55 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
                 before_properties = self._merge_before_properties(name, before)
 
                 self._process_event(ChangeAction.Modify, name, OperationStatus.IN_PROGRESS)
-                event = self._execute_resource_action(
-                    action=ChangeAction.Modify,
-                    logical_resource_id=name,
-                    resource_type=before.resource_type,
-                    before_properties=before_properties,
-                    after_properties=after.properties,
-                )
-                self._process_event(
-                    ChangeAction.Modify,
-                    name,
-                    event.status,
-                    reason=event.message,
-                    resource_type=before.resource_type,
-                )
+                if after.requires_replacement:
+                    event = self._execute_resource_action(
+                        action=ChangeAction.Add,
+                        logical_resource_id=name,
+                        resource_type=before.resource_type,
+                        before_properties=None,
+                        after_properties=after.properties,
+                    )
+                    self._process_event(
+                        ChangeAction.Modify,
+                        name,
+                        event.status,
+                        reason=event.message,
+                        resource_type=before.resource_type,
+                    )
+
+                    def cleanup():
+                        self._process_event(ChangeAction.Remove, name, OperationStatus.IN_PROGRESS)
+                        event = self._execute_resource_action(
+                            action=ChangeAction.Remove,
+                            logical_resource_id=name,
+                            resource_type=before.resource_type,
+                            before_properties=before_properties,
+                            after_properties=None,
+                        )
+                        self._process_event(
+                            ChangeAction.Remove,
+                            name,
+                            event.status,
+                            reason=event.message,
+                            resource_type=before.resource_type,
+                        )
+
+                    self._defer_action(cleanup)
+                else:
+                    event = self._execute_resource_action(
+                        action=ChangeAction.Modify,
+                        logical_resource_id=name,
+                        resource_type=before.resource_type,
+                        before_properties=before_properties,
+                        after_properties=after.properties,
+                    )
+                    self._process_event(
+                        ChangeAction.Modify,
+                        name,
+                        event.status,
+                        reason=event.message,
+                        resource_type=before.resource_type,
+                    )
             # Case: type migration.
             # TODO: Add test to assert that on type change the resources are replaced.
             else:
@@ -269,21 +332,24 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
                 before_properties = self._merge_before_properties(name, before)
                 # Register a Removed for the previous type.
 
-                event = self._execute_resource_action(
-                    action=ChangeAction.Remove,
-                    logical_resource_id=name,
-                    resource_type=before.resource_type,
-                    before_properties=before_properties,
-                    after_properties=None,
-                )
-                # Register a Create for the next type.
-                self._process_event(
-                    ChangeAction.Modify,
-                    name,
-                    event.status,
-                    reason=event.message,
-                    resource_type=before.resource_type,
-                )
+                def perform_deletion():
+                    event = self._execute_resource_action(
+                        action=ChangeAction.Remove,
+                        logical_resource_id=name,
+                        resource_type=before.resource_type,
+                        before_properties=before_properties,
+                        after_properties=None,
+                    )
+                    self._process_event(
+                        ChangeAction.Modify,
+                        name,
+                        event.status,
+                        reason=event.message,
+                        resource_type=before.resource_type,
+                    )
+
+                self._defer_action(perform_deletion)
+
                 event = self._execute_resource_action(
                     action=ChangeAction.Add,
                     logical_resource_id=name,
@@ -303,26 +369,30 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
             # XXX hacky, stick the previous resources' properties into the payload
             # XXX hacky, stick the previous resources' properties into the payload
             before_properties = self._merge_before_properties(name, before)
-            self._process_event(
-                ChangeAction.Remove,
-                name,
-                OperationStatus.IN_PROGRESS,
-                resource_type=before.resource_type,
-            )
-            event = self._execute_resource_action(
-                action=ChangeAction.Remove,
-                logical_resource_id=name,
-                resource_type=before.resource_type,
-                before_properties=before_properties,
-                after_properties=None,
-            )
-            self._process_event(
-                ChangeAction.Remove,
-                name,
-                event.status,
-                reason=event.message,
-                resource_type=before.resource_type,
-            )
+
+            def perform_deletion():
+                self._process_event(
+                    ChangeAction.Remove,
+                    name,
+                    OperationStatus.IN_PROGRESS,
+                    resource_type=before.resource_type,
+                )
+                event = self._execute_resource_action(
+                    action=ChangeAction.Remove,
+                    logical_resource_id=name,
+                    resource_type=before.resource_type,
+                    before_properties=before_properties,
+                    after_properties=None,
+                )
+                self._process_event(
+                    ChangeAction.Remove,
+                    name,
+                    event.status,
+                    reason=event.message,
+                    resource_type=before.resource_type,
+                )
+
+            self._defer_action(perform_deletion)
         elif not is_nothing(after):
             # Case: addition
             self._process_event(
@@ -366,9 +436,6 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
         after_properties: Optional[PreprocProperties],
     ) -> ProgressEvent:
         LOG.debug("Executing resource action: %s for resource '%s'", action, logical_resource_id)
-        resource_provider_executor = ResourceProviderExecutor(
-            stack_name=self._change_set.stack.stack_name, stack_id=self._change_set.stack.stack_id
-        )
         payload = self.create_resource_provider_payload(
             action=action,
             logical_resource_id=logical_resource_id,
@@ -376,13 +443,15 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
             before_properties=before_properties,
             after_properties=after_properties,
         )
-        resource_provider = resource_provider_executor.try_load_resource_provider(resource_type)
+        resource_provider = self.resource_provider_executor.try_load_resource_provider(
+            resource_type
+        )
         track_resource_operation(action, resource_type, missing=resource_provider is not None)
 
         extra_resource_properties = {}
         if resource_provider is not None:
             try:
-                event = resource_provider_executor.deploy_loop(
+                event = self.resource_provider_executor.deploy_loop(
                     resource_provider, extra_resource_properties, payload
                 )
             except Exception as e:
@@ -433,24 +502,26 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
                 # TODO: avoid the use of setdefault (debuggability/readability)
                 # TODO: review the use of merge
 
-                status_from_action = EventOperationFromAction[action.value]
-                physical_resource_id = (
-                    extra_resource_properties["PhysicalResourceId"]
-                    if resource_provider
-                    else MOCKED_REFERENCE
-                )
-                resolved_resource = ResolvedResource(
-                    Properties=event.resource_model,
-                    LogicalResourceId=logical_resource_id,
-                    Type=resource_type,
-                    LastUpdatedTimestamp=datetime.now(timezone.utc),
-                    ResourceStatus=ResourceStatus(f"{status_from_action}_COMPLETE"),
-                    PhysicalResourceId=physical_resource_id,
-                )
-                # TODO: do we actually need this line?
-                resolved_resource.update(extra_resource_properties)
-
-                self.resources[logical_resource_id] = resolved_resource
+                # Don't update the resolved resources if we have deleted that resource
+                if action != ChangeAction.Remove:
+                    status_from_action = EventOperationFromAction[action.value]
+                    physical_resource_id = (
+                        extra_resource_properties["PhysicalResourceId"]
+                        if resource_provider
+                        else MOCKED_REFERENCE
+                    )
+                    resolved_resource = ResolvedResource(
+                        Properties=event.resource_model,
+                        LogicalResourceId=logical_resource_id,
+                        Type=resource_type,
+                        LastUpdatedTimestamp=datetime.now(timezone.utc),
+                        ResourceStatus=ResourceStatus(f"{status_from_action}_COMPLETE"),
+                        PhysicalResourceId=physical_resource_id,
+                    )
+                    # TODO: do we actually need this line?
+                    resolved_resource.update(extra_resource_properties)
+
+                    self.resources[logical_resource_id] = resolved_resource
 
             case OperationStatus.FAILED:
                 reason = event.message
@@ -517,3 +588,44 @@ class ChangeSetModelExecutor(ChangeSetModelPreproc):
             },
         }
         return resource_provider_payload
+
+    @staticmethod
+    def _replace_url_outputs_if_required(value: str) -> str:
+        api_match = REGEX_OUTPUT_APIGATEWAY.match(value)
+        if api_match and value not in config.CFN_STRING_REPLACEMENT_DENY_LIST:
+            prefix = api_match[1]
+            host = api_match[2]
+            path = api_match[3]
+            port = localstack_host().port
+            value = f"{prefix}{host}:{port}/{path}"
+            return value
+
+        return value
+
+    def visit_terminal_value_created(
+        self, value: TerminalValueCreated
+    ) -> PreprocEntityDelta[str, str]:
+        if isinstance(value.value, str):
+            after = self._replace_url_outputs_if_required(value.value)
+        else:
+            after = value.value
+        return PreprocEntityDelta(after=after)
+
+    def visit_terminal_value_modified(
+        self, value: TerminalValueModified
+    ) -> PreprocEntityDelta[str, str]:
+        # we only need to transform the after
+        if isinstance(value.modified_value, str):
+            after = self._replace_url_outputs_if_required(value.modified_value)
+        else:
+            after = value.modified_value
+        return PreprocEntityDelta(before=value.value, after=after)
+
+    def visit_terminal_value_unchanged(
+        self, terminal_value_unchanged: TerminalValueUnchanged
+    ) -> PreprocEntityDelta:
+        if isinstance(terminal_value_unchanged.value, str):
+            value = self._replace_url_outputs_if_required(terminal_value_unchanged.value)
+        else:
+            value = terminal_value_unchanged.value
+        return PreprocEntityDelta(before=value, after=value)

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_preproc.py

@@ -104,6 +104,7 @@ class PreprocResource:
     resource_type: str
     properties: PreprocProperties
     depends_on: Optional[list[str]]
+    requires_replacement: bool
 
     def __init__(
         self,
@@ -113,6 +114,7 @@ class PreprocResource:
         resource_type: str,
         properties: PreprocProperties,
         depends_on: Optional[list[str]],
+        requires_replacement: bool,
     ):
         self.logical_id = logical_id
         self.physical_resource_id = physical_resource_id
@@ -120,6 +122,7 @@ class PreprocResource:
         self.resource_type = resource_type
         self.properties = properties
         self.depends_on = depends_on
+        self.requires_replacement = requires_replacement
 
     @staticmethod
     def _compare_conditions(c1: bool, c2: bool):
@@ -1065,6 +1068,7 @@ class ChangeSetModelPreproc(ChangeSetModelVisitor):
                 resource_type=type_delta.before,
                 properties=properties_delta.before,
                 depends_on=depends_on_before,
+                requires_replacement=False,
             )
         if change_type != ChangeType.REMOVED and is_nothing(condition_after) or condition_after:
             logical_resource_id = node_resource.name
@@ -1081,6 +1085,7 @@ class ChangeSetModelPreproc(ChangeSetModelVisitor):
                 resource_type=type_delta.after,
                 properties=properties_delta.after,
                 depends_on=depends_on_after,
+                requires_replacement=node_resource.requires_replacement,
             )
         return PreprocEntityDelta(before=before, after=after)
 

{localstack_core-4.7.1.dev15 → localstack_core-4.7.1.dev17}/localstack-core/localstack/services/cloudformation/resource_provider.py

@@ -465,15 +465,18 @@ class ResourceProviderExecutor:
                             "A ResourceProvider should always have a SCHEMA property defined."
                         )
                     resource_type_schema = resource_provider.SCHEMA
-                    physical_resource_id = self.extract_physical_resource_id_from_model_with_schema(
-                        event.resource_model,
-                        raw_payload["resourceType"],
-                        resource_type_schema,
-                    )
+                    if raw_payload["action"] != "Remove":
+                        physical_resource_id = (
+                            self.extract_physical_resource_id_from_model_with_schema(
+                                event.resource_model,
+                                raw_payload["resourceType"],
+                                resource_type_schema,
+                            )
+                        )
 
-                    resource["PhysicalResourceId"] = physical_resource_id
-                    resource["Properties"] = event.resource_model
-                    resource["_last_deployed_state"] = copy.deepcopy(event.resource_model)
+                        resource["PhysicalResourceId"] = physical_resource_id
+                        resource["Properties"] = event.resource_model
+                        resource["_last_deployed_state"] = copy.deepcopy(event.resource_model)
                     return event
                 case OperationStatus.IN_PROGRESS:
                     # update the shared state