localstack-core 4.7.1.dev49__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/services/opensearch/provider.py

@@ -26,6 +26,7 @@ from localstack.aws.api.opensearch import (
     CognitoOptions,
     CognitoOptionsStatus,
     ColdStorageOptions,
+    CompatibleVersionsMap,
     CreateDomainRequest,
     CreateDomainResponse,
     DeleteDomainResponse,
@@ -75,7 +76,6 @@ from localstack.aws.api.opensearch import (
     VolumeType,
     VPCDerivedInfoStatus,
 )
-from localstack.constants import OPENSEARCH_DEFAULT_VERSION
 from localstack.services.opensearch import versions
 from localstack.services.opensearch.cluster import SecurityOptions
 from localstack.services.opensearch.cluster_manager import (
@@ -84,6 +84,7 @@ from localstack.services.opensearch.cluster_manager import (
     create_cluster_manager,
 )
 from localstack.services.opensearch.models import OpenSearchStore, opensearch_stores
+from localstack.services.opensearch.packages import OPENSEARCH_DEFAULT_VERSION
 from localstack.services.plugins import ServiceLifecycleHook
 from localstack.state import AssetDirectory, StateVisitor
 from localstack.utils.aws.arns import parse_arn
@@ -466,10 +467,11 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
                         preferred_port=preferred_port,
                     )
                 except Exception:
-                    LOG.exception(
+                    LOG.error(
                         "Could not restore domain %s in region %s.",
                         domain_name,
                         region,
+                        exc_info=LOG.isEnabledFor(logging.DEBUG),
                     )
 
     def on_before_state_reset(self):
@@ -649,6 +651,10 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
                 for comp in versions.compatible_versions
                 if comp["SourceVersion"] == version_filter
             ]
+            if not compatible_versions:
+                compatible_versions = [
+                    CompatibleVersionsMap(SourceVersion=version_filter, TargetVersions=[])
+                ]
         return GetCompatibleVersionsResponse(CompatibleVersions=compatible_versions)
 
     def describe_domain_config(

localstack/services/opensearch/versions.py

@@ -13,20 +13,24 @@ from localstack.utils.common import get_arch
 
 # Internal representation of the OpenSearch versions (without the "OpenSearch_" prefix)
 _opensearch_install_versions = {
+    "3.1": "3.1.0",
+    "2.19": "2.19.3",
+    "2.17": "2.17.1",
+    "2.15": "2.15.0",
     "2.13": "2.13.0",
     "2.11": "2.11.1",
     "2.9": "2.9.0",
     "2.7": "2.7.0",
     "2.5": "2.5.0",
     "2.3": "2.3.0",
-    "1.3": "1.3.12",
+    "1.3": "1.3.20",
     "1.2": "1.2.4",
     "1.1": "1.1.0",
     "1.0": "1.0.0",
 }
 # Internal representation of the Elasticsearch versions (without the "Elasticsearch_" prefix)
 _elasticsearch_install_versions = {
-    "7.10": "7.10.0",
+    "7.10": "7.10.2",
     "7.9": "7.9.3",
     "7.8": "7.8.1",
     "7.7": "7.7.1",
@@ -221,6 +225,9 @@ compatible_versions = [
             "OpenSearch_2.9",
             "OpenSearch_2.11",
             "OpenSearch_2.13",
+            "OpenSearch_2.15",
+            "OpenSearch_2.17",
+            "OpenSearch_2.19",
         ],
     ),
     CompatibleVersionsMap(
@@ -231,28 +238,68 @@ compatible_versions = [
             "OpenSearch_2.9",
             "OpenSearch_2.11",
             "OpenSearch_2.13",
+            "OpenSearch_2.15",
+            "OpenSearch_2.17",
+            "OpenSearch_2.19",
         ],
     ),
     CompatibleVersionsMap(
         SourceVersion="OpenSearch_2.5",
-        TargetVersions=["OpenSearch_2.7", "OpenSearch_2.9", "OpenSearch_2.11", "OpenSearch_2.13"],
+        TargetVersions=[
+            "OpenSearch_2.7",
+            "OpenSearch_2.9",
+            "OpenSearch_2.11",
+            "OpenSearch_2.13",
+            "OpenSearch_2.15",
+            "OpenSearch_2.17",
+            "OpenSearch_2.19",
+        ],
     ),
     CompatibleVersionsMap(
         SourceVersion="OpenSearch_2.7",
-        TargetVersions=["OpenSearch_2.9", "OpenSearch_2.11", "OpenSearch_2.13"],
+        TargetVersions=[
+            "OpenSearch_2.9",
+            "OpenSearch_2.11",
+            "OpenSearch_2.13",
+            "OpenSearch_2.15",
+            "OpenSearch_2.17",
+            "OpenSearch_2.19",
+        ],
     ),
     CompatibleVersionsMap(
         SourceVersion="OpenSearch_2.9",
-        TargetVersions=["OpenSearch_2.11", "OpenSearch_2.13"],
+        TargetVersions=[
+            "OpenSearch_2.11",
+            "OpenSearch_2.13",
+            "OpenSearch_2.15",
+            "OpenSearch_2.17",
+            "OpenSearch_2.19",
+        ],
     ),
     CompatibleVersionsMap(
         SourceVersion="OpenSearch_2.11",
-        TargetVersions=["OpenSearch_2.13"],
+        TargetVersions=["OpenSearch_2.13", "OpenSearch_2.15", "OpenSearch_2.17", "OpenSearch_2.19"],
+    ),
+    CompatibleVersionsMap(
+        SourceVersion="OpenSearch_2.13",
+        TargetVersions=["OpenSearch_2.15", "OpenSearch_2.17", "OpenSearch_2.19"],
+    ),
+    CompatibleVersionsMap(
+        SourceVersion="OpenSearch_2.15",
+        TargetVersions=["OpenSearch_2.17", "OpenSearch_2.19"],
+    ),
+    CompatibleVersionsMap(
+        SourceVersion="OpenSearch_2.17",
+        TargetVersions=["OpenSearch_2.19"],
+    ),
+    CompatibleVersionsMap(
+        SourceVersion="OpenSearch_2.19",
+        TargetVersions=["OpenSearch_3.1"],
     ),
 ]
 
 
-def get_install_type_and_version(version: str) -> (EngineType, str):
+def get_install_type_and_version(version: str) -> tuple[EngineType, str]:
     engine_type = EngineType(version.split("_")[0])
 
     if version not in install_versions:
@@ -297,6 +344,8 @@ def get_download_url(install_version: str, engine_type: EngineType) -> str:
         return _opensearch_url(install_version)
     elif engine_type == EngineType.Elasticsearch:
         return _es_url(install_version)
+    else:
+        raise ValueError(f"Unknown OpenSearch engine type: {engine_type}")
 
 
 def fetch_latest_versions() -> dict[str, str]:  # pragma: no cover

localstack/services/plugins.py

@@ -286,19 +286,19 @@ class ServiceManager:
         container = self.get_service_container(name)
 
         if not container:
-            raise ValueError("no such service %s" % name)
+            raise ValueError(f"no such service {name}")
 
         if container.state == ServiceState.STARTING:
             if not poll_condition(lambda: container.state != ServiceState.STARTING, timeout=30):
-                raise TimeoutError("gave up waiting for service %s to start" % name)
+                raise TimeoutError(f"gave up waiting for service {name} to start")
 
         if container.state == ServiceState.STOPPING:
             if not poll_condition(lambda: container.state == ServiceState.STOPPED, timeout=30):
-                raise TimeoutError("gave up waiting for service %s to stop" % name)
+                raise TimeoutError(f"gave up waiting for service {name} to stop")
 
         with container.lock:
             if container.state == ServiceState.DISABLED:
-                raise ServiceDisabled("service %s is disabled" % name)
+                raise ServiceDisabled(f"service {name} is disabled")
 
             if container.state == ServiceState.RUNNING:
                 return container.service
@@ -314,7 +314,7 @@ class ServiceManager:
                     raise container.errors[-1]
 
         raise ServiceStateException(
-            "service %s is not ready (%s) and could not be started" % (name, container.state)
+            f"service {name} is not ready ({container.state}) and could not be started"
         )
 
     # legacy map compatibility
@@ -692,7 +692,7 @@ def check_service_health(api, expect_shutdown=False):
             LOG.warning('Service "%s" not yet available, retrying...', api)
         else:
             LOG.warning('Service "%s" still shutting down, retrying...', api)
-        raise Exception("Service check failed for api: %s" % api)
+        raise Exception(f"Service check failed for api: {api}")
 
 
 @hooks.on_infra_start(should_load=lambda: config.EAGER_SERVICE_LOADING)
@@ -708,4 +708,8 @@ def eager_load_services():
         except ServiceDisabled as e:
             LOG.debug("%s", e)
         except Exception:
-            LOG.exception("could not load service plugin %s", api)
+            LOG.error(
+                "could not load service plugin %s",
+                api,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )

localstack/services/providers.py

@@ -41,7 +41,7 @@ def apigateway_legacy():
     return Service.for_provider(provider, dispatch_table_factory=MotoFallbackDispatcher)
 
 
-@aws_provider()
+@aws_provider(api="cloudformation", name="engine-legacy")
 def cloudformation():
     from localstack.services.cloudformation.provider import CloudformationProvider
 
@@ -49,7 +49,7 @@ def cloudformation():
     return Service.for_provider(provider)
 
 
-@aws_provider(api="cloudformation", name="engine-v2")
+@aws_provider(api="cloudformation")
 def cloudformation_v2():
     from localstack.services.cloudformation.v2.provider import CloudformationProviderV2
 
@@ -318,6 +318,14 @@ def sns():
     return Service.for_provider(provider, dispatch_table_factory=MotoFallbackDispatcher)
 
 
+@aws_provider(api="sns", name="v2")
+def sns_v2():
+    from localstack.services.sns.v2.provider import SnsProvider
+
+    provider = SnsProvider()
+    return Service.for_provider(provider)
+
+
 @aws_provider()
 def sqs():
     from localstack.services.sqs.provider import SqsProvider

localstack/services/redshift/provider.py

@@ -1,6 +1,5 @@
 import os
 
-from moto.redshift import responses as redshift_responses
 from moto.redshift.models import redshift_backends
 
 from localstack import config
@@ -12,26 +11,6 @@ from localstack.aws.api.redshift import (
 )
 from localstack.services.moto import call_moto
 from localstack.state import AssetDirectory, StateVisitor
-from localstack.utils.common import recurse_object
-from localstack.utils.patch import patch
-
-
-@patch(redshift_responses.itemize)
-def itemize(fn, data, parent_key=None, *args, **kwargs):
-    # TODO: potentially add additional required tags here!
-    list_parent_tags = ["ClusterSubnetGroups"]
-
-    def fix_keys(o, **kwargs):
-        if isinstance(o, dict):
-            for k, v in o.items():
-                if k in list_parent_tags:
-                    if isinstance(v, dict) and "item" in v:
-                        v[k[:-1]] = v.pop("item")
-        return o
-
-    result = fn(data, *args, **kwargs)
-    recurse_object(result, fix_keys)
-    return result
 
 
 class RedshiftProvider(RedshiftApi):

localstack/services/s3/constants.py

@@ -10,8 +10,6 @@ from localstack.aws.api.s3 import (
 )
 from localstack.aws.api.s3 import Type as GranteeType
 
-S3_VIRTUAL_HOST_FORWARDED_HEADER = "x-s3-vhost-forwarded-for"
-
 S3_UPLOAD_PART_MIN_SIZE = 5242880
 """
 This is minimum size allowed by S3 when uploading more than one part for a Multipart Upload, except for the last part
@@ -21,6 +19,11 @@ This is minimum size allowed by S3 when uploading more than one part for a Multi
 DEFAULT_PRE_SIGNED_ACCESS_KEY_ID = "test"
 DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY = "test"
 
+S3_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
+"""
+S3 is returning a Host Id as part of its exceptions
+"""
+
 AUTHENTICATED_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
 ALL_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AllUsers"
 LOG_DELIVERY_ACL_GROUP = "http://acs.amazonaws.com/groups/s3/LogDelivery"

localstack/services/s3/cors.py

@@ -21,13 +21,13 @@ from localstack.aws.protocol.op_router import RestServiceOperationRouter
 from localstack.aws.spec import get_service_catalog
 from localstack.config import S3_VIRTUAL_HOSTNAME
 from localstack.http import Request, Response
+from localstack.services.s3.constants import S3_HOST_ID
 from localstack.services.s3.utils import S3_VIRTUAL_HOSTNAME_REGEX
 
 # TODO: add more logging statements
 LOG = logging.getLogger(__name__)
 
 _s3_virtual_host_regex = re.compile(S3_VIRTUAL_HOSTNAME_REGEX)
-FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
 
 # TODO: refactor those to expose the needed methods maybe in another way that both can import
 add_default_headers = CorsResponseEnricher.add_cors_headers
@@ -135,7 +135,7 @@ class S3CorsHandler(Handler):
             if is_options_request:
                 context.operation = self._get_op_from_request(request)
                 raise BadRequest(
-                    "Insufficient information. Origin request header needed.", HostId=FAKE_HOST_ID
+                    "Insufficient information. Origin request header needed.", HostId=S3_HOST_ID
                 )
             else:
                 # If the header is missing, Amazon S3 doesn't treat the request as a cross-origin request,
@@ -167,7 +167,7 @@ class S3CorsHandler(Handler):
                     context.operation = self._get_op_from_request(request)
                     raise AccessForbidden(
                         message,
-                        HostId=FAKE_HOST_ID,
+                        HostId=S3_HOST_ID,
                         Method=request.headers.get("Access-Control-Request-Method", "OPTIONS"),
                         ResourceType="BUCKET",
                     )
@@ -182,7 +182,7 @@ class S3CorsHandler(Handler):
                 context.operation = self._get_op_from_request(request)
                 raise AccessForbidden(
                     "CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted by the resource's CORS spec.",
-                    HostId=FAKE_HOST_ID,
+                    HostId=S3_HOST_ID,
                     Method=request.headers.get("Access-Control-Request-Method"),
                     ResourceType="OBJECT",
                 )

localstack/services/s3/models.py

@@ -638,7 +638,7 @@ class KeyStore:
 
     def values(self, *_, **__) -> list[S3Object | S3DeleteMarker]:
         # we create a shallow copy with dict to avoid size changed during iteration
-        return [value for value in dict(self._store).values()]
+        return list(dict(self._store).values())
 
     def is_empty(self) -> bool:
         return not self._store

localstack/services/s3/notifications.py

@@ -21,6 +21,7 @@ from localstack.aws.api.s3 import (
     Event,
     EventBridgeConfiguration,
     EventList,
+    InvalidArgument,
     LambdaFunctionArn,
     LambdaFunctionConfiguration,
     NotificationConfiguration,
@@ -34,8 +35,8 @@ from localstack.aws.api.s3 import (
     TopicConfiguration,
 )
 from localstack.aws.connect import connect_to
+from localstack.services.s3.exceptions import MalformedXML
 from localstack.services.s3.models import S3Bucket, S3DeleteMarker, S3Object
-from localstack.services.s3.utils import _create_invalid_argument_exc
 from localstack.utils.aws import arns
 from localstack.utils.aws.arns import ARN_PARTITION_REGEX, get_partition, parse_arn, s3_bucket_arn
 from localstack.utils.aws.client_types import ServicePrincipal
@@ -104,7 +105,7 @@ class S3EventNotificationContext:
     key_storage_class: StorageClass | None
 
     @classmethod
-    def from_request_context_native(
+    def from_request_context(
         cls,
         request_context: RequestContext,
         s3_bucket: S3Bucket,
@@ -272,26 +273,29 @@ class BaseNotifier:
         arn, argument_name = self._get_arn_value_and_name(configuration)
 
         if not re.match(f"{ARN_PARTITION_REGEX}:{self.service_name}:", arn):
-            raise _create_invalid_argument_exc(
-                "The ARN could not be parsed", name=argument_name, value=arn
+            raise InvalidArgument(
+                "The ARN could not be parsed",
+                ArgumentName=argument_name,
+                ArgumentValue=arn,
             )
+
         if not verification_ctx.skip_destination_validation:
             self._verify_target(arn, verification_ctx)
 
         if filter_rules := configuration.get("Filter", {}).get("Key", {}).get("FilterRules"):
             for rule in filter_rules:
-                rule["Name"] = rule["Name"].capitalize()
-                if rule["Name"] not in ["Suffix", "Prefix"]:
-                    raise _create_invalid_argument_exc(
+                if "Name" not in rule or "Value" not in rule:
+                    raise MalformedXML()
+
+                if rule["Name"].lower() not in ["suffix", "prefix"]:
+                    raise InvalidArgument(
                         "filter rule name must be either prefix or suffix",
-                        rule["Name"],
-                        rule["Value"],
-                    )
-                if not rule["Value"]:
-                    raise _create_invalid_argument_exc(
-                        "filter value cannot be empty", rule["Name"], rule["Value"]
+                        ArgumentName="FilterRule.Name",
+                        ArgumentValue=rule["Name"],
                     )
 
+                rule["Name"] = rule["Name"].capitalize()
+
     @staticmethod
     def _get_test_payload(verification_ctx: BucketVerificationContext):
         return {
@@ -382,7 +386,7 @@ class SqsNotifier(BaseNotifier):
 
     @staticmethod
     def _get_arn_value_and_name(queue_configuration: QueueConfiguration) -> tuple[QueueArn, str]:
-        return queue_configuration.get("QueueArn", ""), "QueueArn"
+        return queue_configuration.get("QueueArn", ""), "Queue"
 
     def _verify_target(self, target_arn: str, verification_ctx: BucketVerificationContext) -> None:
         if not is_api_enabled("sqs"):
@@ -402,13 +406,20 @@ class SqsNotifier(BaseNotifier):
             queue_url = sqs_client.get_queue_url(
                 QueueName=arn_data["resource"], QueueOwnerAWSAccountId=arn_data["account"]
             )["QueueUrl"]
-        except ClientError:
-            LOG.exception("Could not validate the notification destination %s", target_arn)
-            raise _create_invalid_argument_exc(
+        except ClientError as e:
+            code = e.response["Error"]["Code"]
+            LOG.error(
+                "Could not validate the notification destination %s: %s",
+                target_arn,
+                code,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=target_arn,
-                value="The destination queue does not exist",
+                ArgumentName=target_arn,
+                ArgumentValue="The destination queue does not exist",
             )
+
         # send test event with the request metadata for permissions
         # https://docs.aws.amazon.com/AmazonS3/latest/userguide/notification-how-to-event-types-and-destinations.html#supported-notification-event-types
         sqs_client = connect_to(region_name=arn_data["region"]).sqs.request_metadata(
@@ -424,10 +435,10 @@ class SqsNotifier(BaseNotifier):
                 verification_ctx.bucket_name,
                 target_arn,
             )
-            raise _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=target_arn,
-                value="Permissions on the destination queue do not allow S3 to publish notifications from this bucket",
+                ArgumentName=target_arn,
+                ArgumentValue="Permissions on the destination queue do not allow S3 to publish notifications from this bucket",
             ) from e
 
     def notify(self, ctx: S3EventNotificationContext, config: QueueConfiguration):
@@ -454,10 +465,11 @@ class SqsNotifier(BaseNotifier):
                 MessageSystemAttributes=system_attributes,
             )
         except Exception:
-            LOG.exception(
+            LOG.error(
                 'Unable to send notification for S3 bucket "%s" to SQS queue "%s"',
                 ctx.bucket_name,
                 parsed_arn["resource"],
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
 
 
@@ -483,10 +495,10 @@ class SnsNotifier(BaseNotifier):
         try:
             sns_client.get_topic_attributes(TopicArn=target_arn)
         except ClientError:
-            raise _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=target_arn,
-                value="The destination topic does not exist",
+                ArgumentName=target_arn,
+                ArgumentValue="The destination topic does not exist",
             )
 
         sns_client = connect_to(region_name=arn_data["region"]).sns.request_metadata(
@@ -506,10 +518,10 @@ class SnsNotifier(BaseNotifier):
                 verification_ctx.bucket_name,
                 target_arn,
             )
-            raise _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=target_arn,
-                value="Permissions on the destination topic do not allow S3 to publish notifications from this bucket",
+                ArgumentName=target_arn,
+                ArgumentValue="Permissions on the destination topic do not allow S3 to publish notifications from this bucket",
             ) from e
 
     def notify(self, ctx: S3EventNotificationContext, config: TopicConfiguration):
@@ -536,10 +548,11 @@ class SnsNotifier(BaseNotifier):
                 Subject="Amazon S3 Notification",
             )
         except Exception:
-            LOG.exception(
+            LOG.error(
                 'Unable to send notification for S3 bucket "%s" to SNS topic "%s"',
                 ctx.bucket_name,
                 topic_arn,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
 
 
@@ -567,10 +580,10 @@ class LambdaNotifier(BaseNotifier):
         try:
             lambda_client.get_function(FunctionName=target_arn)
         except ClientError:
-            raise _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=target_arn,
-                value="The destination Lambda does not exist",
+                ArgumentName=target_arn,
+                ArgumentValue="The destination Lambda does not exist",
             )
         lambda_client = connect_to(region_name=arn_data["region"]).lambda_.request_metadata(
             source_arn=s3_bucket_arn(verification_ctx.bucket_name, region=verification_ctx.region),
@@ -579,10 +592,10 @@ class LambdaNotifier(BaseNotifier):
         try:
             lambda_client.invoke(FunctionName=target_arn, InvocationType=InvocationType.DryRun)
         except ClientError as e:
-            raise _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Unable to validate the following destination configurations",
-                name=f"{target_arn}, null",
-                value=f"Not authorized to invoke function [{target_arn}]",
+                ArgumentName=f"{target_arn}, null",
+                ArgumentValue=f"Not authorized to invoke function [{target_arn}]",
             ) from e
 
     def notify(self, ctx: S3EventNotificationContext, config: LambdaFunctionConfiguration):
@@ -604,10 +617,11 @@ class LambdaNotifier(BaseNotifier):
                 Payload=payload,
             )
         except Exception:
-            LOG.exception(
+            LOG.error(
                 'Unable to send notification for S3 bucket "%s" to Lambda function "%s".',
                 ctx.bucket_name,
                 lambda_arn,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
 
 
@@ -729,8 +743,10 @@ class EventBridgeNotifier(BaseNotifier):
         try:
             events_client.put_events(Entries=[entry])
         except Exception:
-            LOG.exception(
-                'Unable to send notification for S3 bucket "%s" to EventBridge', ctx.bucket_name
+            LOG.error(
+                'Unable to send notification for S3 bucket "%s" to EventBridge',
+                ctx.bucket_name,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )