localstack-core 4.7.1.dev49__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/services/s3/validation.py

@@ -38,7 +38,6 @@ from localstack.aws.api.s3 import Type as GranteeType
 from localstack.services.s3 import constants as s3_constants
 from localstack.services.s3.exceptions import InvalidRequest, MalformedACLError, MalformedXML
 from localstack.services.s3.utils import (
-    _create_invalid_argument_exc,
     get_class_attrs_from_spec_class,
     get_permission_header_name,
     is_bucket_name_valid,
@@ -87,8 +86,11 @@ def validate_canned_acl(canned_acl: str) -> None:
     Validate the canned ACL value, or raise an Exception
     """
     if canned_acl and canned_acl not in VALID_CANNED_ACLS:
-        ex = _create_invalid_argument_exc(None, "x-amz-acl", canned_acl)
-        raise ex
+        raise InvalidArgument(
+            None,
+            ArgumentName="x-amz-acl",
+            ArgumentValue=canned_acl,
+        )
 
 
 def parse_grants_in_headers(permission: Permission, grantees: str) -> Grants:
@@ -98,16 +100,18 @@ def parse_grants_in_headers(permission: Permission, grantees: str) -> Grants:
         grantee_type, grantee_id = seralized_grantee.split("=")
         grantee_id = grantee_id.strip('"')
         if grantee_type not in ("uri", "id", "emailAddress"):
-            ex = _create_invalid_argument_exc(
+            raise InvalidArgument(
                 "Argument format not recognized",
-                get_permission_header_name(permission),
-                seralized_grantee,
+                ArgumentName=get_permission_header_name(permission),
+                ArgumentValue=seralized_grantee,
             )
-            raise ex
         elif grantee_type == "uri":
             if grantee_id not in s3_constants.VALID_ACL_PREDEFINED_GROUPS:
-                ex = _create_invalid_argument_exc("Invalid group uri", "uri", grantee_id)
-                raise ex
+                raise InvalidArgument(
+                    "Invalid group uri",
+                    ArgumentName="uri",
+                    ArgumentValue=grantee_id,
+                )
             grantee = Grantee(
                 Type=GranteeType.Group,
                 URI=grantee_id,
@@ -115,8 +119,11 @@ def parse_grants_in_headers(permission: Permission, grantees: str) -> Grants:
 
         elif grantee_type == "id":
             if not is_valid_canonical_id(grantee_id):
-                ex = _create_invalid_argument_exc("Invalid id", "id", grantee_id)
-                raise ex
+                raise InvalidArgument(
+                    "Invalid id",
+                    ArgumentName="id",
+                    ArgumentValue=grantee_id,
+                )
             grantee = Grantee(
                 Type=GranteeType.CanonicalUser,
                 ID=grantee_id,
@@ -141,8 +148,11 @@ def validate_acl_acp(acp: AccessControlPolicy) -> None:
         )
 
     if not is_valid_canonical_id(owner_id := acp["Owner"].get("ID", "")):
-        ex = _create_invalid_argument_exc("Invalid id", "CanonicalUser/ID", owner_id)
-        raise ex
+        raise InvalidArgument(
+            "Invalid id",
+            ArgumentName="CanonicalUser/ID",
+            ArgumentValue=owner_id,
+        )
 
     for grant in acp["Grants"]:
         if grant.get("Permission") not in s3_constants.VALID_GRANTEE_PERMISSIONS:
@@ -165,8 +175,11 @@ def validate_acl_acp(acp: AccessControlPolicy) -> None:
             and (grant_uri := grantee.get("URI", ""))
             not in s3_constants.VALID_ACL_PREDEFINED_GROUPS
         ):
-            ex = _create_invalid_argument_exc("Invalid group uri", "Group/URI", grant_uri)
-            raise ex
+            raise InvalidArgument(
+                "Invalid group uri",
+                ArgumentName="Group/URI",
+                ArgumentValue=grant_uri,
+            )
 
         elif grant_type == GranteeType.AmazonCustomerByEmail:
             # TODO: add validation here
@@ -175,8 +188,11 @@ def validate_acl_acp(acp: AccessControlPolicy) -> None:
         elif grant_type == GranteeType.CanonicalUser and not is_valid_canonical_id(
             grantee_id := grantee.get("ID", "")
         ):
-            ex = _create_invalid_argument_exc("Invalid id", "CanonicalUser/ID", grantee_id)
-            raise ex
+            raise InvalidArgument(
+                "Invalid id",
+                ArgumentName="CanonicalUser/ID",
+                ArgumentValue=grantee_id,
+            )
 
 
 def validate_lifecycle_configuration(lifecycle_conf: BucketLifecycleConfiguration) -> None:
@@ -242,12 +258,12 @@ def validate_website_configuration(website_config: WebsiteConfiguration) -> None
     """
     if redirect_all_req := website_config.get("RedirectAllRequestsTo", {}):
         if len(website_config) > 1:
-            ex = _create_invalid_argument_exc(
-                message="RedirectAllRequestsTo cannot be provided in conjunction with other Routing Rules.",
-                name="RedirectAllRequestsTo",
-                value="not null",
+            raise InvalidArgument(
+                "RedirectAllRequestsTo cannot be provided in conjunction with other Routing Rules.",
+                ArgumentName="RedirectAllRequestsTo",
+                ArgumentValue="not null",
             )
-            raise ex
+
         if "HostName" not in redirect_all_req:
             raise MalformedXML()
 
@@ -261,20 +277,18 @@ def validate_website_configuration(website_config: WebsiteConfiguration) -> None
     # required
     # https://docs.aws.amazon.com/AmazonS3/latest/API/API_IndexDocument.html
     if not (index_configuration := website_config.get("IndexDocument")):
-        ex = _create_invalid_argument_exc(
-            message="A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty",
-            name="IndexDocument",
-            value="null",
+        raise InvalidArgument(
+            "A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty",
+            ArgumentName="IndexDocument",
+            ArgumentValue="null",
         )
-        raise ex
 
     if not (index_suffix := index_configuration.get("Suffix")) or "/" in index_suffix:
-        ex = _create_invalid_argument_exc(
-            message="The IndexDocument Suffix is not well formed",
-            name="IndexDocument",
-            value=index_suffix or None,
+        raise InvalidArgument(
+            "The IndexDocument Suffix is not well formed",
+            ArgumentName="IndexDocument",
+            ArgumentValue=index_suffix or None,
         )
-        raise ex
 
     if "ErrorDocument" in website_config and not website_config.get("ErrorDocument", {}).get("Key"):
         raise MalformedXML()

localstack/services/s3/website_hosting.py

@@ -93,8 +93,10 @@ class S3WebsiteHostingHandler:
             return Response(response_body, status=e.response["ResponseMetadata"]["HTTPStatusCode"])
 
         except Exception:
-            LOG.exception(
-                "Exception encountered while trying to serve s3-website at %s", request.url
+            LOG.error(
+                "Exception encountered while trying to serve s3-website at %s",
+                request.url,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             return Response(_create_500_error_string(), status=500)
 

localstack/services/ses/provider.py

@@ -182,6 +182,8 @@ class SesProvider(SesApi, ServiceLifecycleHook):
     #
 
     def on_after_init(self):
+        self._apply_patches()
+
         # Allow sent emails to be retrieved from the SES emails endpoint
         register_ses_api_resource()
 
@@ -197,6 +199,12 @@ class SesProvider(SesApi, ServiceLifecycleHook):
                 return entity.replace("From:", "").strip()
         return None
 
+    def _apply_patches(self) -> None:
+        # Suppress Moto's validation of receipt rule actions. These validations use Moto's implementation of S3, Lambda
+        # and SQS, which fail because these services have been internalised in LocalStack.
+        # Besides, AWS does not run the same validations as evidenced by our AWS-validated tests.
+        SESBackend._validate_receipt_rule_actions = lambda *_: None
+
     #
     # Implementations for SES operations
     #
@@ -517,8 +525,10 @@ class SesProvider(SesApi, ServiceLifecycleHook):
         backend.create_receipt_rule_set(rule_set_name)
         original_rule_set = backend.describe_receipt_rule_set(original_rule_set_name)
 
-        for rule in original_rule_set:
-            backend.create_receipt_rule(rule_set_name, rule)
+        after = None
+        for rule in original_rule_set.rules:
+            backend.create_receipt_rule(rule_set_name, rule, after)
+            after = rule["Name"]
 
         return CloneReceiptRuleSetResponse()
 
@@ -547,7 +557,7 @@ class SesProvider(SesApi, ServiceLifecycleHook):
             )
 
         backend = get_ses_backend(context)
-        if identity not in backend.addresses:
+        if identity not in backend.email_identities:
             raise MessageRejected(f"Identity {identity} is not verified or does not exist.")
 
         # Store the setting in the backend
@@ -626,7 +636,7 @@ class SNSEmitter:
                 Subject="Amazon SES Email Event Notification",
             )
         except ClientError:
-            LOGGER.exception("sending SNS message")
+            LOGGER.error("sending SNS message", exc_info=LOGGER.isEnabledFor(logging.DEBUG))
 
     def emit_delivery_event(self, payload: EventDestinationPayload, sns_topic_arn: str):
         now = datetime.now(tz=UTC)
@@ -659,7 +669,7 @@ class SNSEmitter:
                 Subject="Amazon SES Email Event Notification",
             )
         except ClientError:
-            LOGGER.exception("sending SNS message")
+            LOGGER.error("sending SNS message", exc_info=LOGGER.isEnabledFor(logging.DEBUG))
 
     @staticmethod
     def _client_for_topic(topic_arn: str) -> "SNSClient":
@@ -677,7 +687,7 @@ class SNSEmitter:
 def notify_event_destinations(
     context: RequestContext,
     # FIXME: Moto stores the Event Destinations as a single value when it should be a list
-    event_destinations: dict,
+    event_destinations: EventDestination | list[EventDestination],
     payload: EventDestinationPayload,
     email_type: EmailType,
 ):
@@ -690,11 +700,11 @@ def notify_event_destinations(
         if not event_destination["Enabled"]:
             continue
 
-        sns_destination_arn = event_destination.get("SNSDestination")
+        sns_destination_arn = event_destination.get("SNSDestination", {}).get("TopicARN")
         if not sns_destination_arn:
             continue
 
-        matching_event_types = event_destination.get("EventMatchingTypes") or []
+        matching_event_types = event_destination.get("MatchingEventTypes") or []
         if EventType.send in matching_event_types:
             emitter.emit_send_event(
                 payload, sns_destination_arn, emit_source_arn=email_type != EmailType.TEMPLATED

localstack/services/sns/constants.py

@@ -1,5 +1,8 @@
 import re
 from string import ascii_letters, digits
+from typing import get_args
+
+from localstack.services.sns.v2.models import SnsApplicationPlatforms
 
 SNS_PROTOCOLS = [
     "http",
@@ -13,7 +16,7 @@ SNS_PROTOCOLS = [
     "firehose",
 ]
 
-VALID_SUBSCRIPTION_ATTR_NAME = [
+VALID_SUBSCRIPTION_ATTR_NAME: list[str] = [
     "DeliveryPolicy",
     "FilterPolicy",
     "FilterPolicyScope",
@@ -39,3 +42,6 @@ SUBSCRIPTION_TOKENS_ENDPOINT = "/_aws/sns/subscription-tokens"
 SNS_CERT_ENDPOINT = "/_aws/sns/SimpleNotificationService-6c6f63616c737461636b69736e696365.pem"
 
 DUMMY_SUBSCRIPTION_PRINCIPAL = "arn:{partition}:iam::{account_id}:user/DummySNSPrincipal"
+E164_REGEX = re.compile(r"^\+?[1-9]\d{1,14}$")
+
+VALID_APPLICATION_PLATFORMS = list(get_args(SnsApplicationPlatforms))

localstack/services/sns/executor.py

@@ -18,7 +18,10 @@ def _worker(work_queue: queue.Queue):
             del work_item
 
     except Exception:
-        LOG.exception("Exception in worker")
+        LOG.error(
+            "Exception in worker",
+            exc_info=LOG.isEnabledFor(logging.DEBUG),
+        )
 
 
 class _WorkItem:
@@ -31,7 +34,11 @@ class _WorkItem:
         try:
             self.fn(*self.args, **self.kwargs)
         except Exception:
-            LOG.exception("Unhandled Exception in while running %s", self.fn.__name__)
+            LOG.error(
+                "Unhandled Exception in while running %s",
+                self.fn.__name__,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
 
 
 class TopicPartitionedThreadPoolExecutor:

localstack/services/sns/provider.py

@@ -1,4 +1,5 @@
 import base64
+import contextlib
 import copy
 import functools
 import json
@@ -419,6 +420,10 @@ class SnsProvider(SnsApi, ServiceLifecycleHook):
     def unsubscribe(
         self, context: RequestContext, subscription_arn: subscriptionARN, **kwargs
     ) -> None:
+        if subscription_arn is None:
+            raise InvalidParameterException(
+                "Invalid parameter: SubscriptionArn Reason: no value for required parameter",
+            )
         count = len(subscription_arn.split(":"))
         try:
             parsed_arn = parse_arn(subscription_arn)
@@ -469,7 +474,8 @@ class SnsProvider(SnsApi, ServiceLifecycleHook):
                 subscription_arn=subscription_arn,
             )
 
-        store.topic_subscriptions[subscription["TopicArn"]].remove(subscription_arn)
+        with contextlib.suppress(ValueError):
+            store.topic_subscriptions[subscription["TopicArn"]].remove(subscription_arn)
         store.subscription_filter_policy.pop(subscription_arn, None)
         store.subscriptions.pop(subscription_arn, None)
 
@@ -583,10 +589,7 @@ class SnsProvider(SnsApi, ServiceLifecycleHook):
             raise InvalidParameterException(
                 "Invalid parameter: MessageDeduplicationId Reason: The request includes MessageDeduplicationId parameter that is not valid for this topic type"
             )
-        elif message_group_id:
-            raise InvalidParameterException(
-                "Invalid parameter: MessageGroupId Reason: The request includes MessageGroupId parameter that is not valid for this topic type"
-            )
+
         is_endpoint_publish = target_arn and ":endpoint/" in target_arn
         if message_structure == "json":
             try:

localstack/services/sns/publisher.py

@@ -89,9 +89,10 @@ class TopicPublisher(abc.ABC):
         try:
             self._publish(context=context, subscriber=subscriber)
         except Exception:
-            LOG.exception(
+            LOG.error(
                 "An internal error occurred while trying to send the SNS message %s",
                 context.message,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             return
 
@@ -147,9 +148,10 @@ class EndpointPublisher(abc.ABC):
         try:
             self._publish(context=context, endpoint=endpoint)
         except Exception:
-            LOG.exception(
+            LOG.error(
                 "An internal error occurred while trying to send the SNS message %s",
                 context.message,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             return
 
@@ -295,7 +297,10 @@ class SqsTopicPublisher(TopicPublisher):
             )
             kwargs = self.get_sqs_kwargs(msg_context=message_context, subscriber=subscriber)
         except Exception:
-            LOG.exception("An internal error occurred while trying to format the message for SQS")
+            LOG.error(
+                "An internal error occurred while trying to format the message for SQS",
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
             return
         try:
             queue_url: str = sqs_queue_url_for_arn(subscriber["Endpoint"])
@@ -335,19 +340,29 @@ class SqsTopicPublisher(TopicPublisher):
 
         # SNS now allows regular non-fifo subscriptions to FIFO topics. Validate that the subscription target is fifo
         # before passing the FIFO-only parameters
-        if subscriber["Endpoint"].endswith(".fifo"):
-            if msg_context.message_group_id:
-                kwargs["MessageGroupId"] = msg_context.message_group_id
-            if msg_context.message_deduplication_id:
-                kwargs["MessageDeduplicationId"] = msg_context.message_deduplication_id
-            elif subscriber["TopicArn"].endswith(".fifo"):
-                # Amazon SNS uses the message body provided to generate a unique hash value to use as the deduplication
-                # ID for each message, so you don't need to set a deduplication ID when you send each message.
-                # https://docs.aws.amazon.com/sns/latest/dg/fifo-message-dedup.html
-                content = msg_context.message_content("sqs")
-                kwargs["MessageDeduplicationId"] = hashlib.sha256(
-                    content.encode("utf-8")
-                ).hexdigest()
+
+        # SNS will only forward the `MessageGroupId` for Fair Queues in some scenarios:
+        # - non-FIFO SNS topic to Fair Queue
+        # - FIFO topic to FIFO queue
+        # It will NOT forward it with FIFO topic to regular Queue (possibly used for internal grouping without relying
+        # on SQS capabilities)
+        if subscriber["TopicArn"].endswith(".fifo"):
+            if subscriber["Endpoint"].endswith(".fifo"):
+                if msg_context.message_group_id:
+                    kwargs["MessageGroupId"] = msg_context.message_group_id
+                if msg_context.message_deduplication_id:
+                    kwargs["MessageDeduplicationId"] = msg_context.message_deduplication_id
+                else:
+                    # SNS uses the message body provided to generate a unique hash value to use as the deduplication ID
+                    # for each message, so you don't need to set a deduplication ID when you send each message.
+                    # https://docs.aws.amazon.com/sns/latest/dg/fifo-message-dedup.html
+                    content = msg_context.message_content("sqs")
+                    kwargs["MessageDeduplicationId"] = hashlib.sha256(
+                        content.encode("utf-8")
+                    ).hexdigest()
+
+        elif msg_context.message_group_id:
+            kwargs["MessageGroupId"] = msg_context.message_group_id
 
         # TODO: for message deduplication, we are using the underlying features of the SQS queue
         # however, SQS queue only deduplicate at the Queue level, where the SNS topic deduplicate on the topic level

localstack/services/sns/v2/models.py

@@ -0,0 +1,167 @@
+import itertools
+import time
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Literal, TypedDict
+
+from localstack.aws.api.sns import (
+    MessageAttributeMap,
+    PlatformApplication,
+    PublishBatchRequestEntry,
+    TopicAttributesMap,
+    subscriptionARN,
+    topicARN,
+)
+from localstack.services.stores import (
+    AccountRegionBundle,
+    BaseStore,
+    CrossRegionAttribute,
+    LocalAttribute,
+)
+from localstack.utils.objects import singleton_factory
+from localstack.utils.strings import long_uid
+from localstack.utils.tagging import TaggingService
+
+
+class Topic(TypedDict, total=True):
+    arn: str
+    name: str
+    attributes: TopicAttributesMap
+    subscriptions: list[str]
+
+
+SnsProtocols = Literal[
+    "http", "https", "email", "email-json", "sms", "sqs", "application", "lambda", "firehose"
+]
+
+SnsApplicationPlatforms = Literal[
+    "APNS", "APNS_SANDBOX", "ADM", "FCM", "Baidu", "GCM", "MPNS", "WNS"
+]
+
+
+SMS_ATTRIBUTE_NAMES = [
+    "DeliveryStatusIAMRole",
+    "DeliveryStatusSuccessSamplingRate",
+    "DefaultSenderID",
+    "DefaultSMSType",
+    "UsageReportS3Bucket",
+]
+SMS_TYPES = ["Promotional", "Transactional"]
+SMS_DEFAULT_SENDER_REGEX = r"^(?=[A-Za-z0-9]{1,11}$)(?=.*[A-Za-z])[A-Za-z0-9]+$"
+SnsMessageProtocols = Literal[SnsProtocols, SnsApplicationPlatforms]
+
+
+class SnsSubscription(TypedDict, total=False):
+    """
+    In SNS, Subscription can be represented with only TopicArn, Endpoint, Protocol, SubscriptionArn and Owner, for
+    example in ListSubscriptions. However, when getting a subscription with GetSubscriptionAttributes, it will return
+    the Subscription object merged with its own attributes.
+    This represents this merged object, for internal use and in GetSubscriptionAttributes
+    https://docs.aws.amazon.com/cli/latest/reference/sns/get-subscription-attributes.html
+    """
+
+    TopicArn: topicARN
+    Endpoint: str
+    Protocol: SnsProtocols
+    SubscriptionArn: subscriptionARN
+    PendingConfirmation: Literal["true", "false"]
+    Owner: str | None
+    SubscriptionPrincipal: str | None
+    FilterPolicy: str | None
+    FilterPolicyScope: Literal["MessageAttributes", "MessageBody"]
+    RawMessageDelivery: Literal["true", "false"]
+    ConfirmationWasAuthenticated: Literal["true", "false"]
+    SubscriptionRoleArn: str | None
+    DeliveryPolicy: str | None
+
+
+@singleton_factory
+def global_sns_message_sequence():
+    # creates a 20-digit number used as the start for the global sequence, adds 100 for it to be different from SQS's
+    # mostly for testing purpose, both global sequence would be initialized at the same and be identical
+    start = int(time.time() + 100) << 33
+    # itertools.count is thread safe over the GIL since its getAndIncrement operation is a single python bytecode op
+    return itertools.count(start)
+
+
+def get_next_sequence_number():
+    return next(global_sns_message_sequence())
+
+
+class SnsMessageType(StrEnum):
+    Notification = "Notification"
+    SubscriptionConfirmation = "SubscriptionConfirmation"
+    UnsubscribeConfirmation = "UnsubscribeConfirmation"
+
+
+@dataclass
+class SnsMessage:
+    type: SnsMessageType
+    message: (
+        str | dict
+    )  # can be Dict if after being JSON decoded for validation if structure is `json`
+    message_attributes: MessageAttributeMap | None = None
+    message_structure: str | None = None
+    subject: str | None = None
+    message_deduplication_id: str | None = None
+    message_group_id: str | None = None
+    token: str | None = None
+    message_id: str = field(default_factory=long_uid)
+    is_fifo: bool | None = False
+    sequencer_number: str | None = None
+
+    def __post_init__(self):
+        if self.message_attributes is None:
+            self.message_attributes = {}
+        if self.is_fifo:
+            self.sequencer_number = str(get_next_sequence_number())
+
+    def message_content(self, protocol: SnsMessageProtocols) -> str:
+        """
+        Helper function to retrieve the message content for the right protocol if the StructureMessage is `json`
+        See https://docs.aws.amazon.com/sns/latest/dg/sns-send-custom-platform-specific-payloads-mobile-devices.html
+        https://docs.aws.amazon.com/sns/latest/dg/example_sns_Publish_section.html
+        :param protocol:
+        :return: message content as string
+        """
+        if self.message_structure == "json":
+            return self.message.get(protocol, self.message.get("default"))
+
+        return self.message
+
+    @classmethod
+    def from_batch_entry(cls, entry: PublishBatchRequestEntry, is_fifo=False) -> "SnsMessage":
+        return cls(
+            type=SnsMessageType.Notification,
+            message=entry["Message"],
+            subject=entry.get("Subject"),
+            message_structure=entry.get("MessageStructure"),
+            message_attributes=entry.get("MessageAttributes"),
+            message_deduplication_id=entry.get("MessageDeduplicationId"),
+            message_group_id=entry.get("MessageGroupId"),
+            is_fifo=is_fifo,
+        )
+
+
+class SnsStore(BaseStore):
+    topics: dict[str, Topic] = LocalAttribute(default=dict)
+
+    # maps subscription ARN to SnsSubscription
+    subscriptions: dict[str, SnsSubscription] = LocalAttribute(default=dict)
+
+    # filter policy are stored as JSON string in subscriptions, store the decoded result Dict
+    subscription_filter_policy: dict[subscriptionARN, dict] = LocalAttribute(default=dict)
+
+    # maps confirmation token to subscription ARN
+    subscription_tokens: dict[str, str] = LocalAttribute(default=dict)
+
+    # maps platform application arns to platform applications
+    platform_applications: dict[str, PlatformApplication] = LocalAttribute(default=dict)
+
+    # topic/subscription independent default values for sending sms messages
+    sms_attributes: dict[str, str] = LocalAttribute(default=dict)
+
+    TAGS: TaggingService = CrossRegionAttribute(default=TaggingService)
+
+
+sns_stores = AccountRegionBundle("sns", SnsStore)