localstack-core 4.6.1.dev60__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/services/sns/v2/utils.py

@@ -0,0 +1,130 @@
+import base64
+import json
+from uuid import uuid4
+
+from botocore.utils import InvalidArnException
+
+from localstack.aws.api.sns import InvalidParameterException
+from localstack.services.sns.constants import E164_REGEX, VALID_SUBSCRIPTION_ATTR_NAME
+from localstack.utils.aws.arns import ArnData, parse_arn
+from localstack.utils.strings import short_uid, to_bytes, to_str
+
+
+def parse_and_validate_topic_arn(topic_arn: str | None) -> ArnData:
+    return _parse_and_validate_arn(topic_arn, "Topic")
+
+
+def parse_and_validate_platform_application_arn(platform_application_arn: str | None) -> ArnData:
+    return _parse_and_validate_arn(platform_application_arn, "PlatformApplication")
+
+
+def _parse_and_validate_arn(arn: str | None, resource_type: str) -> ArnData:
+    arn = arn or ""
+    try:
+        return parse_arn(arn)
+    except InvalidArnException:
+        count = len(arn.split(":"))
+        raise InvalidParameterException(
+            f"Invalid parameter: {resource_type}Arn Reason: An ARN must have at least 6 elements, not {count}"
+        )
+
+
+def is_valid_e164_number(number: str) -> bool:
+    return E164_REGEX.match(number) is not None
+
+
+def validate_subscription_attribute(
+    attribute_name: str,
+    attribute_value: str,
+    topic_arn: str,
+    endpoint: str,
+    is_subscribe_call: bool = False,
+) -> None:
+    """
+    Validate the subscription attribute to be set. See:
+    https://docs.aws.amazon.com/sns/latest/api/API_SetSubscriptionAttributes.html
+    :param attribute_name: the subscription attribute name, must be in VALID_SUBSCRIPTION_ATTR_NAME
+    :param attribute_value: the subscription attribute value
+    :param topic_arn: the topic_arn of the subscription, needed to know if it is FIFO
+    :param endpoint: the subscription endpoint (like an SQS queue ARN)
+    :param is_subscribe_call: the error message is different if called from Subscribe or SetSubscriptionAttributes
+    :raises InvalidParameterException
+    :return:
+    """
+    error_prefix = (
+        "Invalid parameter: Attributes Reason: " if is_subscribe_call else "Invalid parameter: "
+    )
+    if attribute_name not in VALID_SUBSCRIPTION_ATTR_NAME:
+        raise InvalidParameterException(f"{error_prefix}AttributeName")
+
+    if attribute_name == "FilterPolicy":
+        try:
+            json.loads(attribute_value or "{}")
+        except json.JSONDecodeError:
+            raise InvalidParameterException(f"{error_prefix}FilterPolicy: failed to parse JSON.")
+    elif attribute_name == "FilterPolicyScope":
+        if attribute_value not in ("MessageAttributes", "MessageBody"):
+            raise InvalidParameterException(
+                f"{error_prefix}FilterPolicyScope: Invalid value [{attribute_value}]. "
+                f"Please use either MessageBody or MessageAttributes"
+            )
+    elif attribute_name == "RawMessageDelivery":
+        # TODO: only for SQS and https(s) subs, + firehose
+        if attribute_value.lower() not in ("true", "false"):
+            raise InvalidParameterException(
+                f"{error_prefix}RawMessageDelivery: Invalid value [{attribute_value}]. "
+                f"Must be true or false."
+            )
+
+    elif attribute_name == "RedrivePolicy":
+        try:
+            dlq_target_arn = json.loads(attribute_value).get("deadLetterTargetArn", "")
+        except json.JSONDecodeError:
+            raise InvalidParameterException(f"{error_prefix}RedrivePolicy: failed to parse JSON.")
+        try:
+            parsed_arn = parse_arn(dlq_target_arn)
+        except InvalidArnException:
+            raise InvalidParameterException(
+                f"{error_prefix}RedrivePolicy: deadLetterTargetArn is an invalid arn"
+            )
+
+        if topic_arn.endswith(".fifo"):
+            if endpoint.endswith(".fifo") and (
+                not parsed_arn["resource"].endswith(".fifo") or "sqs" not in parsed_arn["service"]
+            ):
+                raise InvalidParameterException(
+                    f"{error_prefix}RedrivePolicy: must use a FIFO queue as DLQ for a FIFO Subscription to a FIFO Topic."
+                )
+
+
+def create_subscription_arn(topic_arn: str) -> str:
+    # This is the format of a Subscription ARN
+    # arn:aws:sns:us-west-2:123456789012:my-topic:8a21d249-4329-4871-acc6-7be709c6ea7f
+    return f"{topic_arn}:{uuid4()}"
+
+
+def encode_subscription_token_with_region(region: str) -> str:
+    """
+    Create a 64 characters Subscription Token with the region encoded
+    :param region:
+    :return: a subscription token with the region encoded
+    """
+    return ((region.encode() + b"/").hex() + short_uid() * 8)[:64]
+
+
+def get_next_page_token_from_arn(resource_arn: str) -> str:
+    return to_str(base64.b64encode(to_bytes(resource_arn)))
+
+
+def get_region_from_subscription_token(token: str) -> str:
+    """
+    Try to decode and return the region from a subscription token
+    :param token:
+    :return: the region if able to decode it
+    :raises: InvalidParameterException if the token is invalid
+    """
+    try:
+        region = token.split("2f", maxsplit=1)[0]
+        return bytes.fromhex(region).decode("utf-8")
+    except (IndexError, ValueError, TypeError, UnicodeDecodeError):
+        raise InvalidParameterException("Invalid parameter: Token")

localstack/services/sqs/constants.py

@@ -9,7 +9,7 @@ MSG_CONTENT_REGEX = "^[\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd\U00010000-\U
 ATTR_NAME_CHAR_REGEX = "^[\u00c0-\u017fa-zA-Z0-9_.-]*$"
 ATTR_NAME_PREFIX_SUFFIX_REGEX = r"^(?!(aws\.|amazon\.|\.)).*(?<!\.)$"
 ATTR_TYPE_REGEX = "^(String|Number|Binary).*$"
-FIFO_MSG_REGEX = "^[0-9a-zA-z!\"#$%&'()*+,./:;<=>?@[\\]^_`{|}~-]*$"
+FIFO_MSG_REGEX = "^[0-9a-zA-z!\"#$%&'()*+,./:;<=>?@[\\]^_`{|}~-]{1,128}$"
 
 DEDUPLICATION_INTERVAL_IN_SEC = 5 * 60
 
@@ -18,7 +18,7 @@ DEDUPLICATION_INTERVAL_IN_SEC = 5 * 60
 RECENTLY_DELETED_TIMEOUT = 60
 
 # the default maximum message size in SQS
-DEFAULT_MAXIMUM_MESSAGE_SIZE = 262144
+DEFAULT_MAXIMUM_MESSAGE_SIZE = 1048576
 INTERNAL_QUEUE_ATTRIBUTES = [
     # these attributes cannot be changed by set_queue_attributes and should
     # therefore be ignored when comparing queue attributes for create_queue

localstack/services/sqs/developer_api.py

@@ -0,0 +1,205 @@
+import logging
+from typing import Literal
+
+from werkzeug import Request as WerkzeugRequest
+
+from localstack.aws.api import CommonServiceException, ServiceException
+from localstack.aws.api.sqs import (
+    Message,
+    QueueAttributeName,
+    QueueDoesNotExist,
+    ReceiveMessageResult,
+)
+from localstack.aws.protocol.parser import create_parser
+from localstack.aws.protocol.serializer import aws_response_serializer
+from localstack.aws.spec import load_service
+from localstack.http import Request, route
+from localstack.services.sqs.models import (
+    FifoQueue,
+    SqsMessage,
+    SqsQueue,
+    StandardQueue,
+    sqs_stores,
+    to_sqs_api_message,
+)
+from localstack.services.sqs.utils import (
+    parse_queue_url,
+)
+from localstack.utils.aws.request_context import extract_region_from_headers
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidAddress(ServiceException):
+    code = "InvalidAddress"
+    message = "The address https://queue.amazonaws.com/ is not valid for this endpoint."
+    sender_fault = True
+    status_code = 404
+
+
+def get_sqs_protocol(request: Request) -> Literal["query", "json"]:
+    content_type = request.headers.get("Content-Type")
+    return "json" if content_type == "application/x-amz-json-1.0" else "query"
+
+
+def sqs_auto_protocol_aws_response_serializer(service_name: str, operation: str):
+    def _decorate(fn):
+        def _proxy(*args, **kwargs):
+            # extract request from function invocation (decorator can be used for methods as well as for functions).
+            if len(args) > 0 and isinstance(args[0], WerkzeugRequest):
+                # function
+                request = args[0]
+            elif len(args) > 1 and isinstance(args[1], WerkzeugRequest):
+                # method (arg[0] == self)
+                request = args[1]
+            elif "request" in kwargs:
+                request = kwargs["request"]
+            else:
+                raise ValueError(f"could not find Request in signature of function {fn}")
+
+            protocol = get_sqs_protocol(request)
+            return aws_response_serializer(service_name, operation, protocol)(fn)(*args, **kwargs)
+
+        return _proxy
+
+    return _decorate
+
+
+class SqsDeveloperApi:
+    """
+    A set of SQS developer tool endpoints:
+
+    - ``/_aws/sqs/messages``: list SQS messages without side effects, compatible with ``ReceiveMessage``.
+    """
+
+    def __init__(self, stores=None):
+        self.stores = stores or sqs_stores
+
+    @route("/_aws/sqs/messages", methods=["GET", "POST"])
+    @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
+    def list_messages(self, request: Request) -> ReceiveMessageResult:
+        """
+        This endpoint expects a ``QueueUrl`` request parameter (either as query arg or form parameter), similar to
+        the ``ReceiveMessage`` operation. It will parse the Queue URL generated by one of the SQS endpoint strategies.
+        """
+
+        if "x-amz-" in request.mimetype or "x-www-form-urlencoded" in request.mimetype:
+            # only parse the request using a parser if it comes from an AWS client
+            protocol = get_sqs_protocol(request)
+            operation, service_request = create_parser(
+                load_service("sqs", protocol=protocol)
+            ).parse(request)
+            if operation.name != "ReceiveMessage":
+                raise CommonServiceException(
+                    "InvalidRequest", "This endpoint only accepts ReceiveMessage calls"
+                )
+        else:
+            service_request = dict(request.values)
+
+        if not service_request.get("QueueUrl"):
+            raise QueueDoesNotExist()
+
+        try:
+            account_id, region, queue_name = parse_queue_url(service_request.get("QueueUrl"))
+        except ValueError:
+            LOG.error(
+                "Error while parsing Queue URL from request values: %s",
+                service_request.get,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
+            raise InvalidAddress()
+
+        if not region:
+            region = extract_region_from_headers(request.headers)
+
+        return self._get_and_serialize_messages(request, region, account_id, queue_name)
+
+    @route("/_aws/sqs/messages/<region>/<account_id>/<queue_name>")
+    @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
+    def list_messages_for_queue_url(
+        self, request: Request, region: str, account_id: str, queue_name: str
+    ) -> ReceiveMessageResult:
+        """
+        This endpoint extracts the region, account_id, and queue_name directly from the URL rather than requiring the
+        QueueUrl as parameter.
+        """
+        return self._get_and_serialize_messages(request, region, account_id, queue_name)
+
+    def _get_and_serialize_messages(
+        self,
+        request: Request,
+        region: str,
+        account_id: str,
+        queue_name: str,
+    ) -> ReceiveMessageResult:
+        show_invisible = request.values.get("ShowInvisible", "").lower() in ["true", "1"]
+        show_delayed = request.values.get("ShowDelayed", "").lower() in ["true", "1"]
+
+        try:
+            store = self.stores[account_id][region]
+            queue = store.queues[queue_name]
+        except KeyError:
+            LOG.info(
+                "no queue named %s in region %s and account %s", queue_name, region, account_id
+            )
+            raise QueueDoesNotExist()
+
+        messages = self._collect_messages(
+            queue, show_invisible=show_invisible, show_delayed=show_delayed
+        )
+
+        return ReceiveMessageResult(Messages=messages)
+
+    def _collect_messages(
+        self, queue: SqsQueue, show_invisible: bool = False, show_delayed: bool = False
+    ) -> list[Message]:
+        """
+        Retrieves from a given SqsQueue all visible messages without causing any side effects (not setting any
+        receive timestamps, receive counts, or visibility state).
+
+        :param queue: the queue
+        :param show_invisible: show invisible messages as well
+        :param show_delayed: show delayed messages as well
+        :return: a list of messages
+        """
+        receipt_handle = "SQS/BACKDOOR/ACCESS"  # dummy receipt handle
+
+        sqs_messages: list[SqsMessage] = []
+
+        if show_invisible:
+            sqs_messages.extend(queue.inflight)
+
+        if isinstance(queue, StandardQueue):
+            sqs_messages.extend(queue.visible.queue)
+        elif isinstance(queue, FifoQueue):
+            if show_invisible:
+                for inflight_group in queue.inflight_groups:
+                    # messages that have been received are held in ``queue.inflight``, even for FIFO queues. however,
+                    # for fifo queues, messages that are in the same message group as messages that have been
+                    # received, are also considered invisible, and are held here in ``inflight_group.messages``.
+                    for sqs_message in inflight_group.messages:
+                        sqs_messages.append(sqs_message)
+
+            for message_group in queue.message_group_queue.queue:
+                # these are all messages of message groups that are visible
+                for sqs_message in message_group.messages:
+                    sqs_messages.append(sqs_message)
+        else:
+            raise ValueError(f"unknown queue type {type(queue)}")
+
+        if show_delayed:
+            sqs_messages.extend(queue.delayed)
+
+        messages = []
+
+        for sqs_message in sqs_messages:
+            message: Message = to_sqs_api_message(sqs_message, [QueueAttributeName.All], ["All"])
+            # these are all non-standard fields so we squelch the linter
+            if show_invisible:
+                message["Attributes"]["IsVisible"] = str(sqs_message.is_visible).lower()  # noqa
+            if show_delayed:
+                message["Attributes"]["IsDelayed"] = str(sqs_message.is_delayed).lower()  # noqa
+            messages.append(message)
+            message["ReceiptHandle"] = receipt_handle
+
+        return messages

localstack/services/sqs/models.py

@@ -1,3 +1,4 @@
+import copy
 import hashlib
 import heapq
 import inspect
@@ -8,7 +9,6 @@ import threading
 import time
 from datetime import datetime
 from queue import Empty
-from typing import Dict, Optional, Set
 
 from localstack import config
 from localstack.aws.api import RequestContext
@@ -16,6 +16,7 @@ from localstack.aws.api.sqs import (
     AttributeNameList,
     InvalidAttributeName,
     Message,
+    MessageAttributeNameList,
     MessageSystemAttributeName,
     QueueAttributeMap,
     QueueAttributeName,
@@ -30,12 +31,15 @@ from localstack.services.sqs.exceptions import (
 )
 from localstack.services.sqs.queue import InterruptiblePriorityQueue, InterruptibleQueue
 from localstack.services.sqs.utils import (
+    create_message_attribute_hash,
     encode_move_task_handle,
     encode_receipt_handle,
     extract_receipt_handle_info,
     global_message_sequence,
     guess_endpoint_strategy_and_host,
     is_message_deduplication_id_required,
+    message_filter_attributes,
+    message_filter_message_attributes,
 )
 from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
 from localstack.utils.aws.arns import get_partition
@@ -53,11 +57,11 @@ class SqsMessage:
     created: float
     visibility_timeout: int
     receive_count: int
-    delay_seconds: Optional[int]
-    receipt_handles: Set[str]
-    last_received: Optional[float]
-    first_received: Optional[float]
-    visibility_deadline: Optional[float]
+    delay_seconds: int | None
+    receipt_handles: set[str]
+    last_received: float | None
+    first_received: float | None
+    visibility_deadline: float | None
     deleted: bool
     priority: float
     message_deduplication_id: str
@@ -104,15 +108,15 @@ class SqsMessage:
         )
 
     @property
-    def message_group_id(self) -> Optional[str]:
+    def message_group_id(self) -> str | None:
         return self.message["Attributes"].get(MessageSystemAttributeName.MessageGroupId)
 
     @property
-    def message_deduplication_id(self) -> Optional[str]:
+    def message_deduplication_id(self) -> str | None:
         return self.message["Attributes"].get(MessageSystemAttributeName.MessageDeduplicationId)
 
     @property
-    def dead_letter_queue_source_arn(self) -> Optional[str]:
+    def dead_letter_queue_source_arn(self) -> str | None:
         return self.message["Attributes"].get(MessageSystemAttributeName.DeadLetterQueueSourceArn)
 
     @property
@@ -154,7 +158,7 @@ class SqsMessage:
         """
         Returns false if the message has a visibility deadline that is in the future.
 
-        :return: whether the message is visibile or not.
+        :return: whether the message is visible or not.
         """
         if self.visibility_deadline is None:
             return True
@@ -191,6 +195,41 @@ class SqsMessage:
         return f"SqsMessage(id={self.message_id},group={self.message_group_id})"
 
 
+def to_sqs_api_message(
+    standard_message: SqsMessage,
+    attribute_names: AttributeNameList = None,
+    message_attribute_names: MessageAttributeNameList = None,
+) -> Message:
+    """
+    Utility function to convert an SQS message from LocalStack's internal representation to the AWS API
+    concept 'Message', which is the format returned by the ``ReceiveMessage`` operation.
+
+    :param standard_message: A LocalStack SQS message
+    :param attribute_names: the attribute name list to filter
+    :param message_attribute_names: the message attribute names to filter
+    :return: a copy of the original Message with updated message attributes and MD5 attribute hash sums
+    """
+    # prepare message for receiver
+    message = copy.deepcopy(standard_message.message)
+
+    # update system attributes of the message copy
+    message["Attributes"][MessageSystemAttributeName.ApproximateFirstReceiveTimestamp] = str(
+        int((standard_message.first_received or 0) * 1000)
+    )
+
+    # filter attributes for receiver
+    message_filter_attributes(message, attribute_names)
+    message_filter_message_attributes(message, message_attribute_names)
+    if message.get("MessageAttributes"):
+        message["MD5OfMessageAttributes"] = create_message_attribute_hash(
+            message["MessageAttributes"]
+        )
+    else:
+        # delete the value that was computed when creating the message
+        message.pop("MD5OfMessageAttributes", None)
+    return message
+
+
 class ReceiveMessageResult:
     """
     Object to communicate the result of a "receive messages" operation between the SqsProvider and
@@ -272,11 +311,11 @@ class SqsQueue:
     tags: TagMap
 
     purge_in_progress: bool
-    purge_timestamp: Optional[float]
+    purge_timestamp: float | None
 
-    delayed: Set[SqsMessage]
-    inflight: Set[SqsMessage]
-    receipts: Dict[str, SqsMessage]
+    delayed: set[SqsMessage]
+    inflight: set[SqsMessage]
+    receipts: dict[str, SqsMessage]
 
     def __init__(self, name: str, region: str, account_id: str, attributes=None, tags=None) -> None:
         self.name = name
@@ -393,13 +432,13 @@ class SqsQueue:
         )
 
     @property
-    def redrive_policy(self) -> Optional[dict]:
+    def redrive_policy(self) -> dict | None:
         if policy_document := self.attributes.get(QueueAttributeName.RedrivePolicy):
             return json.loads(policy_document)
         return None
 
     @property
-    def max_receive_count(self) -> Optional[int]:
+    def max_receive_count(self) -> int | None:
         """
         Returns the maxReceiveCount attribute of the redrive policy. If no redrive policy is set, then it
         returns None.
@@ -678,7 +717,7 @@ class SqsQueue:
         if QueueAttributeName.All in attribute_names:
             attribute_names = self.attributes.keys()
 
-        result: Dict[QueueAttributeName, str] = {}
+        result: dict[QueueAttributeName, str] = {}
 
         for attr in attribute_names:
             try:
@@ -740,7 +779,7 @@ class SqsQueue:
 
 class StandardQueue(SqsQueue):
     visible: InterruptiblePriorityQueue[SqsMessage]
-    inflight: Set[SqsMessage]
+    inflight: set[SqsMessage]
 
     def __init__(self, name: str, region: str, account_id: str, attributes=None, tags=None) -> None:
         super().__init__(name, region, account_id, attributes, tags)
@@ -771,13 +810,12 @@ class StandardQueue(SqsQueue):
                 f"Value {message_deduplication_id} for parameter MessageDeduplicationId is invalid. Reason: The "
                 f"request includes a parameter that is not valid for this queue type."
             )
-        if isinstance(message_group_id, str):
-            raise InvalidParameterValueException(
-                f"Value {message_group_id} for parameter MessageGroupId is invalid. Reason: The request include "
-                f"parameter that is not valid for this queue type."
-            )
 
-        standard_message = SqsMessage(time.time(), message)
+        standard_message = SqsMessage(
+            time.time(),
+            message,
+            message_group_id=message_group_id,
+        )
 
         if visibility_timeout is not None:
             standard_message.visibility_timeout = visibility_timeout
@@ -960,7 +998,7 @@ class FifoQueue(SqsQueue):
     TODO: raise exceptions when trying to remove a message with an expired receipt handle
     """
 
-    deduplication: Dict[str, SqsMessage]
+    deduplication: dict[str, SqsMessage]
     message_groups: dict[str, MessageGroup]
     inflight_groups: set[MessageGroup]
     message_group_queue: InterruptibleQueue
@@ -1010,13 +1048,12 @@ class FifoQueue(SqsQueue):
         }
 
     def update_delay_seconds(self, value: int):
-        super(FifoQueue, self).update_delay_seconds(value)
+        super().update_delay_seconds(value)
         for message in self.delayed:
             message.delay_seconds = value
 
     def _pre_delete_checks(self, message: SqsMessage, receipt_handle: str) -> None:
-        _, _, _, last_received = extract_receipt_handle_info(receipt_handle)
-        if time.time() - float(last_received) > message.visibility_timeout:
+        if message.is_visible:
             raise InvalidParameterValueException(
                 f"Value {receipt_handle} for parameter ReceiptHandle is invalid. Reason: The receipt handle has expired."
             )
@@ -1067,10 +1104,8 @@ class FifoQueue(SqsQueue):
             # use the attribute from the queue
             fifo_message.visibility_timeout = self.visibility_timeout
 
-        if delay_seconds is not None:
-            fifo_message.delay_seconds = delay_seconds
-        else:
-            fifo_message.delay_seconds = self.delay_seconds
+        # FIFO queues always use the queue level setting for 'DelaySeconds'
+        fifo_message.delay_seconds = self.delay_seconds
 
         original_message = self.deduplication.get(dedup_id)
         if (
@@ -1159,7 +1194,7 @@ class FifoQueue(SqsQueue):
         timeout = wait_time_seconds or 0
         start = time.time()
 
-        received_groups: Set[MessageGroup] = set()
+        received_groups: set[MessageGroup] = set()
 
         # collect messages over potentially multiple groups
         while True:
@@ -1315,11 +1350,11 @@ class FifoQueue(SqsQueue):
 
 
 class SqsStore(BaseStore):
-    queues: Dict[str, SqsQueue] = LocalAttribute(default=dict)
+    queues: dict[str, SqsQueue] = LocalAttribute(default=dict)
 
-    deleted: Dict[str, float] = LocalAttribute(default=dict)
+    deleted: dict[str, float] = LocalAttribute(default=dict)
 
-    move_tasks: Dict[str, MessageMoveTask] = LocalAttribute(default=dict)
+    move_tasks: dict[str, MessageMoveTask] = LocalAttribute(default=dict)
     """Maps task IDs to their ``MoveMessageTask`` object. Task IDs can be found by decoding a task handle."""
 
     def expire_deleted(self):