localstack-core 4.7.1.dev139__py3-none-any.whl → 4.10.1.dev42__py3-none-any.whl

localstack/services/sns/v2/utils.py

@@ -0,0 +1,138 @@
+import base64
+import json
+from uuid import uuid4
+
+from botocore.utils import InvalidArnException
+
+from localstack.aws.api.sns import InvalidParameterException
+from localstack.services.sns.constants import E164_REGEX, VALID_SUBSCRIPTION_ATTR_NAME
+from localstack.utils.aws.arns import ArnData, parse_arn
+from localstack.utils.strings import short_uid, to_bytes, to_str
+
+
+def parse_and_validate_topic_arn(topic_arn: str | None) -> ArnData:
+    return _parse_and_validate_arn(topic_arn, "Topic")
+
+
+def parse_and_validate_platform_application_arn(platform_application_arn: str | None) -> ArnData:
+    return _parse_and_validate_arn(platform_application_arn, "PlatformApplication")
+
+
+def _parse_and_validate_arn(arn: str | None, resource_type: str) -> ArnData:
+    arn = arn or ""
+    try:
+        return parse_arn(arn)
+    except InvalidArnException:
+        count = len(arn.split(":"))
+        raise InvalidParameterException(
+            f"Invalid parameter: {resource_type}Arn Reason: An ARN must have at least 6 elements, not {count}"
+        )
+
+
+def is_valid_e164_number(number: str) -> bool:
+    return E164_REGEX.match(number) is not None
+
+
+def validate_subscription_attribute(
+    attribute_name: str,
+    attribute_value: str,
+    topic_arn: str,
+    endpoint: str,
+    is_subscribe_call: bool = False,
+) -> None:
+    """
+    Validate the subscription attribute to be set. See:
+    https://docs.aws.amazon.com/sns/latest/api/API_SetSubscriptionAttributes.html
+    :param attribute_name: the subscription attribute name, must be in VALID_SUBSCRIPTION_ATTR_NAME
+    :param attribute_value: the subscription attribute value
+    :param topic_arn: the topic_arn of the subscription, needed to know if it is FIFO
+    :param endpoint: the subscription endpoint (like an SQS queue ARN)
+    :param is_subscribe_call: the error message is different if called from Subscribe or SetSubscriptionAttributes
+    :raises InvalidParameterException
+    :return:
+    """
+    error_prefix = (
+        "Invalid parameter: Attributes Reason: " if is_subscribe_call else "Invalid parameter: "
+    )
+    if attribute_name not in VALID_SUBSCRIPTION_ATTR_NAME:
+        raise InvalidParameterException(f"{error_prefix}AttributeName")
+
+    if attribute_name == "FilterPolicy":
+        try:
+            json.loads(attribute_value or "{}")
+        except json.JSONDecodeError:
+            raise InvalidParameterException(f"{error_prefix}FilterPolicy: failed to parse JSON.")
+    elif attribute_name == "FilterPolicyScope":
+        if attribute_value not in ("MessageAttributes", "MessageBody"):
+            raise InvalidParameterException(
+                f"{error_prefix}FilterPolicyScope: Invalid value [{attribute_value}]. "
+                f"Please use either MessageBody or MessageAttributes"
+            )
+    elif attribute_name == "RawMessageDelivery":
+        # TODO: only for SQS and https(s) subs, + firehose
+        if attribute_value.lower() not in ("true", "false"):
+            raise InvalidParameterException(
+                f"{error_prefix}RawMessageDelivery: Invalid value [{attribute_value}]. "
+                f"Must be true or false."
+            )
+
+    elif attribute_name == "RedrivePolicy":
+        try:
+            dlq_target_arn = json.loads(attribute_value).get("deadLetterTargetArn", "")
+        except json.JSONDecodeError:
+            raise InvalidParameterException(f"{error_prefix}RedrivePolicy: failed to parse JSON.")
+        try:
+            parsed_arn = parse_arn(dlq_target_arn)
+        except InvalidArnException:
+            raise InvalidParameterException(
+                f"{error_prefix}RedrivePolicy: deadLetterTargetArn is an invalid arn"
+            )
+
+        if topic_arn.endswith(".fifo"):
+            if endpoint.endswith(".fifo") and (
+                not parsed_arn["resource"].endswith(".fifo") or "sqs" not in parsed_arn["service"]
+            ):
+                raise InvalidParameterException(
+                    f"{error_prefix}RedrivePolicy: must use a FIFO queue as DLQ for a FIFO Subscription to a FIFO Topic."
+                )
+
+
+def create_subscription_arn(topic_arn: str) -> str:
+    # This is the format of a Subscription ARN
+    # arn:aws:sns:us-west-2:123456789012:my-topic:8a21d249-4329-4871-acc6-7be709c6ea7f
+    return f"{topic_arn}:{uuid4()}"
+
+
+def create_platform_endpoint_arn(
+    platform_application_arn: str,
+) -> str:
+    # This is the format of an Endpoint Arn
+    # arn:aws:sns:us-west-2:1234567890:endpoint/GCM/MyApplication/12345678-abcd-9012-efgh-345678901234
+    return f"{platform_application_arn.replace('app', 'endpoint', 1)}/{uuid4()}"
+
+
+def encode_subscription_token_with_region(region: str) -> str:
+    """
+    Create a 64 characters Subscription Token with the region encoded
+    :param region:
+    :return: a subscription token with the region encoded
+    """
+    return ((region.encode() + b"/").hex() + short_uid() * 8)[:64]
+
+
+def get_next_page_token_from_arn(resource_arn: str) -> str:
+    return to_str(base64.b64encode(to_bytes(resource_arn)))
+
+
+def get_region_from_subscription_token(token: str) -> str:
+    """
+    Try to decode and return the region from a subscription token
+    :param token:
+    :return: the region if able to decode it
+    :raises: InvalidParameterException if the token is invalid
+    """
+    try:
+        region = token.split("2f", maxsplit=1)[0]
+        return bytes.fromhex(region).decode("utf-8")
+    except (IndexError, ValueError, TypeError, UnicodeDecodeError):
+        raise InvalidParameterException("Invalid parameter: Token")

localstack/services/sqs/developer_api.py

@@ -0,0 +1,205 @@
+import logging
+from typing import Literal
+
+from werkzeug import Request as WerkzeugRequest
+
+from localstack.aws.api import CommonServiceException, ServiceException
+from localstack.aws.api.sqs import (
+    Message,
+    QueueAttributeName,
+    QueueDoesNotExist,
+    ReceiveMessageResult,
+)
+from localstack.aws.protocol.parser import create_parser
+from localstack.aws.protocol.serializer import aws_response_serializer
+from localstack.aws.spec import load_service
+from localstack.http import Request, route
+from localstack.services.sqs.models import (
+    FifoQueue,
+    SqsMessage,
+    SqsQueue,
+    StandardQueue,
+    sqs_stores,
+    to_sqs_api_message,
+)
+from localstack.services.sqs.utils import (
+    parse_queue_url,
+)
+from localstack.utils.aws.request_context import extract_region_from_headers
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidAddress(ServiceException):
+    code = "InvalidAddress"
+    message = "The address https://queue.amazonaws.com/ is not valid for this endpoint."
+    sender_fault = True
+    status_code = 404
+
+
+def get_sqs_protocol(request: Request) -> Literal["query", "json"]:
+    content_type = request.headers.get("Content-Type")
+    return "json" if content_type == "application/x-amz-json-1.0" else "query"
+
+
+def sqs_auto_protocol_aws_response_serializer(service_name: str, operation: str):
+    def _decorate(fn):
+        def _proxy(*args, **kwargs):
+            # extract request from function invocation (decorator can be used for methods as well as for functions).
+            if len(args) > 0 and isinstance(args[0], WerkzeugRequest):
+                # function
+                request = args[0]
+            elif len(args) > 1 and isinstance(args[1], WerkzeugRequest):
+                # method (arg[0] == self)
+                request = args[1]
+            elif "request" in kwargs:
+                request = kwargs["request"]
+            else:
+                raise ValueError(f"could not find Request in signature of function {fn}")
+
+            protocol = get_sqs_protocol(request)
+            return aws_response_serializer(service_name, operation, protocol)(fn)(*args, **kwargs)
+
+        return _proxy
+
+    return _decorate
+
+
+class SqsDeveloperApi:
+    """
+    A set of SQS developer tool endpoints:
+
+    - ``/_aws/sqs/messages``: list SQS messages without side effects, compatible with ``ReceiveMessage``.
+    """
+
+    def __init__(self, stores=None):
+        self.stores = stores or sqs_stores
+
+    @route("/_aws/sqs/messages", methods=["GET", "POST"])
+    @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
+    def list_messages(self, request: Request) -> ReceiveMessageResult:
+        """
+        This endpoint expects a ``QueueUrl`` request parameter (either as query arg or form parameter), similar to
+        the ``ReceiveMessage`` operation. It will parse the Queue URL generated by one of the SQS endpoint strategies.
+        """
+
+        if "x-amz-" in request.mimetype or "x-www-form-urlencoded" in request.mimetype:
+            # only parse the request using a parser if it comes from an AWS client
+            protocol = get_sqs_protocol(request)
+            operation, service_request = create_parser(
+                load_service("sqs", protocol=protocol)
+            ).parse(request)
+            if operation.name != "ReceiveMessage":
+                raise CommonServiceException(
+                    "InvalidRequest", "This endpoint only accepts ReceiveMessage calls"
+                )
+        else:
+            service_request = dict(request.values)
+
+        if not service_request.get("QueueUrl"):
+            raise QueueDoesNotExist()
+
+        try:
+            account_id, region, queue_name = parse_queue_url(service_request.get("QueueUrl"))
+        except ValueError:
+            LOG.error(
+                "Error while parsing Queue URL from request values: %s",
+                service_request.get,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
+            raise InvalidAddress()
+
+        if not region:
+            region = extract_region_from_headers(request.headers)
+
+        return self._get_and_serialize_messages(request, region, account_id, queue_name)
+
+    @route("/_aws/sqs/messages/<region>/<account_id>/<queue_name>")
+    @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
+    def list_messages_for_queue_url(
+        self, request: Request, region: str, account_id: str, queue_name: str
+    ) -> ReceiveMessageResult:
+        """
+        This endpoint extracts the region, account_id, and queue_name directly from the URL rather than requiring the
+        QueueUrl as parameter.
+        """
+        return self._get_and_serialize_messages(request, region, account_id, queue_name)
+
+    def _get_and_serialize_messages(
+        self,
+        request: Request,
+        region: str,
+        account_id: str,
+        queue_name: str,
+    ) -> ReceiveMessageResult:
+        show_invisible = request.values.get("ShowInvisible", "").lower() in ["true", "1"]
+        show_delayed = request.values.get("ShowDelayed", "").lower() in ["true", "1"]
+
+        try:
+            store = self.stores[account_id][region]
+            queue = store.queues[queue_name]
+        except KeyError:
+            LOG.info(
+                "no queue named %s in region %s and account %s", queue_name, region, account_id
+            )
+            raise QueueDoesNotExist()
+
+        messages = self._collect_messages(
+            queue, show_invisible=show_invisible, show_delayed=show_delayed
+        )
+
+        return ReceiveMessageResult(Messages=messages)
+
+    def _collect_messages(
+        self, queue: SqsQueue, show_invisible: bool = False, show_delayed: bool = False
+    ) -> list[Message]:
+        """
+        Retrieves from a given SqsQueue all visible messages without causing any side effects (not setting any
+        receive timestamps, receive counts, or visibility state).
+
+        :param queue: the queue
+        :param show_invisible: show invisible messages as well
+        :param show_delayed: show delayed messages as well
+        :return: a list of messages
+        """
+        receipt_handle = "SQS/BACKDOOR/ACCESS"  # dummy receipt handle
+
+        sqs_messages: list[SqsMessage] = []
+
+        if show_invisible:
+            sqs_messages.extend(queue.inflight)
+
+        if isinstance(queue, StandardQueue):
+            sqs_messages.extend(queue.visible.queue)
+        elif isinstance(queue, FifoQueue):
+            if show_invisible:
+                for inflight_group in queue.inflight_groups:
+                    # messages that have been received are held in ``queue.inflight``, even for FIFO queues. however,
+                    # for fifo queues, messages that are in the same message group as messages that have been
+                    # received, are also considered invisible, and are held here in ``inflight_group.messages``.
+                    for sqs_message in inflight_group.messages:
+                        sqs_messages.append(sqs_message)
+
+            for message_group in queue.message_group_queue.queue:
+                # these are all messages of message groups that are visible
+                for sqs_message in message_group.messages:
+                    sqs_messages.append(sqs_message)
+        else:
+            raise ValueError(f"unknown queue type {type(queue)}")
+
+        if show_delayed:
+            sqs_messages.extend(queue.delayed)
+
+        messages = []
+
+        for sqs_message in sqs_messages:
+            message: Message = to_sqs_api_message(sqs_message, [QueueAttributeName.All], ["All"])
+            # these are all non-standard fields so we squelch the linter
+            if show_invisible:
+                message["Attributes"]["IsVisible"] = str(sqs_message.is_visible).lower()  # noqa
+            if show_delayed:
+                message["Attributes"]["IsDelayed"] = str(sqs_message.is_delayed).lower()  # noqa
+            messages.append(message)
+            message["ReceiptHandle"] = receipt_handle
+
+        return messages

localstack/services/sqs/models.py

@@ -1,3 +1,4 @@
+import copy
 import hashlib
 import heapq
 import inspect
@@ -15,6 +16,7 @@ from localstack.aws.api.sqs import (
     AttributeNameList,
     InvalidAttributeName,
     Message,
+    MessageAttributeNameList,
     MessageSystemAttributeName,
     QueueAttributeMap,
     QueueAttributeName,
@@ -29,12 +31,15 @@ from localstack.services.sqs.exceptions import (
 )
 from localstack.services.sqs.queue import InterruptiblePriorityQueue, InterruptibleQueue
 from localstack.services.sqs.utils import (
+    create_message_attribute_hash,
     encode_move_task_handle,
     encode_receipt_handle,
     extract_receipt_handle_info,
     global_message_sequence,
     guess_endpoint_strategy_and_host,
     is_message_deduplication_id_required,
+    message_filter_attributes,
+    message_filter_message_attributes,
 )
 from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
 from localstack.utils.aws.arns import get_partition
@@ -153,7 +158,7 @@ class SqsMessage:
         """
         Returns false if the message has a visibility deadline that is in the future.
 
-        :return: whether the message is visibile or not.
+        :return: whether the message is visible or not.
         """
         if self.visibility_deadline is None:
             return True
@@ -190,6 +195,41 @@ class SqsMessage:
         return f"SqsMessage(id={self.message_id},group={self.message_group_id})"
 
 
+def to_sqs_api_message(
+    standard_message: SqsMessage,
+    attribute_names: AttributeNameList = None,
+    message_attribute_names: MessageAttributeNameList = None,
+) -> Message:
+    """
+    Utility function to convert an SQS message from LocalStack's internal representation to the AWS API
+    concept 'Message', which is the format returned by the ``ReceiveMessage`` operation.
+
+    :param standard_message: A LocalStack SQS message
+    :param attribute_names: the attribute name list to filter
+    :param message_attribute_names: the message attribute names to filter
+    :return: a copy of the original Message with updated message attributes and MD5 attribute hash sums
+    """
+    # prepare message for receiver
+    message = copy.deepcopy(standard_message.message)
+
+    # update system attributes of the message copy
+    message["Attributes"][MessageSystemAttributeName.ApproximateFirstReceiveTimestamp] = str(
+        int((standard_message.first_received or 0) * 1000)
+    )
+
+    # filter attributes for receiver
+    message_filter_attributes(message, attribute_names)
+    message_filter_message_attributes(message, message_attribute_names)
+    if message.get("MessageAttributes"):
+        message["MD5OfMessageAttributes"] = create_message_attribute_hash(
+            message["MessageAttributes"]
+        )
+    else:
+        # delete the value that was computed when creating the message
+        message.pop("MD5OfMessageAttributes", None)
+    return message
+
+
 class ReceiveMessageResult:
     """
     Object to communicate the result of a "receive messages" operation between the SqsProvider and
@@ -274,7 +314,8 @@ class SqsQueue:
     purge_timestamp: float | None
 
     delayed: set[SqsMessage]
-    inflight: set[SqsMessage]
+    # Simulating an ordered set in python. Only the keys are used and of interest.
+    inflight: dict[SqsMessage, None]
     receipts: dict[str, SqsMessage]
 
     def __init__(self, name: str, region: str, account_id: str, attributes=None, tags=None) -> None:
@@ -286,7 +327,7 @@ class SqsQueue:
         self.tags = tags or {}
 
         self.delayed = set()
-        self.inflight = set()
+        self.inflight = {}
         self.receipts = {}
 
         self.attributes = self.default_attributes()
@@ -473,7 +514,7 @@ class SqsQueue:
                 )
                 # Terminating the visibility timeout for a message
                 # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html#terminating-message-visibility-timeout
-                self.inflight.remove(standard_message)
+                del self.inflight[standard_message]
                 self._put_message(standard_message)
 
     def remove(self, receipt_handle: str):
@@ -566,9 +607,17 @@ class SqsQueue:
                     standard_message,
                     self.arn,
                 )
-                self.inflight.remove(standard_message)
+                del self.inflight[standard_message]
                 self._put_message(standard_message)
 
+    def add_inflight_message(self, message: SqsMessage):
+        """
+        We are simulating an ordered set with a dict. When a value is added, it is added as key to the dict, which
+        is all we need. Hence all "values" in this ordered set are None
+        :param message: The message to put in flight
+        """
+        self.inflight[message] = None
+
     def enqueue_delayed_messages(self):
         if not self.delayed:
             return
@@ -739,7 +788,6 @@ class SqsQueue:
 
 class StandardQueue(SqsQueue):
     visible: InterruptiblePriorityQueue[SqsMessage]
-    inflight: set[SqsMessage]
 
     def __init__(self, name: str, region: str, account_id: str, attributes=None, tags=None) -> None:
         super().__init__(name, region, account_id, attributes, tags)
@@ -883,13 +931,13 @@ class StandardQueue(SqsQueue):
             if message.visibility_timeout == 0:
                 self.visible.put_nowait(message)
             else:
-                self.inflight.add(message)
+                self.add_inflight_message(message)
 
         return result
 
     def _on_remove_message(self, message: SqsMessage):
         try:
-            self.inflight.remove(message)
+            del self.inflight[message]
         except KeyError:
             # this likely means the message was removed with an expired receipt handle unfortunately this
             # means we need to scan the queue for the element and remove it from there, and then re-heapify
@@ -1013,8 +1061,7 @@ class FifoQueue(SqsQueue):
             message.delay_seconds = value
 
     def _pre_delete_checks(self, message: SqsMessage, receipt_handle: str) -> None:
-        _, _, _, last_received = extract_receipt_handle_info(receipt_handle)
-        if time.time() - float(last_received) > message.visibility_timeout:
+        if message.is_visible:
             raise InvalidParameterValueException(
                 f"Value {receipt_handle} for parameter ReceiptHandle is invalid. Reason: The receipt handle has expired."
             )
@@ -1110,6 +1157,26 @@ class FifoQueue(SqsQueue):
             elif previously_empty:
                 self.message_group_queue.put_nowait(message_group)
 
+    def requeue_inflight_messages(self):
+        if not self.inflight:
+            return
+
+        with self.mutex:
+            messages = list(self.inflight)
+            for standard_message in messages:
+                # in fifo, an invisible message blocks potentially visible messages afterwards
+                # this can happen for example if multiple message of the same group are received at once, then one
+                # message of this batch has its visibility timeout extended
+                if not standard_message.is_visible:
+                    return
+                LOG.debug(
+                    "re-queueing inflight messages %s into queue %s",
+                    standard_message,
+                    self.arn,
+                )
+                del self.inflight[standard_message]
+                self._put_message(standard_message)
+
     def remove_expired_messages(self):
         with self.mutex:
             retention_period = self.message_retention_period
@@ -1239,8 +1306,7 @@ class FifoQueue(SqsQueue):
                 if message.visibility_timeout == 0:
                     self._put_message(message)
                 else:
-                    self.inflight.add(message)
-
+                    self.add_inflight_message(message)
         return result
 
     def _on_remove_message(self, message: SqsMessage):
@@ -1249,7 +1315,7 @@ class FifoQueue(SqsQueue):
 
         with self.mutex:
             try:
-                self.inflight.remove(message)
+                del self.inflight[message]
             except KeyError:
                 # in FIFO queues, this should not happen, as expired receipt handles cannot be used to
                 # delete a message.