localstack-core 4.6.1.dev74__py3-none-any.whl → 4.10.1.dev7__py3-none-any.whl

localstack/aws/forwarder.py

@@ -3,10 +3,13 @@ This module contains utilities to call a backend (e.g., an external service proc
 DynamoDBLocal) from a service provider.
 """
 
-from typing import Any, Callable, Mapping, Optional, Union
+from collections.abc import Callable, Mapping
+from typing import Any
 
 from botocore.awsrequest import AWSPreparedRequest, prepare_request_dict
 from botocore.config import Config as BotoConfig
+from botocore.model import OperationModel
+from botocore.serialize import create_serializer
 from werkzeug.datastructures import Headers
 
 from localstack.aws.api.core import (
@@ -18,7 +21,7 @@ from localstack.aws.api.core import (
 from localstack.aws.client import create_http_request, parse_response, raise_service_exception
 from localstack.aws.connect import connect_to
 from localstack.aws.skeleton import DispatchTable, create_dispatch_table
-from localstack.aws.spec import load_service
+from localstack.aws.spec import ProtocolName, load_service
 from localstack.constants import AWS_REGION_US_EAST_1
 from localstack.http import Response
 from localstack.http.proxy import Proxy
@@ -54,7 +57,7 @@ class AwsRequestProxy:
         self,
         context: RequestContext,
         service_request: ServiceRequest = None,
-    ) -> Optional[Union[ServiceResponse, Response]]:
+    ) -> ServiceResponse | Response | None:
         """Method to satisfy the ``ServiceRequestHandler`` protocol."""
         return self.forward(context, service_request)
 
@@ -62,7 +65,7 @@ class AwsRequestProxy:
         self,
         context: RequestContext,
         service_request: ServiceRequest = None,
-    ) -> Optional[Union[ServiceResponse, Response]]:
+    ) -> ServiceResponse | Response | None:
         """
         Forwards the given request to the backend configured by ``endpoint_url``.
 
@@ -78,7 +81,7 @@ class AwsRequestProxy:
         if not self.parse_response:
             return http_response
         parsed_response = parse_response(
-            context.operation, http_response, self.include_response_metadata
+            context.operation, context.protocol, http_response, self.include_response_metadata
         )
         raise_service_exception(http_response, parsed_response)
         return parsed_response
@@ -89,6 +92,7 @@ class AwsRequestProxy:
             action=original.operation.name,
             parameters=service_request,
             region=original.region,
+            protocol=original.protocol,
         )
         # update the newly created context with non-payload specific request headers (the payload can differ from
         # the original request, f.e. it could be JSON encoded now while the initial request was CBOR encoded)
@@ -183,7 +187,9 @@ def dispatch_to_backend(
     :raises ServiceException: if the dispatcher returned an error response
     """
     http_response = http_request_dispatcher(context)
-    parsed_response = parse_response(context.operation, http_response, include_response_metadata)
+    parsed_response = parse_response(
+        context.operation, context.protocol, http_response, include_response_metadata
+    )
     raise_service_exception(http_response, parsed_response)
     return parsed_response
 
@@ -195,9 +201,10 @@ _non_validating_boto_config = BotoConfig(parameter_validation=False)
 def create_aws_request_context(
     service_name: str,
     action: str,
+    protocol: ProtocolName = None,
     parameters: Mapping[str, Any] = None,
     region: str = None,
-    endpoint_url: Optional[str] = None,
+    endpoint_url: str | None = None,
 ) -> RequestContext:
     """
     This is a stripped-down version of what the botocore client does to perform an HTTP request from a client call. A
@@ -209,6 +216,7 @@ def create_aws_request_context(
 
     :param service_name: the AWS service
     :param action: the action to invoke
+    :param protocol: the protocol to use
     :param parameters: the invocation parameters
     :param region: the region name (default is us-east-1)
     :param endpoint_url: the endpoint to call (defaults to localstack)
@@ -221,6 +229,8 @@ def create_aws_request_context(
 
     service = load_service(service_name)
     operation = service.operation_model(action)
+    # TODO: remove this once every usage upstream has been removed
+    protocol = protocol or service.resolved_protocol
 
     # we re-use botocore internals here to serialize the HTTP request,
     # but deactivate validation (validation errors should be handled by the backend)
@@ -242,8 +252,14 @@ def create_aws_request_context(
         endpoint_url = "http://localhost.localstack.cloud"
     # pre-process the request args (some params are modified using botocore event handlers)
     parameters = client._emit_api_params(parameters, operation, request_context)
-    request_dict = client._convert_to_request_dict(
-        parameters, operation, endpoint_url, context=request_context
+
+    request_dict = _convert_to_request_dict_with_protocol(
+        client=client,
+        protocol=protocol,
+        api_params=parameters,
+        operation_model=operation,
+        endpoint_url=endpoint_url,
+        context=request_context,
     )
 
     if auth_path := request_dict.get("auth_path"):
@@ -265,7 +281,39 @@ def create_aws_request_context(
     context = RequestContext(request=create_http_request(aws_request))
     context.service = service
     context.operation = operation
+    context.protocol = protocol
     context.region = region
     context.service_request = parameters
 
     return context
+
+
+def _convert_to_request_dict_with_protocol(
+    client,
+    protocol: ProtocolName,
+    api_params: dict,
+    operation_model: OperationModel,
+    endpoint_url: str,
+    context: dict,
+    set_user_agent_header: bool = True,
+) -> dict:
+    """
+    This function is taken from botocore Client._convert_to_request_dict, but we are overriding the serializer
+    Botocore does not expose a way to create a client with a specific protocol, but we need this functionality
+    to support multi-protocols.
+    """
+    serializer = create_serializer(protocol, include_validation=False)
+    request_dict = serializer.serialize_to_request(api_params, operation_model)
+    if not client._client_config.inject_host_prefix:
+        request_dict.pop("host_prefix", None)
+    if set_user_agent_header:
+        user_agent = client._user_agent_creator.to_string()
+    else:
+        user_agent = None
+    prepare_request_dict(
+        request_dict,
+        endpoint_url=endpoint_url,
+        user_agent=user_agent,
+        context=context,
+    )
+    return request_dict

localstack/aws/gateway.py

@@ -1,5 +1,3 @@
-import typing as t
-
 from rolo.gateway import Gateway as RoloGateway
 from rolo.response import Response
 
@@ -17,7 +15,7 @@ class Gateway(RoloGateway):
         response_handlers: list[Handler] = None,
         finalizers: list[Handler] = None,
         exception_handlers: list[ExceptionHandler] = None,
-        context_class: t.Type[RequestContext] = None,
+        context_class: type[RequestContext] = None,
     ) -> None:
         super().__init__(
             request_handlers,

localstack/aws/handlers/analytics.py

@@ -1,6 +1,5 @@
 import logging
 import threading
-from typing import Optional
 
 from localstack import config
 from localstack.aws.api import RequestContext
@@ -52,7 +51,7 @@ class ServiceRequestCounter:
             )
         )
 
-    def _get_err_type(self, context: RequestContext, response: Response) -> Optional[str]:
+    def _get_err_type(self, context: RequestContext, response: Response) -> str | None:
         """
         Attempts to re-use the existing service_response, or parse and return the error type from the response body,
         e.g. ``ResourceInUseException``.
@@ -61,7 +60,7 @@ class ServiceRequestCounter:
             if context.service_exception:
                 return context.service_exception.code
 
-            response = parse_response(context.operation, response)
+            response = parse_response(context.operation, context.protocol, response)
             return response["Error"]["Code"]
         except Exception:
             if config.DEBUG_ANALYTICS:

localstack/aws/handlers/cors.py

@@ -4,7 +4,6 @@ A set of handlers which handle Cross Origin Resource Sharing (CORS).
 
 import logging
 import re
-from typing import List, Set
 from urllib.parse import urlparse
 
 from werkzeug.datastructures import Headers
@@ -83,7 +82,7 @@ ALLOWED_CORS_RESPONSE_HEADERS = [
 ]
 
 
-def _get_allowed_cors_internal_domains() -> Set[str]:
+def _get_allowed_cors_internal_domains() -> set[str]:
     """
     Construct the list of allowed internal domains for CORS enforcement purposes
     Defined as function to allow easier testing with monkeypatch of config values
@@ -94,7 +93,7 @@ def _get_allowed_cors_internal_domains() -> Set[str]:
 _ALLOWED_INTERNAL_DOMAINS = _get_allowed_cors_internal_domains()
 
 
-def _get_allowed_cors_ports() -> Set[int]:
+def _get_allowed_cors_ports() -> set[int]:
     """
     Construct the list of allowed ports for CORS enforcement purposes
     Defined as function to allow easier testing with monkeypatch of config values
@@ -105,7 +104,7 @@ def _get_allowed_cors_ports() -> Set[int]:
 _ALLOWED_INTERNAL_PORTS = _get_allowed_cors_ports()
 
 
-def _get_allowed_cors_origins() -> List[str]:
+def _get_allowed_cors_origins() -> list[str]:
     """Construct the list of allowed origins for CORS enforcement purposes"""
     result = [
         # allow access from Web app and localhost domains
@@ -210,7 +209,7 @@ class CorsEnforcer(Handler):
         return True
 
     @staticmethod
-    def _is_in_allowed_origins(allowed_origins: List[str], origin: str) -> bool:
+    def _is_in_allowed_origins(allowed_origins: list[str], origin: str) -> bool:
         """Returns true if the `origin` is in the `allowed_origins`."""
         for allowed_origin in allowed_origins:
             if allowed_origin == "*" or origin == allowed_origin:

localstack/aws/handlers/internal_requests.py

@@ -20,7 +20,12 @@ class InternalRequestParamsEnricher(Handler):
             try:
                 dto = MappingProxyType(load_dto(header))
             except Exception as e:
-                LOG.exception("Error loading request parameters '%s', Error: %s", header, e)
+                LOG.error(
+                    "Error loading request parameters '%s', Error: %s",
+                    header,
+                    e,
+                    exc_info=LOG.isEnabledFor(logging.DEBUG),
+                )
                 return
 
             context.internal_request_params = dto

localstack/aws/handlers/logging.py

@@ -1,8 +1,8 @@
 """Handlers for logging."""
 
 import logging
+import types
 from functools import cached_property
-from typing import Type
 
 from localstack.aws.api import RequestContext, ServiceException
 from localstack.aws.chain import ExceptionHandler, HandlerChain
@@ -22,6 +22,14 @@ class ExceptionLogger(ExceptionHandler):
     def __init__(self, logger=None):
         self.logger = logger or LOG
 
+        try:
+            import moto.core.exceptions
+
+            self._moto_service_exception = moto.core.exceptions.ServiceException
+        except (ModuleNotFoundError, AttributeError):
+            # Moto may not be available in stripped-down versions of LocalStack, like LocalStack S3 image.
+            self._moto_service_exception = types.EllipsisType
+
     def __call__(
         self,
         chain: HandlerChain,
@@ -29,9 +37,10 @@ class ExceptionLogger(ExceptionHandler):
         context: RequestContext,
         response: Response,
     ):
-        if isinstance(exception, ServiceException):
+        if isinstance(exception, (ServiceException, self._moto_service_exception)):
             # We do not want to log an error/stacktrace if the handler is working as expected, but chooses to throw
-            # a service exception
+            # a service exception. It may also throw a Moto ServiceException, which should not be logged either
+            # because ServiceExceptionHandler understands it.
             return
         if self.logger.isEnabledFor(level=logging.DEBUG):
             self.logger.exception("exception during call chain", exc_info=exception)
@@ -71,7 +80,7 @@ class ResponseLogger:
         )
 
     # make sure loggers are loaded after logging config is loaded
-    def _prepare_logger(self, logger: logging.Logger, formatter: Type):
+    def _prepare_logger(self, logger: logging.Logger, formatter: type):
         if logger.isEnabledFor(logging.DEBUG):
             logger.propagate = False
             handler = create_default_handler(logger.level)

localstack/aws/handlers/metric_handler.py

@@ -1,10 +1,15 @@
+import csv
 import logging
-from typing import List, Optional
+import os
+from datetime import datetime
+from pathlib import Path
 
 from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.chain import HandlerChain
+from localstack.constants import ENV_INTERNAL_TEST_STORE_METRICS_PATH
 from localstack.http import Response
+from localstack.utils.strings import short_uid
 
 LOG = logging.getLogger(__name__)
 
@@ -16,7 +21,7 @@ class MetricHandlerItem:
 
     request_id: str
     request_context: RequestContext
-    parameters_after_parse: Optional[List[str]]
+    parameters_after_parse: list[str] | None
 
     def __init__(self, request_contex: RequestContext) -> None:
         super().__init__()
@@ -35,7 +40,7 @@ class Metric:
     headers: str
     parameters: str
     status_code: int
-    response_code: Optional[str]
+    response_code: str | None
     exception: str
     origin: str
     xfail: bool
@@ -134,10 +139,36 @@ class Metric:
 
 
 class MetricHandler:
-    metric_data: List[Metric] = []
+    metric_data: list[Metric] = []
 
     def __init__(self) -> None:
         self.metrics_handler_items = {}
+        self.local_filename = None
+
+        if self.should_store_metric_locally():
+            self.local_filename = self.create_local_file()
+
+    @staticmethod
+    def should_store_metric_locally() -> bool:
+        return config.is_collect_metrics_mode() and config.store_test_metrics_in_local_filesystem()
+
+    @staticmethod
+    def create_local_file():
+        folder = Path(
+            os.environ.get(ENV_INTERNAL_TEST_STORE_METRICS_PATH, "/tmp/localstack-metrics")
+        )
+        if not folder.exists():
+            folder.mkdir(parents=True, exist_ok=True)
+        LOG.debug("Metric reports will be stored in %s", folder)
+        filename = (
+            folder
+            / f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv"
+        )
+        with open(filename, "w") as fd:
+            LOG.debug("Creating new metric data file %s", filename)
+            writer = csv.writer(fd)
+            writer.writerow(Metric.RAW_DATA_HEADER)
+        return filename
 
     def create_metric_handler_item(
         self, chain: HandlerChain, context: RequestContext, response: Response
@@ -195,7 +226,15 @@ class MetricHandler:
         )
         # refrain from adding duplicates
         if metric not in MetricHandler.metric_data:
-            MetricHandler.metric_data.append(metric)
+            self.append_metric(metric)
 
         # cleanup
         del self.metrics_handler_items[context]
+
+    def append_metric(self, metric: Metric):
+        if self.should_store_metric_locally():
+            with open(self.local_filename, "a") as fd:
+                writer = csv.writer(fd)
+                writer.writerow(metric)
+        else:
+            MetricHandler.metric_data.append(metric)

localstack/aws/handlers/service.py

@@ -2,22 +2,23 @@
 
 import logging
 import traceback
+import types
 from collections import defaultdict
-from typing import Any, Dict, Union
+from typing import Any
 
 from botocore.model import OperationModel, ServiceModel
 
 from localstack import config
 from localstack.http import Response
-from localstack.utils.coverage_docs import get_coverage_link_for_service
 
+from ...utils.coverage_docs import get_coverage_link_for_service
 from ..api import CommonServiceException, RequestContext, ServiceException
 from ..api.core import ServiceOperation
 from ..chain import CompositeResponseHandler, ExceptionHandler, Handler, HandlerChain
 from ..client import parse_response, parse_service_exception
 from ..protocol.parser import RequestParser, create_parser
 from ..protocol.serializer import create_serializer
-from ..protocol.service_router import determine_aws_service_model
+from ..protocol.service_router import determine_aws_protocol, determine_aws_service_model
 from ..skeleton import Skeleton, create_skeleton
 
 LOG = logging.getLogger(__name__)
@@ -33,6 +34,8 @@ class ServiceNameParser(Handler):
         # example). If it is already set, we can skip the parsing of the request. It is very important for S3, because
         # parsing the request will consume the data stream and prevent streaming.
         if context.service:
+            if not context.protocol:
+                context.protocol = determine_aws_protocol(context.request, context.service)
             return
 
         service_model = determine_aws_service_model(context.request)
@@ -41,6 +44,7 @@ class ServiceNameParser(Handler):
             return
 
         context.service = service_model
+        context.protocol = determine_aws_protocol(context.request, service_model)
 
 
 class ServiceRequestParser(Handler):
@@ -49,10 +53,10 @@ class ServiceRequestParser(Handler):
     already be resolved in the RequestContext (e.g., through a ServiceNameParser)
     """
 
-    parsers: Dict[str, RequestParser]
+    parsers: dict[str, RequestParser]
 
     def __init__(self):
-        self.parsers = dict()
+        self.parsers = {}
 
     def __call__(self, chain: HandlerChain, context: RequestContext, response: Response):
         # determine service
@@ -63,7 +67,7 @@ class ServiceRequestParser(Handler):
         return self.parse_and_enrich(context)
 
     def parse_and_enrich(self, context: RequestContext):
-        parser = create_parser(context.service)
+        parser = create_parser(context.service, context.protocol)
         operation, instance = parser.parse(context.request)
 
         # enrich context
@@ -89,10 +93,10 @@ class ServiceRequestRouter(Handler):
     Routes ServiceOperations to Handlers.
     """
 
-    handlers: Dict[ServiceOperation, Handler]
+    handlers: dict[ServiceOperation, Handler]
 
     def __init__(self):
-        self.handlers = dict()
+        self.handlers = {}
 
     def __call__(self, chain: HandlerChain, context: RequestContext, response: Response):
         if not context.service:
@@ -118,7 +122,7 @@ class ServiceRequestRouter(Handler):
 
         self.handlers[key] = handler
 
-    def add_provider(self, provider: Any, service: Union[str, ServiceModel]):
+    def add_provider(self, provider: Any, service: str | ServiceModel):
         self.add_skeleton(create_skeleton(service, provider))
 
     def add_skeleton(self, skeleton: Skeleton):
@@ -137,7 +141,7 @@ class ServiceRequestRouter(Handler):
         operation_name = operation.name
         message = f"no handler for operation '{operation_name}' on service '{service_name}'"
         error = CommonServiceException("InternalFailure", message, status_code=501)
-        serializer = create_serializer(context.service)
+        serializer = create_serializer(context.service, context.protocol)
         return serializer.serialize_error_to_response(
             error, operation, context.request.headers, context.request_id
         )
@@ -153,6 +157,15 @@ class ServiceExceptionSerializer(ExceptionHandler):
     def __init__(self):
         self.handle_internal_failures = True
 
+        try:
+            import moto.core.exceptions
+
+            self._moto_service_exception = moto.core.exceptions.ServiceException
+        except (ModuleNotFoundError, AttributeError) as exc:
+            # Moto may not be available in stripped-down versions of LocalStack, like LocalStack S3 image.
+            LOG.debug("Unable to set up Moto ServiceException translation: %s", exc)
+            self._moto_service_exception = types.EllipsisType
+
     def __call__(
         self,
         chain: HandlerChain,
@@ -176,43 +189,48 @@ class ServiceExceptionSerializer(ExceptionHandler):
             action_name = operation.name
             exception_message: str | None = exception.args[0] if exception.args else None
             message = exception_message or get_coverage_link_for_service(service_name, action_name)
-            LOG.info(message)
             error = CommonServiceException("InternalFailure", message, status_code=501)
-            context.service_exception = error
+            LOG.info(message)
+
+        elif isinstance(exception, self._moto_service_exception):
+            # Translate Moto ServiceException to native ServiceException if Moto is available.
+            # This allows handler chain to gracefully handles Moto errors when provider handlers invoke Moto methods directly.
+            error = CommonServiceException(
+                code=exception.code,
+                message=exception.message,
+            )
 
         elif not isinstance(exception, ServiceException):
             if not self.handle_internal_failures:
                 return
 
-            if config.DEBUG:
-                exception = "".join(
+            if config.INCLUDE_STACK_TRACES_IN_HTTP_RESPONSE:
+                message = "".join(
                     traceback.format_exception(
                         type(exception), value=exception, tb=exception.__traceback__
                     )
                 )
+            else:
+                message = str(exception)
 
             # wrap exception for serialization
             if operation:
                 operation_name = operation.name
-                msg = "exception while calling %s.%s: %s" % (
-                    service_name,
-                    operation_name,
-                    exception,
-                )
+                msg = f"exception while calling {service_name}.{operation_name}: {message}"
             else:
                 # just use any operation for mocking purposes (the parser needs it to populate the default response)
                 operation = context.service.operation_model(context.service.operation_names[0])
-                msg = "exception while calling %s with unknown operation: %s" % (
-                    service_name,
-                    exception,
-                )
+                msg = f"exception while calling {service_name} with unknown operation: {message}"
 
             status_code = 501 if config.FAIL_FAST else 500
 
-            error = CommonServiceException("InternalError", msg, status_code=status_code)
-            context.service_exception = error
+            error = CommonServiceException(
+                "InternalError", msg, status_code=status_code
+            ).with_traceback(exception.__traceback__)
 
-        serializer = create_serializer(context.service)  # TODO: serializer cache
+        context.service_exception = error
+
+        serializer = create_serializer(context.service, context.protocol)
         return serializer.serialize_error_to_response(
             error, operation, context.request.headers, context.request_id
         )
@@ -255,7 +273,9 @@ class ServiceResponseParser(Handler):
             return
 
         # in this case we need to parse the raw response
-        parsed = parse_response(context.operation, response, include_response_metadata=False)
+        parsed = parse_response(
+            context.operation, context.protocol, response, include_response_metadata=False
+        )
         if service_exception := parse_service_exception(response, parsed):
             context.service_exception = service_exception
         else:
@@ -278,7 +298,7 @@ class ServiceResponseHandlers(Handler):
     are only called if the request context has a service, and there are handlers for that particular service.
     """
 
-    handlers: Dict[str, CompositeResponseHandler]
+    handlers: dict[str, CompositeResponseHandler]
 
     def __init__(self):
         self.handlers = defaultdict(CompositeResponseHandler)