localstack-core 4.7.1.dev139__py3-none-any.whl → 4.10.1.dev42__py3-none-any.whl

localstack/aws/client.py

@@ -21,6 +21,7 @@ from localstack.utils.strings import to_str
 from .api import CommonServiceException, RequestContext, ServiceException, ServiceResponse
 from .connect import get_service_endpoint
 from .gateway import Gateway
+from .spec import ProtocolName
 
 LOG = logging.getLogger(__name__)
 
@@ -284,13 +285,17 @@ def _patch_botocore_endpoint_in_memory():
 
 
 def parse_response(
-    operation: OperationModel, response: Response, include_response_metadata: bool = True
+    operation: OperationModel,
+    protocol: ProtocolName,
+    response: Response,
+    include_response_metadata: bool = True,
 ) -> ServiceResponse:
     """
     Parses an HTTP Response object into an AWS response object using botocore. It does this by adapting the
     procedure of ``botocore.endpoint.convert_to_response_dict`` to work with Werkzeug's server-side response object.
 
     :param operation: the operation of the original request
+    :param protocol: the protocol of the original request
     :param response: the HTTP response object containing the response of the operation
     :param include_response_metadata: True if the ResponseMetadata (typical for boto response dicts) should be included
     :return: a parsed dictionary as it is returned by botocore
@@ -322,7 +327,7 @@ def parse_response(
             timestamp_parser=_cbor_timestamp_parser, blob_parser=_cbor_blob_parser
         )
 
-    parser = factory.create_parser(operation.service_model.protocol)
+    parser = factory.create_parser(protocol)
     parsed_response = parser.parse(response_dict, operation.output_shape)
 
     if response.status_code >= 301:

localstack/aws/forwarder.py

@@ -8,6 +8,8 @@ from typing import Any
 
 from botocore.awsrequest import AWSPreparedRequest, prepare_request_dict
 from botocore.config import Config as BotoConfig
+from botocore.model import OperationModel
+from botocore.serialize import create_serializer
 from werkzeug.datastructures import Headers
 
 from localstack.aws.api.core import (
@@ -19,7 +21,7 @@ from localstack.aws.api.core import (
 from localstack.aws.client import create_http_request, parse_response, raise_service_exception
 from localstack.aws.connect import connect_to
 from localstack.aws.skeleton import DispatchTable, create_dispatch_table
-from localstack.aws.spec import load_service
+from localstack.aws.spec import ProtocolName, load_service
 from localstack.constants import AWS_REGION_US_EAST_1
 from localstack.http import Response
 from localstack.http.proxy import Proxy
@@ -79,7 +81,7 @@ class AwsRequestProxy:
         if not self.parse_response:
             return http_response
         parsed_response = parse_response(
-            context.operation, http_response, self.include_response_metadata
+            context.operation, context.protocol, http_response, self.include_response_metadata
         )
         raise_service_exception(http_response, parsed_response)
         return parsed_response
@@ -90,6 +92,7 @@ class AwsRequestProxy:
             action=original.operation.name,
             parameters=service_request,
             region=original.region,
+            protocol=original.protocol,
         )
         # update the newly created context with non-payload specific request headers (the payload can differ from
         # the original request, f.e. it could be JSON encoded now while the initial request was CBOR encoded)
@@ -184,7 +187,9 @@ def dispatch_to_backend(
     :raises ServiceException: if the dispatcher returned an error response
     """
     http_response = http_request_dispatcher(context)
-    parsed_response = parse_response(context.operation, http_response, include_response_metadata)
+    parsed_response = parse_response(
+        context.operation, context.protocol, http_response, include_response_metadata
+    )
     raise_service_exception(http_response, parsed_response)
     return parsed_response
 
@@ -196,6 +201,7 @@ _non_validating_boto_config = BotoConfig(parameter_validation=False)
 def create_aws_request_context(
     service_name: str,
     action: str,
+    protocol: ProtocolName = None,
     parameters: Mapping[str, Any] = None,
     region: str = None,
     endpoint_url: str | None = None,
@@ -210,6 +216,7 @@ def create_aws_request_context(
 
     :param service_name: the AWS service
     :param action: the action to invoke
+    :param protocol: the protocol to use
     :param parameters: the invocation parameters
     :param region: the region name (default is us-east-1)
     :param endpoint_url: the endpoint to call (defaults to localstack)
@@ -222,6 +229,8 @@ def create_aws_request_context(
 
     service = load_service(service_name)
     operation = service.operation_model(action)
+    # TODO: remove this once every usage upstream has been removed
+    protocol = protocol or service.resolved_protocol
 
     # we re-use botocore internals here to serialize the HTTP request,
     # but deactivate validation (validation errors should be handled by the backend)
@@ -243,8 +252,14 @@ def create_aws_request_context(
         endpoint_url = "http://localhost.localstack.cloud"
     # pre-process the request args (some params are modified using botocore event handlers)
     parameters = client._emit_api_params(parameters, operation, request_context)
-    request_dict = client._convert_to_request_dict(
-        parameters, operation, endpoint_url, context=request_context
+
+    request_dict = _convert_to_request_dict_with_protocol(
+        client=client,
+        protocol=protocol,
+        api_params=parameters,
+        operation_model=operation,
+        endpoint_url=endpoint_url,
+        context=request_context,
     )
 
     if auth_path := request_dict.get("auth_path"):
@@ -266,7 +281,39 @@ def create_aws_request_context(
     context = RequestContext(request=create_http_request(aws_request))
     context.service = service
     context.operation = operation
+    context.protocol = protocol
     context.region = region
     context.service_request = parameters
 
     return context
+
+
+def _convert_to_request_dict_with_protocol(
+    client,
+    protocol: ProtocolName,
+    api_params: dict,
+    operation_model: OperationModel,
+    endpoint_url: str,
+    context: dict,
+    set_user_agent_header: bool = True,
+) -> dict:
+    """
+    This function is taken from botocore Client._convert_to_request_dict, but we are overriding the serializer
+    Botocore does not expose a way to create a client with a specific protocol, but we need this functionality
+    to support multi-protocols.
+    """
+    serializer = create_serializer(protocol, include_validation=False)
+    request_dict = serializer.serialize_to_request(api_params, operation_model)
+    if not client._client_config.inject_host_prefix:
+        request_dict.pop("host_prefix", None)
+    if set_user_agent_header:
+        user_agent = client._user_agent_creator.to_string()
+    else:
+        user_agent = None
+    prepare_request_dict(
+        request_dict,
+        endpoint_url=endpoint_url,
+        user_agent=user_agent,
+        context=context,
+    )
+    return request_dict

localstack/aws/handlers/analytics.py

@@ -60,7 +60,7 @@ class ServiceRequestCounter:
             if context.service_exception:
                 return context.service_exception.code
 
-            response = parse_response(context.operation, response)
+            response = parse_response(context.operation, context.protocol, response)
             return response["Error"]["Code"]
         except Exception:
             if config.DEBUG_ANALYTICS:

localstack/aws/handlers/logging.py

@@ -1,6 +1,7 @@
 """Handlers for logging."""
 
 import logging
+import types
 from functools import cached_property
 
 from localstack.aws.api import RequestContext, ServiceException
@@ -21,6 +22,14 @@ class ExceptionLogger(ExceptionHandler):
     def __init__(self, logger=None):
         self.logger = logger or LOG
 
+        try:
+            import moto.core.exceptions
+
+            self._moto_service_exception = moto.core.exceptions.ServiceException
+        except (ModuleNotFoundError, AttributeError):
+            # Moto may not be available in stripped-down versions of LocalStack, like LocalStack S3 image.
+            self._moto_service_exception = types.EllipsisType
+
     def __call__(
         self,
         chain: HandlerChain,
@@ -28,9 +37,10 @@ class ExceptionLogger(ExceptionHandler):
         context: RequestContext,
         response: Response,
     ):
-        if isinstance(exception, ServiceException):
+        if isinstance(exception, (ServiceException, self._moto_service_exception)):
             # We do not want to log an error/stacktrace if the handler is working as expected, but chooses to throw
-            # a service exception
+            # a service exception. It may also throw a Moto ServiceException, which should not be logged either
+            # because ServiceExceptionHandler understands it.
             return
         if self.logger.isEnabledFor(level=logging.DEBUG):
             self.logger.exception("exception during call chain", exc_info=exception)

localstack/aws/handlers/metric_handler.py

@@ -1,9 +1,15 @@
+import csv
 import logging
+import os
+from datetime import datetime
+from pathlib import Path
 
 from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.chain import HandlerChain
+from localstack.constants import ENV_INTERNAL_TEST_STORE_METRICS_PATH
 from localstack.http import Response
+from localstack.utils.strings import short_uid
 
 LOG = logging.getLogger(__name__)
 
@@ -137,6 +143,32 @@ class MetricHandler:
 
     def __init__(self) -> None:
         self.metrics_handler_items = {}
+        self.local_filename = None
+
+        if self.should_store_metric_locally():
+            self.local_filename = self.create_local_file()
+
+    @staticmethod
+    def should_store_metric_locally() -> bool:
+        return config.is_collect_metrics_mode() and config.store_test_metrics_in_local_filesystem()
+
+    @staticmethod
+    def create_local_file():
+        folder = Path(
+            os.environ.get(ENV_INTERNAL_TEST_STORE_METRICS_PATH, "/tmp/localstack-metrics")
+        )
+        if not folder.exists():
+            folder.mkdir(parents=True, exist_ok=True)
+        LOG.debug("Metric reports will be stored in %s", folder)
+        filename = (
+            folder
+            / f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv"
+        )
+        with open(filename, "w") as fd:
+            LOG.debug("Creating new metric data file %s", filename)
+            writer = csv.writer(fd)
+            writer.writerow(Metric.RAW_DATA_HEADER)
+        return filename
 
     def create_metric_handler_item(
         self, chain: HandlerChain, context: RequestContext, response: Response
@@ -194,7 +226,15 @@ class MetricHandler:
         )
         # refrain from adding duplicates
         if metric not in MetricHandler.metric_data:
-            MetricHandler.metric_data.append(metric)
+            self.append_metric(metric)
 
         # cleanup
         del self.metrics_handler_items[context]
+
+    def append_metric(self, metric: Metric):
+        if self.should_store_metric_locally():
+            with open(self.local_filename, "a") as fd:
+                writer = csv.writer(fd)
+                writer.writerow(metric)
+        else:
+            MetricHandler.metric_data.append(metric)

localstack/aws/handlers/service.py

@@ -2,22 +2,24 @@
 
 import logging
 import traceback
+import types
 from collections import defaultdict
 from typing import Any
 
 from botocore.model import OperationModel, ServiceModel
+from plux.core.plugin import PluginDisabled
 
 from localstack import config
 from localstack.http import Response
-from localstack.utils.coverage_docs import get_coverage_link_for_service
 
+from ...utils.coverage_docs import get_coverage_link_for_service
 from ..api import CommonServiceException, RequestContext, ServiceException
 from ..api.core import ServiceOperation
 from ..chain import CompositeResponseHandler, ExceptionHandler, Handler, HandlerChain
 from ..client import parse_response, parse_service_exception
 from ..protocol.parser import RequestParser, create_parser
 from ..protocol.serializer import create_serializer
-from ..protocol.service_router import determine_aws_service_model
+from ..protocol.service_router import determine_aws_protocol, determine_aws_service_model
 from ..skeleton import Skeleton, create_skeleton
 
 LOG = logging.getLogger(__name__)
@@ -33,6 +35,8 @@ class ServiceNameParser(Handler):
         # example). If it is already set, we can skip the parsing of the request. It is very important for S3, because
         # parsing the request will consume the data stream and prevent streaming.
         if context.service:
+            if not context.protocol:
+                context.protocol = determine_aws_protocol(context.request, context.service)
             return
 
         service_model = determine_aws_service_model(context.request)
@@ -41,6 +45,7 @@ class ServiceNameParser(Handler):
             return
 
         context.service = service_model
+        context.protocol = determine_aws_protocol(context.request, service_model)
 
 
 class ServiceRequestParser(Handler):
@@ -63,7 +68,7 @@ class ServiceRequestParser(Handler):
         return self.parse_and_enrich(context)
 
     def parse_and_enrich(self, context: RequestContext):
-        parser = create_parser(context.service)
+        parser = create_parser(context.service, context.protocol)
         operation, instance = parser.parse(context.request)
 
         # enrich context
@@ -137,7 +142,7 @@ class ServiceRequestRouter(Handler):
         operation_name = operation.name
         message = f"no handler for operation '{operation_name}' on service '{service_name}'"
         error = CommonServiceException("InternalFailure", message, status_code=501)
-        serializer = create_serializer(context.service)
+        serializer = create_serializer(context.service, context.protocol)
         return serializer.serialize_error_to_response(
             error, operation, context.request.headers, context.request_id
         )
@@ -153,6 +158,15 @@ class ServiceExceptionSerializer(ExceptionHandler):
     def __init__(self):
         self.handle_internal_failures = True
 
+        try:
+            import moto.core.exceptions
+
+            self._moto_service_exception = moto.core.exceptions.ServiceException
+        except (ModuleNotFoundError, AttributeError) as exc:
+            # Moto may not be available in stripped-down versions of LocalStack, like LocalStack S3 image.
+            LOG.debug("Unable to set up Moto ServiceException translation: %s", exc)
+            self._moto_service_exception = types.EllipsisType
+
     def __call__(
         self,
         chain: HandlerChain,
@@ -176,9 +190,16 @@ class ServiceExceptionSerializer(ExceptionHandler):
             action_name = operation.name
             exception_message: str | None = exception.args[0] if exception.args else None
             message = exception_message or get_coverage_link_for_service(service_name, action_name)
-            LOG.info(message)
             error = CommonServiceException("InternalFailure", message, status_code=501)
-            context.service_exception = error
+            LOG.info(message)
+
+        elif isinstance(exception, self._moto_service_exception):
+            # Translate Moto ServiceException to native ServiceException if Moto is available.
+            # This allows handler chain to gracefully handles Moto errors when provider handlers invoke Moto methods directly.
+            error = CommonServiceException(
+                code=exception.code,
+                message=exception.message,
+            )
 
         elif not isinstance(exception, ServiceException):
             if not self.handle_internal_failures:
@@ -202,14 +223,24 @@ class ServiceExceptionSerializer(ExceptionHandler):
                 operation = context.service.operation_model(context.service.operation_names[0])
                 msg = f"exception while calling {service_name} with unknown operation: {message}"
 
-            status_code = 501 if config.FAIL_FAST else 500
+            # Check for license restricted plugin message and set status code to 501
+            if (
+                isinstance(exception, PluginDisabled)
+                and "not part of the active license agreement"
+                in str(getattr(exception, "reason", "")).lower()
+            ):
+                status_code = 501
+                msg = f"exception while calling {service_name}.{operation.name}: {str(getattr(exception, 'reason', ''))}"
+            else:
+                status_code = 501 if config.FAIL_FAST else 500
 
             error = CommonServiceException(
                 "InternalError", msg, status_code=status_code
             ).with_traceback(exception.__traceback__)
-            context.service_exception = error
 
-        serializer = create_serializer(context.service)  # TODO: serializer cache
+        context.service_exception = error
+
+        serializer = create_serializer(context.service, context.protocol)
         return serializer.serialize_error_to_response(
             error, operation, context.request.headers, context.request_id
         )
@@ -252,7 +283,9 @@ class ServiceResponseParser(Handler):
             return
 
         # in this case we need to parse the raw response
-        parsed = parse_response(context.operation, response, include_response_metadata=False)
+        parsed = parse_response(
+            context.operation, context.protocol, response, include_response_metadata=False
+        )
         if service_exception := parse_service_exception(response, parsed):
             context.service_exception = service_exception
         else: