localstack-core 4.6.1.dev60__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/aws/connect.py

@@ -10,10 +10,11 @@ import logging
 import re
 import threading
 from abc import ABC, abstractmethod
+from collections.abc import Callable
 from functools import lru_cache, partial
 from random import choice
 from socket import socket
-from typing import Any, Callable, Generic, Optional, TypedDict, TypeVar
+from typing import Any, Generic, TypedDict, TypeVar
 
 import dns.message
 import dns.query
@@ -88,8 +89,8 @@ def make_hash(o):
     return hash(frozenset(sorted(new_o.items())))
 
 
-def config_equality_patch(self, other: object):
-    return type(self) == type(other) and self._user_provided_options == other._user_provided_options
+def config_equality_patch(self, other: object) -> bool:
+    return type(self) is type(other) and self._user_provided_options == other._user_provided_options
 
 
 def config_hash_patch(self):
@@ -277,10 +278,10 @@ class ClientFactory(ABC):
     def __call__(
         self,
         *,
-        region_name: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_session_token: Optional[str] = None,
+        region_name: str | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
         endpoint_url: str = None,
         config: Config = None,
     ) -> ServiceLevelClientFactory:
@@ -318,11 +319,11 @@ class ClientFactory(ABC):
         self,
         *,
         role_arn: str,
-        service_principal: Optional[ServicePrincipal] = None,
-        session_name: Optional[str] = None,
-        region_name: Optional[str] = None,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        service_principal: ServicePrincipal | None = None,
+        session_name: str | None = None,
+        region_name: str | None = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> ServiceLevelClientFactory:
         """
         Create a service level client factory with credentials from assuming the given role ARN.
@@ -362,12 +363,12 @@ class ClientFactory(ABC):
     def get_client(
         self,
         service_name: str,
-        region_name: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_session_token: Optional[str] = None,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        region_name: str | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ):
         raise NotImplementedError()
 
@@ -389,11 +390,11 @@ class ClientFactory(ABC):
         service_name: str,
         region_name: str,
         use_ssl: bool,
-        verify: Optional[bool],
-        endpoint_url: Optional[str],
-        aws_access_key_id: Optional[str],
-        aws_secret_access_key: Optional[str],
-        aws_session_token: Optional[str],
+        verify: bool | None,
+        endpoint_url: str | None,
+        aws_access_key_id: str | None,
+        aws_secret_access_key: str | None,
+        aws_session_token: str | None,
         config: Config,
     ) -> BaseClient:
         """
@@ -480,12 +481,12 @@ class InternalClientFactory(ClientFactory):
     def get_client(
         self,
         service_name: str,
-        region_name: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_session_token: Optional[str] = None,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        region_name: str | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> BaseClient:
         """
         Build and return client for connections originating within LocalStack.
@@ -535,12 +536,12 @@ class ExternalClientFactory(ClientFactory):
     def get_client(
         self,
         service_name: str,
-        region_name: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_session_token: Optional[str] = None,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        region_name: str | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> BaseClient:
         """
         Build and return client for connections originating outside LocalStack and targeting Localstack.
@@ -604,12 +605,12 @@ class ExternalAwsClientFactory(ClientFactory):
     def get_client(
         self,
         service_name: str,
-        region_name: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_session_token: Optional[str] = None,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        region_name: str | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> BaseClient:
         """
         Build and return client for connections originating outside LocalStack and targeting AWS.

localstack/aws/forwarder.py

@@ -3,10 +3,13 @@ This module contains utilities to call a backend (e.g., an external service proc
 DynamoDBLocal) from a service provider.
 """
 
-from typing import Any, Callable, Mapping, Optional, Union
+from collections.abc import Callable, Mapping
+from typing import Any
 
 from botocore.awsrequest import AWSPreparedRequest, prepare_request_dict
 from botocore.config import Config as BotoConfig
+from botocore.model import OperationModel
+from botocore.serialize import create_serializer
 from werkzeug.datastructures import Headers
 
 from localstack.aws.api.core import (
@@ -18,7 +21,7 @@ from localstack.aws.api.core import (
 from localstack.aws.client import create_http_request, parse_response, raise_service_exception
 from localstack.aws.connect import connect_to
 from localstack.aws.skeleton import DispatchTable, create_dispatch_table
-from localstack.aws.spec import load_service
+from localstack.aws.spec import ProtocolName, load_service
 from localstack.constants import AWS_REGION_US_EAST_1
 from localstack.http import Response
 from localstack.http.proxy import Proxy
@@ -54,7 +57,7 @@ class AwsRequestProxy:
         self,
         context: RequestContext,
         service_request: ServiceRequest = None,
-    ) -> Optional[Union[ServiceResponse, Response]]:
+    ) -> ServiceResponse | Response | None:
         """Method to satisfy the ``ServiceRequestHandler`` protocol."""
         return self.forward(context, service_request)
 
@@ -62,7 +65,7 @@ class AwsRequestProxy:
         self,
         context: RequestContext,
         service_request: ServiceRequest = None,
-    ) -> Optional[Union[ServiceResponse, Response]]:
+    ) -> ServiceResponse | Response | None:
         """
         Forwards the given request to the backend configured by ``endpoint_url``.
 
@@ -78,7 +81,7 @@ class AwsRequestProxy:
         if not self.parse_response:
             return http_response
         parsed_response = parse_response(
-            context.operation, http_response, self.include_response_metadata
+            context.operation, context.protocol, http_response, self.include_response_metadata
         )
         raise_service_exception(http_response, parsed_response)
         return parsed_response
@@ -89,6 +92,7 @@ class AwsRequestProxy:
             action=original.operation.name,
             parameters=service_request,
             region=original.region,
+            protocol=original.protocol,
         )
         # update the newly created context with non-payload specific request headers (the payload can differ from
         # the original request, f.e. it could be JSON encoded now while the initial request was CBOR encoded)
@@ -183,7 +187,9 @@ def dispatch_to_backend(
     :raises ServiceException: if the dispatcher returned an error response
     """
     http_response = http_request_dispatcher(context)
-    parsed_response = parse_response(context.operation, http_response, include_response_metadata)
+    parsed_response = parse_response(
+        context.operation, context.protocol, http_response, include_response_metadata
+    )
     raise_service_exception(http_response, parsed_response)
     return parsed_response
 
@@ -195,9 +201,10 @@ _non_validating_boto_config = BotoConfig(parameter_validation=False)
 def create_aws_request_context(
     service_name: str,
     action: str,
+    protocol: ProtocolName = None,
     parameters: Mapping[str, Any] = None,
     region: str = None,
-    endpoint_url: Optional[str] = None,
+    endpoint_url: str | None = None,
 ) -> RequestContext:
     """
     This is a stripped-down version of what the botocore client does to perform an HTTP request from a client call. A
@@ -209,6 +216,7 @@ def create_aws_request_context(
 
     :param service_name: the AWS service
     :param action: the action to invoke
+    :param protocol: the protocol to use
     :param parameters: the invocation parameters
     :param region: the region name (default is us-east-1)
     :param endpoint_url: the endpoint to call (defaults to localstack)
@@ -221,6 +229,8 @@ def create_aws_request_context(
 
     service = load_service(service_name)
     operation = service.operation_model(action)
+    # TODO: remove this once every usage upstream has been removed
+    protocol = protocol or service.resolved_protocol
 
     # we re-use botocore internals here to serialize the HTTP request,
     # but deactivate validation (validation errors should be handled by the backend)
@@ -242,8 +252,14 @@ def create_aws_request_context(
         endpoint_url = "http://localhost.localstack.cloud"
     # pre-process the request args (some params are modified using botocore event handlers)
     parameters = client._emit_api_params(parameters, operation, request_context)
-    request_dict = client._convert_to_request_dict(
-        parameters, operation, endpoint_url, context=request_context
+
+    request_dict = _convert_to_request_dict_with_protocol(
+        client=client,
+        protocol=protocol,
+        api_params=parameters,
+        operation_model=operation,
+        endpoint_url=endpoint_url,
+        context=request_context,
     )
 
     if auth_path := request_dict.get("auth_path"):
@@ -265,7 +281,39 @@ def create_aws_request_context(
     context = RequestContext(request=create_http_request(aws_request))
     context.service = service
     context.operation = operation
+    context.protocol = protocol
     context.region = region
     context.service_request = parameters
 
     return context
+
+
+def _convert_to_request_dict_with_protocol(
+    client,
+    protocol: ProtocolName,
+    api_params: dict,
+    operation_model: OperationModel,
+    endpoint_url: str,
+    context: dict,
+    set_user_agent_header: bool = True,
+) -> dict:
+    """
+    This function is taken from botocore Client._convert_to_request_dict, but we are overriding the serializer
+    Botocore does not expose a way to create a client with a specific protocol, but we need this functionality
+    to support multi-protocols.
+    """
+    serializer = create_serializer(protocol, include_validation=False)
+    request_dict = serializer.serialize_to_request(api_params, operation_model)
+    if not client._client_config.inject_host_prefix:
+        request_dict.pop("host_prefix", None)
+    if set_user_agent_header:
+        user_agent = client._user_agent_creator.to_string()
+    else:
+        user_agent = None
+    prepare_request_dict(
+        request_dict,
+        endpoint_url=endpoint_url,
+        user_agent=user_agent,
+        context=context,
+    )
+    return request_dict

localstack/aws/gateway.py

@@ -1,5 +1,3 @@
-import typing as t
-
 from rolo.gateway import Gateway as RoloGateway
 from rolo.response import Response
 
@@ -17,7 +15,7 @@ class Gateway(RoloGateway):
         response_handlers: list[Handler] = None,
         finalizers: list[Handler] = None,
         exception_handlers: list[ExceptionHandler] = None,
-        context_class: t.Type[RequestContext] = None,
+        context_class: type[RequestContext] = None,
     ) -> None:
         super().__init__(
             request_handlers,

localstack/aws/handlers/analytics.py

@@ -1,6 +1,5 @@
 import logging
 import threading
-from typing import Optional
 
 from localstack import config
 from localstack.aws.api import RequestContext
@@ -52,7 +51,7 @@ class ServiceRequestCounter:
             )
         )
 
-    def _get_err_type(self, context: RequestContext, response: Response) -> Optional[str]:
+    def _get_err_type(self, context: RequestContext, response: Response) -> str | None:
         """
         Attempts to re-use the existing service_response, or parse and return the error type from the response body,
         e.g. ``ResourceInUseException``.
@@ -61,7 +60,7 @@ class ServiceRequestCounter:
             if context.service_exception:
                 return context.service_exception.code
 
-            response = parse_response(context.operation, response)
+            response = parse_response(context.operation, context.protocol, response)
             return response["Error"]["Code"]
         except Exception:
             if config.DEBUG_ANALYTICS:

localstack/aws/handlers/cors.py

@@ -4,7 +4,6 @@ A set of handlers which handle Cross Origin Resource Sharing (CORS).
 
 import logging
 import re
-from typing import List, Set
 from urllib.parse import urlparse
 
 from werkzeug.datastructures import Headers
@@ -83,7 +82,7 @@ ALLOWED_CORS_RESPONSE_HEADERS = [
 ]
 
 
-def _get_allowed_cors_internal_domains() -> Set[str]:
+def _get_allowed_cors_internal_domains() -> set[str]:
     """
     Construct the list of allowed internal domains for CORS enforcement purposes
     Defined as function to allow easier testing with monkeypatch of config values
@@ -94,7 +93,7 @@ def _get_allowed_cors_internal_domains() -> Set[str]:
 _ALLOWED_INTERNAL_DOMAINS = _get_allowed_cors_internal_domains()
 
 
-def _get_allowed_cors_ports() -> Set[int]:
+def _get_allowed_cors_ports() -> set[int]:
     """
     Construct the list of allowed ports for CORS enforcement purposes
     Defined as function to allow easier testing with monkeypatch of config values
@@ -105,7 +104,7 @@ def _get_allowed_cors_ports() -> Set[int]:
 _ALLOWED_INTERNAL_PORTS = _get_allowed_cors_ports()
 
 
-def _get_allowed_cors_origins() -> List[str]:
+def _get_allowed_cors_origins() -> list[str]:
     """Construct the list of allowed origins for CORS enforcement purposes"""
     result = [
         # allow access from Web app and localhost domains
@@ -210,7 +209,7 @@ class CorsEnforcer(Handler):
         return True
 
     @staticmethod
-    def _is_in_allowed_origins(allowed_origins: List[str], origin: str) -> bool:
+    def _is_in_allowed_origins(allowed_origins: list[str], origin: str) -> bool:
         """Returns true if the `origin` is in the `allowed_origins`."""
         for allowed_origin in allowed_origins:
             if allowed_origin == "*" or origin == allowed_origin:

localstack/aws/handlers/internal_requests.py

@@ -20,7 +20,12 @@ class InternalRequestParamsEnricher(Handler):
             try:
                 dto = MappingProxyType(load_dto(header))
             except Exception as e:
-                LOG.exception("Error loading request parameters '%s', Error: %s", header, e)
+                LOG.error(
+                    "Error loading request parameters '%s', Error: %s",
+                    header,
+                    e,
+                    exc_info=LOG.isEnabledFor(logging.DEBUG),
+                )
                 return
 
             context.internal_request_params = dto

localstack/aws/handlers/logging.py

@@ -1,8 +1,8 @@
 """Handlers for logging."""
 
 import logging
+import types
 from functools import cached_property
-from typing import Type
 
 from localstack.aws.api import RequestContext, ServiceException
 from localstack.aws.chain import ExceptionHandler, HandlerChain
@@ -22,6 +22,14 @@ class ExceptionLogger(ExceptionHandler):
     def __init__(self, logger=None):
         self.logger = logger or LOG
 
+        try:
+            import moto.core.exceptions
+
+            self._moto_service_exception = moto.core.exceptions.ServiceException
+        except (ModuleNotFoundError, AttributeError):
+            # Moto may not be available in stripped-down versions of LocalStack, like LocalStack S3 image.
+            self._moto_service_exception = types.EllipsisType
+
     def __call__(
         self,
         chain: HandlerChain,
@@ -29,9 +37,10 @@ class ExceptionLogger(ExceptionHandler):
         context: RequestContext,
         response: Response,
     ):
-        if isinstance(exception, ServiceException):
+        if isinstance(exception, (ServiceException, self._moto_service_exception)):
             # We do not want to log an error/stacktrace if the handler is working as expected, but chooses to throw
-            # a service exception
+            # a service exception. It may also throw a Moto ServiceException, which should not be logged either
+            # because ServiceExceptionHandler understands it.
             return
         if self.logger.isEnabledFor(level=logging.DEBUG):
             self.logger.exception("exception during call chain", exc_info=exception)
@@ -71,7 +80,7 @@ class ResponseLogger:
         )
 
     # make sure loggers are loaded after logging config is loaded
-    def _prepare_logger(self, logger: logging.Logger, formatter: Type):
+    def _prepare_logger(self, logger: logging.Logger, formatter: type):
         if logger.isEnabledFor(logging.DEBUG):
             logger.propagate = False
             handler = create_default_handler(logger.level)

localstack/aws/handlers/metric_handler.py

@@ -1,10 +1,15 @@
+import csv
 import logging
-from typing import List, Optional
+import os
+from datetime import datetime
+from pathlib import Path
 
 from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.chain import HandlerChain
+from localstack.constants import ENV_INTERNAL_TEST_STORE_METRICS_PATH
 from localstack.http import Response
+from localstack.utils.strings import short_uid
 
 LOG = logging.getLogger(__name__)
 
@@ -16,7 +21,7 @@ class MetricHandlerItem:
 
     request_id: str
     request_context: RequestContext
-    parameters_after_parse: Optional[List[str]]
+    parameters_after_parse: list[str] | None
 
     def __init__(self, request_contex: RequestContext) -> None:
         super().__init__()
@@ -35,7 +40,7 @@ class Metric:
     headers: str
     parameters: str
     status_code: int
-    response_code: Optional[str]
+    response_code: str | None
     exception: str
     origin: str
     xfail: bool
@@ -134,10 +139,36 @@ class Metric:
 
 
 class MetricHandler:
-    metric_data: List[Metric] = []
+    metric_data: list[Metric] = []
 
     def __init__(self) -> None:
         self.metrics_handler_items = {}
+        self.local_filename = None
+
+        if self.should_store_metric_locally():
+            self.local_filename = self.create_local_file()
+
+    @staticmethod
+    def should_store_metric_locally() -> bool:
+        return config.is_collect_metrics_mode() and config.store_test_metrics_in_local_filesystem()
+
+    @staticmethod
+    def create_local_file():
+        folder = Path(
+            os.environ.get(ENV_INTERNAL_TEST_STORE_METRICS_PATH, "/tmp/localstack-metrics")
+        )
+        if not folder.exists():
+            folder.mkdir(parents=True, exist_ok=True)
+        LOG.debug("Metric reports will be stored in %s", folder)
+        filename = (
+            folder
+            / f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv"
+        )
+        with open(filename, "w") as fd:
+            LOG.debug("Creating new metric data file %s", filename)
+            writer = csv.writer(fd)
+            writer.writerow(Metric.RAW_DATA_HEADER)
+        return filename
 
     def create_metric_handler_item(
         self, chain: HandlerChain, context: RequestContext, response: Response
@@ -195,7 +226,15 @@ class MetricHandler:
         )
         # refrain from adding duplicates
         if metric not in MetricHandler.metric_data:
-            MetricHandler.metric_data.append(metric)
+            self.append_metric(metric)
 
         # cleanup
         del self.metrics_handler_items[context]
+
+    def append_metric(self, metric: Metric):
+        if self.should_store_metric_locally():
+            with open(self.local_filename, "a") as fd:
+                writer = csv.writer(fd)
+                writer.writerow(metric)
+        else:
+            MetricHandler.metric_data.append(metric)