localstack-core 4.7.1.dev139__py3-none-any.whl → 4.10.1.dev7__py3-none-any.whl

localstack/aws/protocol/service_router.py

@@ -6,9 +6,11 @@ from werkzeug.exceptions import RequestEntityTooLarge
 from werkzeug.http import parse_dict_header
 
 from localstack.aws.spec import (
+    ProtocolName,
     ServiceCatalog,
     ServiceModelIdentifier,
     get_service_catalog,
+    is_protocol_in_service_model_identifier,
 )
 from localstack.http import Request
 from localstack.services.s3.utils import uses_host_addressing
@@ -17,6 +19,23 @@ from localstack.utils.strings import to_bytes
 
 LOG = logging.getLogger(__name__)
 
+_PROTOCOL_DETECTION_PRIORITY: list[ProtocolName] = [
+    "smithy-rpc-v2-cbor",
+    "json",
+    "query",
+    "ec2",
+    "rest-json",
+    "rest-xml",
+]
+
+
+class ProtocolError(Exception):
+    """
+    Error which is thrown if we cannot detect the protocol for the request.
+    """
+
+    pass
+
 
 class _ServiceIndicators(NamedTuple):
     """
@@ -43,6 +62,7 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
     """Extracts all different fields that might indicate which service a request is targeting."""
     x_amz_target = request.headers.get("x-amz-target")
     authorization = request.headers.get("authorization")
+    is_rpc_v2 = "rpc-v2-cbor" in request.headers.get("Smithy-Protocol", "")
 
     signing_name = None
     if authorization:
@@ -55,7 +75,15 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
         except (ValueError, KeyError):
             LOG.debug("auth header could not be parsed for service routing: %s", authorization)
             pass
-    if x_amz_target:
+    if is_rpc_v2:
+        # https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html#requests
+        rpc_v2_params = request.path.lstrip("/").split("/")
+        if len(rpc_v2_params) >= 4:
+            *_, service_shape_name, __, operation = rpc_v2_params
+            target_prefix = service_shape_name.split("#")[-1]
+        else:
+            target_prefix, operation = None, None
+    elif x_amz_target:
         if "." in x_amz_target:
             target_prefix, operation = x_amz_target.split(".", 1)
         else:
@@ -67,6 +95,48 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
     return _ServiceIndicators(signing_name, target_prefix, operation, request.host, request.path)
 
 
+def _matches_protocol(request: Request, protocol: ProtocolName) -> bool:
+    headers = request.headers
+    mimetype = request.mimetype.lower()
+    match protocol:
+        case "smithy-rpc-v2-cbor":
+            # Every request for the rpcv2Cbor protocol MUST contain a `Smithy-Protocol` header with the value
+            # of `rpc-v2-cbor`.
+            # https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html
+            return headers.get("Smithy-Protocol", "") == "rpc-v2-cbor"
+        case "json":
+            return mimetype.startswith("application/x-amz-json")
+        case "query" | "ec2":
+            # https://smithy.io/2.0/aws/protocols/aws-query-protocol.html#request-serialization
+            return (
+                mimetype.startswith("application/x-www-form-urlencoded") or "Action" in request.args
+            )
+        case "rest-xml" | "rest-json":
+            # `rest-json` and `rest-xml` can accept any kind of Content-Type, and it can be configured on the operation
+            # level.
+            # https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html
+            return True
+        case _:
+            return False
+
+
+def match_available_protocols(
+    request: Request, available_protocols: list[ProtocolName]
+) -> ProtocolName | None:
+    """
+    Tries to match the current request and determine the protocol used amongst the available protocols given.
+    We use a priority order to try to determine the protocol, as some protocols are more permissive that others.
+    :param request: the incoming request
+    :param available_protocols: the available protocols of the Service the request is directed to
+    :return: the protocol matched, if any
+    """
+    for protocol in _PROTOCOL_DETECTION_PRIORITY:
+        if protocol in available_protocols and _matches_protocol(request, protocol):
+            return protocol
+
+    return None
+
+
 signing_name_path_prefix_rules = {
     # custom rules based on URI path prefixes that are not easily generalizable
     "apigateway": {
@@ -248,10 +318,10 @@ def resolve_conflicts(
         # The `application/x-amz-json-1.0` header is mandatory for requests targeting SQS with the `json` protocol. We
         # can safely route them to the `sqs` JSON parser/serializer. If not present, route the request to the
         # sqs-query protocol.
-        content_type = request.headers.get("Content-Type")
+        protocol = match_available_protocols(request, available_protocols=["json", "query"])
         return (
             ServiceModelIdentifier("sqs")
-            if content_type == "application/x-amz-json-1.0"
+            if protocol == "json"
             else ServiceModelIdentifier("sqs", "query")
         )
 
@@ -266,7 +336,25 @@ def determine_aws_service_model_for_data_plane(
     custom_host_match = custom_host_addressing_rules(request.host)
     if custom_host_match:
         services = services or get_service_catalog()
-        return services.get(*custom_host_match)
+        return services.get(custom_host_match.name, custom_host_match.protocol)
+
+
+def determine_aws_protocol(request: Request, service_model: ServiceModel) -> ProtocolName:
+    if not (protocols := service_model.metadata.get("protocols")):
+        # if the service does not define multiple protocols, return the `protocol` defined for the service
+        return service_model.protocol
+
+    if len(protocols) == 1:
+        return protocols[0]
+
+    if protocol := match_available_protocols(request, available_protocols=protocols):
+        return protocol
+
+    raise ProtocolError(
+        f"Could not determine the protocol for the request: "
+        f"{request.method} {request.path} for the service '{service_model.service_name}' "
+        f"(available protocols: {protocols})"
+    )
 
 
 def determine_aws_service_model(
@@ -287,12 +375,13 @@ def determine_aws_service_model(
         signing_name_candidates = services.by_signing_name(signing_name)
         if len(signing_name_candidates) == 1:
             # a unique signing-name -> service name mapping is the case for ~75% of service operations
-            return services.get(*signing_name_candidates[0])
+            candidate = signing_name_candidates[0]
+            return services.get(candidate.name, candidate.protocol)
 
         # try to find a match with the custom signing name rules
         custom_match = custom_signing_name_rules(signing_name, path)
         if custom_match:
-            return services.get(*custom_match)
+            return services.get(custom_match.name, custom_match.protocol)
 
         # still ambiguous - add the services to the list of candidates
         candidates.update(signing_name_candidates)
@@ -302,33 +391,34 @@ def determine_aws_service_model(
         target_candidates = services.by_target_prefix(target_prefix)
         if len(target_candidates) == 1:
             # a unique target prefix
-            return services.get(*target_candidates[0])
+            candidate = target_candidates[0]
+            return services.get(candidate.name, candidate.protocol)
 
         # still ambiguous - add the services to the list of candidates
         candidates.update(target_candidates)
 
         # exclude services where the operation is not contained in the service spec
         for service_identifier in list(candidates):
-            service = services.get(*service_identifier)
+            service = services.get(service_identifier.name, service_identifier.protocol)
             if operation not in service.operation_names:
                 candidates.remove(service_identifier)
     else:
         # exclude services which have a target prefix (the current request does not have one)
         for service_identifier in list(candidates):
-            service = services.get(*service_identifier)
+            service = services.get(service_identifier.name, service_identifier.protocol)
             if service.metadata.get("targetPrefix") is not None:
                 candidates.remove(service_identifier)
 
     if len(candidates) == 1:
         service_identifier = candidates.pop()
-        return services.get(*service_identifier)
+        return services.get(service_identifier.name, service_identifier.protocol)
 
     # 3. check the path if it is set and not a trivial root path
     if path and path != "/":
         # try to find a match with the custom path rules
         custom_path_match = custom_path_addressing_rules(path)
         if custom_path_match:
-            return services.get(*custom_path_match)
+            return services.get(custom_path_match.name, custom_path_match.protocol)
 
     # 4. check the host (custom host addressing rules)
     if host:
@@ -337,12 +427,14 @@ def determine_aws_service_model(
             # this prevents a virtual host addressed bucket to be wrongly recognized
             if host.startswith(f"{prefix}.") and ".s3." not in host:
                 if len(services_per_prefix) == 1:
-                    return services.get(*services_per_prefix[0])
+                    candidate = services_per_prefix[0]
+                    return services.get(candidate.name, candidate.protocol)
                 candidates.update(services_per_prefix)
 
         custom_host_match = custom_host_addressing_rules(host)
         if custom_host_match:
-            return services.get(*custom_host_match)
+            candidate = custom_host_match[0]
+            return services.get(candidate.name, candidate.protocol)
 
     if request.shallow:
         # from here on we would need access to the request body, which doesn't exist for shallow requests like
@@ -357,21 +449,28 @@ def determine_aws_service_model(
             query_candidates = [
                 service
                 for service in services.by_operation(values["Action"])
-                if service.protocol in ("ec2", "query")
+                if any(
+                    is_protocol_in_service_model_identifier(protocol, service)
+                    for protocol in ("ec2", "query")
+                )
             ]
 
             if len(query_candidates) == 1:
-                return services.get(*query_candidates[0])
+                candidate = query_candidates[0]
+                return services.get(candidate.name, candidate.protocol)
 
             if "Version" in values:
                 for service_identifier in list(query_candidates):
-                    service_model = services.get(*service_identifier)
+                    service_model = services.get(
+                        service_identifier.name, service_identifier.protocol
+                    )
                     if values["Version"] != service_model.api_version:
                         # the combination of Version and Action is not unique, add matches to the candidates
                         query_candidates.remove(service_identifier)
 
             if len(query_candidates) == 1:
-                return services.get(*query_candidates[0])
+                candidate = query_candidates[0]
+                return services.get(candidate.name, candidate.protocol)
 
             candidates.update(query_candidates)
 
@@ -387,15 +486,16 @@ def determine_aws_service_model(
     # 6. resolve service spec conflicts
     resolved_conflict = resolve_conflicts(candidates, request)
     if resolved_conflict:
-        return services.get(*resolved_conflict)
+        return services.get(resolved_conflict.name, resolved_conflict.protocol)
 
     # 7. check the legacy S3 rules in the end
     legacy_match = legacy_s3_rules(request)
     if legacy_match:
-        return services.get(*legacy_match)
+        return services.get(legacy_match.name, legacy_match.protocol)
 
     if signing_name:
         return services.get(name=signing_name)
     if candidates:
-        return services.get(*candidates.pop())
+        candidate = candidates.pop()
+        return services.get(candidate.name, candidate.protocol)
     return None

localstack/aws/skeleton.py

@@ -130,14 +130,16 @@ class Skeleton:
             self.dispatch_table = create_dispatch_table(implementation)
 
     def invoke(self, context: RequestContext) -> Response:
-        serializer = create_serializer(context.service)
+        serializer = create_serializer(context.service, context.protocol)
 
         if context.operation and context.service_request:
             # if the parsed request is already set in the context, re-use them
             operation, instance = context.operation, context.service_request
         else:
             # otherwise, parse the incoming HTTPRequest
-            operation, instance = create_parser(context.service).parse(context.request)
+            operation, instance = create_parser(context.service, context.protocol).parse(
+                context.request
+            )
             context.operation = operation
 
         try:

localstack/aws/spec-patches.json

@@ -1329,6 +1329,26 @@
         "documentation": "<p>The Content-MD5 you specified did not match what we received.</p>",
         "exception": true
       }
+    },
+    {
+      "op": "add",
+      "path": "/shapes/AuthorizationHeaderMalformed",
+      "value": {
+        "type": "structure",
+        "members": {
+          "Region": {
+            "shape": "BucketRegion"
+          },
+          "HostId": {
+            "shape": "HostId"
+          }
+        },
+        "error": {
+          "httpStatusCode": 400
+        },
+        "documentation": "<p>The authorization header is malformed.</p>",
+        "exception": true
+      }
     }
   ],
   "apigatewayv2/2018-11-29/service-2": [
@@ -1352,5 +1372,43 @@
       "path": "/operations/CreateApiMapping/http/responseCode",
       "value": 200
     }
+  ],
+  "cloudwatch/2010-08-01/service-2": [
+    {
+      "op": "add",
+      "path": "/metadata/awsQueryCompatible",
+      "value": {}
+    },
+    {
+      "op": "add",
+      "path": "/metadata/jsonVersion",
+      "value": "1.0"
+    },
+    {
+      "op": "add",
+      "path": "/metadata/targetPrefix",
+      "value": "GraniteServiceVersion20100801"
+    },
+    {
+      "op": "replace",
+      "path": "/metadata/protocol",
+      "value": "smithy-rpc-v2-cbor"
+    },
+    {
+      "op": "replace",
+      "path": "/metadata/protocols",
+      "value": [
+        "smithy-rpc-v2-cbor",
+        "json",
+        "query"
+      ]
+    },
+    {
+      "op": "add",
+      "path": "/shapes/ConflictException/error",
+      "value": {
+        "httpStatusCode": 409
+      }
+    }
   ]
 }

localstack/aws/spec.py

@@ -21,7 +21,7 @@ from localstack.utils.objects import singleton_factory
 LOG = logging.getLogger(__name__)
 
 ServiceName = str
-ProtocolName = Literal["query", "json", "rest-json", "rest-xml", "ec2"]
+ProtocolName = Literal["query", "json", "rest-json", "rest-xml", "ec2", "smithy-rpc-v2-cbor"]
 
 
 class ServiceModelIdentifier(NamedTuple):
@@ -33,6 +33,7 @@ class ServiceModelIdentifier(NamedTuple):
 
     name: ServiceName
     protocol: ProtocolName | None = None
+    protocols: tuple[ProtocolName] | None = None
 
 
 spec_patches_json = os.path.join(os.path.dirname(__file__), "spec-patches.json")
@@ -114,9 +115,13 @@ def load_service(
     :raises: UnknownServiceProtocolError if the specific protocol of the service cannot be found
     """
     service_description = loader.load_service_model(service, "service-2", version)
+    service_metadata = service_description.get("metadata", {})
+    service_protocols = {service_metadata.get("protocol")}
+    if protocols := service_metadata.get("protocols"):
+        service_protocols.update(protocols)
 
     # check if the protocol is defined, and if so, if the loaded service defines this protocol
-    if protocol is not None and protocol != service_description.get("metadata", {}).get("protocol"):
+    if protocol is not None and protocol not in service_protocols:
         # if the protocol is defined, but not the one of the currently loaded service,
         # check if we already loaded the custom spec based on the naming convention (<service>-<protocol>),
         # f.e. "sqs-query"
@@ -132,7 +137,7 @@ def load_service(
 
     # remove potential protocol names from the service name
     # FIXME add more protocols here if we have to internalize more than just sqs-query
-    # TODO this should not contain specific internalized serivce names
+    # TODO this should not contain specific internalized service names
     service = {"sqs-query": "sqs"}.get(service, service)
     return ServiceModel(service_description, service)
 
@@ -149,6 +154,27 @@ def iterate_service_operations() -> Generator[tuple[ServiceModel, OperationModel
             yield service, service.operation_model(op_name)
 
 
+def is_protocol_in_service_model_identifier(
+    protocol: ProtocolName, service_model_identifier: ServiceModelIdentifier
+) -> bool:
+    """
+    :param protocol: the protocol name to check
+    :param service_model_identifier:
+    :return: boolean to indicate if the protocol is available for that service
+    """
+    protocols = service_model_identifier.protocols or []
+    return protocol in protocols or protocol == service_model_identifier.protocol
+
+
+def get_service_model_identifier(service_model: ServiceModel) -> ServiceModelIdentifier:
+    protocols = service_model.metadata.get("protocols")
+    return ServiceModelIdentifier(
+        name=service_model.service_name,
+        protocol=service_model.protocol,
+        protocols=tuple(protocols) if protocols else None,
+    )
+
+
 @dataclasses.dataclass
 class ServiceCatalogIndex:
     """
@@ -178,9 +204,7 @@ class LazyServiceCatalogIndex:
             for service_model in service_models:
                 target_prefix = service_model.metadata.get("targetPrefix")
                 if target_prefix:
-                    result[target_prefix].append(
-                        ServiceModelIdentifier(service_model.service_name, service_model.protocol)
-                    )
+                    result[target_prefix].append(get_service_model_identifier(service_model))
         return dict(result)
 
     @cached_property
@@ -189,7 +213,7 @@ class LazyServiceCatalogIndex:
         for service_models in self._services.values():
             for service_model in service_models:
                 result[service_model.signing_name].append(
-                    ServiceModelIdentifier(service_model.service_name, service_model.protocol)
+                    get_service_model_identifier(service_model)
                 )
         return dict(result)
 
@@ -201,11 +225,7 @@ class LazyServiceCatalogIndex:
                 operations = service_model.operation_names
                 if operations:
                     for operation in operations:
-                        result[operation].append(
-                            ServiceModelIdentifier(
-                                service_model.service_name, service_model.protocol
-                            )
-                        )
+                        result[operation].append(get_service_model_identifier(service_model))
         return dict(result)
 
     @cached_property
@@ -214,7 +234,7 @@ class LazyServiceCatalogIndex:
         for service_models in self._services.values():
             for service_model in service_models:
                 result[service_model.endpoint_prefix].append(
-                    ServiceModelIdentifier(service_model.service_name, service_model.protocol)
+                    get_service_model_identifier(service_model)
                 )
         return dict(result)
 

localstack/cli/exceptions.py

@@ -12,7 +12,7 @@ class CLIError(ClickException):
     def format_message(self) -> str:
         return click.style(f"❌ Error: {self.message}", fg="red")
 
-    def show(self, file: t.Optional[t.IO[t.Any]] = None) -> None:
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
         if file is None:
             file = get_text_stderr()
 

localstack/cli/localstack.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 import traceback
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 import click
 import requests
@@ -320,8 +320,8 @@ class DockerStatus(TypedDict, total=False):
     image_tag: str
     image_id: str
     image_created: str
-    container_name: Optional[str]
-    container_ip: Optional[str]
+    container_name: str | None
+    container_ip: str | None
 
 
 def _print_docker_status(format_: str) -> None:
@@ -699,7 +699,7 @@ def cmd_logs(follow: bool, tail: int) -> None:
     metavar="N",
 )
 @publish_invocation
-def cmd_wait(timeout: Optional[float] = None) -> None:
+def cmd_wait(timeout: float | None = None) -> None:
     """
     Wait for the LocalStack runtime to be up and running.
 

localstack/cli/lpm.py

@@ -1,7 +1,6 @@
 import itertools
 import logging
 from multiprocessing.pool import ThreadPool
-from typing import Optional
 
 import click
 from rich.console import Console
@@ -75,9 +74,9 @@ def _do_install_package(package: Package, version: str = None, target: InstallTa
 )
 def install(
     package: list[str],
-    parallel: Optional[int] = 1,
-    version: Optional[str] = None,
-    target: Optional[str] = None,
+    parallel: int | None = 1,
+    version: str | None = None,
+    target: str | None = None,
 ):
     """Install one or more packages."""
     try:

localstack/cli/profiles.py

@@ -1,7 +1,6 @@
 import argparse
 import os
 import sys
-from typing import Optional
 
 # important: this needs to be free of localstack imports
 
@@ -44,7 +43,7 @@ def set_and_remove_profile_from_sys_argv():
         os.environ["CONFIG_PROFILE"] = profile.strip()
 
 
-def parse_p_argument(args) -> Optional[str]:
+def parse_p_argument(args) -> str | None:
     """
     Lightweight arg parsing to find the first occurrence of ``-p <config>``, or ``-p=<config>`` and return the value of
     ``<config>`` from the given arguments.

localstack/config.py

@@ -10,7 +10,7 @@ import time
 import warnings
 from collections import defaultdict
 from collections.abc import Mapping
-from typing import Any, Optional, TypeVar, Union
+from typing import Any, TypeVar
 
 from localstack import constants
 from localstack.constants import (
@@ -19,6 +19,7 @@ from localstack.constants import (
     DEFAULT_VOLUME_DIR,
     ENV_INTERNAL_TEST_COLLECT_METRIC,
     ENV_INTERNAL_TEST_RUN,
+    ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK,
     FALSE_STRINGS,
     LOCALHOST,
     LOCALHOST_IP,
@@ -208,13 +209,13 @@ class Directories:
         return str(self.__dict__)
 
 
-def eval_log_type(env_var_name: str) -> Union[str, bool]:
+def eval_log_type(env_var_name: str) -> str | bool:
     """Get the log type from environment variable"""
     ls_log = os.environ.get(env_var_name, "").lower().strip()
     return ls_log if ls_log in LOG_LEVELS else False
 
 
-def parse_boolean_env(env_var_name: str) -> Optional[bool]:
+def parse_boolean_env(env_var_name: str) -> bool | None:
     """Parse the value of the given env variable and return True/False, or None if it is not a boolean value."""
     value = os.environ.get(env_var_name, "").lower().strip()
     if value in TRUE_STRINGS:
@@ -649,7 +650,7 @@ class UniqueHostAndPortList(list[HostAndPort]):
         - Identical identical hosts and ports are de-duped
     """
 
-    def __init__(self, iterable: Union[list[HostAndPort], None] = None):
+    def __init__(self, iterable: list[HostAndPort] | None = None):
         super().__init__(iterable or [])
         self._ensure_unique()
 
@@ -1451,6 +1452,11 @@ def is_collect_metrics_mode() -> bool:
     return is_env_true(ENV_INTERNAL_TEST_COLLECT_METRIC)
 
 
+def store_test_metrics_in_local_filesystem() -> bool:
+    """Returns True if test metrics should be stored in the local filesystem (instead of the system that runs pytest)."""
+    return is_env_true(ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK)
+
+
 def collect_config_items() -> list[tuple[str, Any]]:
     """Returns a list of key-value tuples of LocalStack configuration values."""
     none = object()  # sentinel object
@@ -1503,10 +1509,10 @@ def get_protocol() -> str:
 
 
 def external_service_url(
-    host: Optional[str] = None,
-    port: Optional[int] = None,
-    protocol: Optional[str] = None,
-    subdomains: Optional[str] = None,
+    host: str | None = None,
+    port: int | None = None,
+    protocol: str | None = None,
+    subdomains: str | None = None,
 ) -> str:
     """Returns a service URL (e.g., SQS queue URL) to an external client (e.g., boto3) potentially running on another
     machine than LocalStack. The configurations LOCALSTACK_HOST and USE_SSL can customize these returned URLs.
@@ -1523,10 +1529,10 @@ def external_service_url(
 
 
 def internal_service_url(
-    host: Optional[str] = None,
-    port: Optional[int] = None,
-    protocol: Optional[str] = None,
-    subdomains: Optional[str] = None,
+    host: str | None = None,
+    port: int | None = None,
+    protocol: str | None = None,
+    subdomains: str | None = None,
 ) -> str:
     """Returns a service URL for internal use within LocalStack (i.e., same host).
     The configuration USE_SSL can customize these returned URLs but LOCALSTACK_HOST has no effect.