localstack-core 4.7.1.dev49__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/aws/protocol/service_router.py

@@ -6,9 +6,11 @@ from werkzeug.exceptions import RequestEntityTooLarge
 from werkzeug.http import parse_dict_header
 
 from localstack.aws.spec import (
+    ProtocolName,
     ServiceCatalog,
     ServiceModelIdentifier,
     get_service_catalog,
+    is_protocol_in_service_model_identifier,
 )
 from localstack.http import Request
 from localstack.services.s3.utils import uses_host_addressing
@@ -17,6 +19,23 @@ from localstack.utils.strings import to_bytes
 
 LOG = logging.getLogger(__name__)
 
+_PROTOCOL_DETECTION_PRIORITY: list[ProtocolName] = [
+    "smithy-rpc-v2-cbor",
+    "json",
+    "query",
+    "ec2",
+    "rest-json",
+    "rest-xml",
+]
+
+
+class ProtocolError(Exception):
+    """
+    Error which is thrown if we cannot detect the protocol for the request.
+    """
+
+    pass
+
 
 class _ServiceIndicators(NamedTuple):
     """
@@ -43,6 +62,7 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
     """Extracts all different fields that might indicate which service a request is targeting."""
     x_amz_target = request.headers.get("x-amz-target")
     authorization = request.headers.get("authorization")
+    is_rpc_v2 = "rpc-v2-cbor" in request.headers.get("Smithy-Protocol", "")
 
     signing_name = None
     if authorization:
@@ -55,7 +75,15 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
         except (ValueError, KeyError):
             LOG.debug("auth header could not be parsed for service routing: %s", authorization)
             pass
-    if x_amz_target:
+    if is_rpc_v2:
+        # https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html#requests
+        rpc_v2_params = request.path.lstrip("/").split("/")
+        if len(rpc_v2_params) >= 4:
+            *_, service_shape_name, __, operation = rpc_v2_params
+            target_prefix = service_shape_name.split("#")[-1]
+        else:
+            target_prefix, operation = None, None
+    elif x_amz_target:
         if "." in x_amz_target:
             target_prefix, operation = x_amz_target.split(".", 1)
         else:
@@ -67,6 +95,48 @@ def _extract_service_indicators(request: Request) -> _ServiceIndicators:
     return _ServiceIndicators(signing_name, target_prefix, operation, request.host, request.path)
 
 
+def _matches_protocol(request: Request, protocol: ProtocolName) -> bool:
+    headers = request.headers
+    mimetype = request.mimetype.lower()
+    match protocol:
+        case "smithy-rpc-v2-cbor":
+            # Every request for the rpcv2Cbor protocol MUST contain a `Smithy-Protocol` header with the value
+            # of `rpc-v2-cbor`.
+            # https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html
+            return headers.get("Smithy-Protocol", "") == "rpc-v2-cbor"
+        case "json":
+            return mimetype.startswith("application/x-amz-json")
+        case "query" | "ec2":
+            # https://smithy.io/2.0/aws/protocols/aws-query-protocol.html#request-serialization
+            return (
+                mimetype.startswith("application/x-www-form-urlencoded") or "Action" in request.args
+            )
+        case "rest-xml" | "rest-json":
+            # `rest-json` and `rest-xml` can accept any kind of Content-Type, and it can be configured on the operation
+            # level.
+            # https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html
+            return True
+        case _:
+            return False
+
+
+def match_available_protocols(
+    request: Request, available_protocols: list[ProtocolName]
+) -> ProtocolName | None:
+    """
+    Tries to match the current request and determine the protocol used amongst the available protocols given.
+    We use a priority order to try to determine the protocol, as some protocols are more permissive that others.
+    :param request: the incoming request
+    :param available_protocols: the available protocols of the Service the request is directed to
+    :return: the protocol matched, if any
+    """
+    for protocol in _PROTOCOL_DETECTION_PRIORITY:
+        if protocol in available_protocols and _matches_protocol(request, protocol):
+            return protocol
+
+    return None
+
+
 signing_name_path_prefix_rules = {
     # custom rules based on URI path prefixes that are not easily generalizable
     "apigateway": {
@@ -248,10 +318,10 @@ def resolve_conflicts(
         # The `application/x-amz-json-1.0` header is mandatory for requests targeting SQS with the `json` protocol. We
         # can safely route them to the `sqs` JSON parser/serializer. If not present, route the request to the
         # sqs-query protocol.
-        content_type = request.headers.get("Content-Type")
+        protocol = match_available_protocols(request, available_protocols=["json", "query"])
         return (
             ServiceModelIdentifier("sqs")
-            if content_type == "application/x-amz-json-1.0"
+            if protocol == "json"
             else ServiceModelIdentifier("sqs", "query")
         )
 
@@ -266,7 +336,25 @@ def determine_aws_service_model_for_data_plane(
     custom_host_match = custom_host_addressing_rules(request.host)
     if custom_host_match:
         services = services or get_service_catalog()
-        return services.get(*custom_host_match)
+        return services.get(custom_host_match.name, custom_host_match.protocol)
+
+
+def determine_aws_protocol(request: Request, service_model: ServiceModel) -> ProtocolName:
+    if not (protocols := service_model.metadata.get("protocols")):
+        # if the service does not define multiple protocols, return the `protocol` defined for the service
+        return service_model.protocol
+
+    if len(protocols) == 1:
+        return protocols[0]
+
+    if protocol := match_available_protocols(request, available_protocols=protocols):
+        return protocol
+
+    raise ProtocolError(
+        f"Could not determine the protocol for the request: "
+        f"{request.method} {request.path} for the service '{service_model.service_name}' "
+        f"(available protocols: {protocols})"
+    )
 
 
 def determine_aws_service_model(
@@ -287,12 +375,13 @@ def determine_aws_service_model(
         signing_name_candidates = services.by_signing_name(signing_name)
         if len(signing_name_candidates) == 1:
             # a unique signing-name -> service name mapping is the case for ~75% of service operations
-            return services.get(*signing_name_candidates[0])
+            candidate = signing_name_candidates[0]
+            return services.get(candidate.name, candidate.protocol)
 
         # try to find a match with the custom signing name rules
         custom_match = custom_signing_name_rules(signing_name, path)
         if custom_match:
-            return services.get(*custom_match)
+            return services.get(custom_match.name, custom_match.protocol)
 
         # still ambiguous - add the services to the list of candidates
         candidates.update(signing_name_candidates)
@@ -302,33 +391,34 @@ def determine_aws_service_model(
         target_candidates = services.by_target_prefix(target_prefix)
         if len(target_candidates) == 1:
             # a unique target prefix
-            return services.get(*target_candidates[0])
+            candidate = target_candidates[0]
+            return services.get(candidate.name, candidate.protocol)
 
         # still ambiguous - add the services to the list of candidates
         candidates.update(target_candidates)
 
         # exclude services where the operation is not contained in the service spec
         for service_identifier in list(candidates):
-            service = services.get(*service_identifier)
+            service = services.get(service_identifier.name, service_identifier.protocol)
             if operation not in service.operation_names:
                 candidates.remove(service_identifier)
     else:
         # exclude services which have a target prefix (the current request does not have one)
         for service_identifier in list(candidates):
-            service = services.get(*service_identifier)
+            service = services.get(service_identifier.name, service_identifier.protocol)
             if service.metadata.get("targetPrefix") is not None:
                 candidates.remove(service_identifier)
 
     if len(candidates) == 1:
         service_identifier = candidates.pop()
-        return services.get(*service_identifier)
+        return services.get(service_identifier.name, service_identifier.protocol)
 
     # 3. check the path if it is set and not a trivial root path
     if path and path != "/":
         # try to find a match with the custom path rules
         custom_path_match = custom_path_addressing_rules(path)
         if custom_path_match:
-            return services.get(*custom_path_match)
+            return services.get(custom_path_match.name, custom_path_match.protocol)
 
     # 4. check the host (custom host addressing rules)
     if host:
@@ -337,12 +427,14 @@ def determine_aws_service_model(
             # this prevents a virtual host addressed bucket to be wrongly recognized
             if host.startswith(f"{prefix}.") and ".s3." not in host:
                 if len(services_per_prefix) == 1:
-                    return services.get(*services_per_prefix[0])
+                    candidate = services_per_prefix[0]
+                    return services.get(candidate.name, candidate.protocol)
                 candidates.update(services_per_prefix)
 
         custom_host_match = custom_host_addressing_rules(host)
         if custom_host_match:
-            return services.get(*custom_host_match)
+            candidate = custom_host_match[0]
+            return services.get(candidate.name, candidate.protocol)
 
     if request.shallow:
         # from here on we would need access to the request body, which doesn't exist for shallow requests like
@@ -357,21 +449,28 @@ def determine_aws_service_model(
             query_candidates = [
                 service
                 for service in services.by_operation(values["Action"])
-                if service.protocol in ("ec2", "query")
+                if any(
+                    is_protocol_in_service_model_identifier(protocol, service)
+                    for protocol in ("ec2", "query")
+                )
             ]
 
             if len(query_candidates) == 1:
-                return services.get(*query_candidates[0])
+                candidate = query_candidates[0]
+                return services.get(candidate.name, candidate.protocol)
 
             if "Version" in values:
                 for service_identifier in list(query_candidates):
-                    service_model = services.get(*service_identifier)
+                    service_model = services.get(
+                        service_identifier.name, service_identifier.protocol
+                    )
                     if values["Version"] != service_model.api_version:
                         # the combination of Version and Action is not unique, add matches to the candidates
                         query_candidates.remove(service_identifier)
 
             if len(query_candidates) == 1:
-                return services.get(*query_candidates[0])
+                candidate = query_candidates[0]
+                return services.get(candidate.name, candidate.protocol)
 
             candidates.update(query_candidates)
 
@@ -387,15 +486,16 @@ def determine_aws_service_model(
     # 6. resolve service spec conflicts
     resolved_conflict = resolve_conflicts(candidates, request)
     if resolved_conflict:
-        return services.get(*resolved_conflict)
+        return services.get(resolved_conflict.name, resolved_conflict.protocol)
 
     # 7. check the legacy S3 rules in the end
     legacy_match = legacy_s3_rules(request)
     if legacy_match:
-        return services.get(*legacy_match)
+        return services.get(legacy_match.name, legacy_match.protocol)
 
     if signing_name:
         return services.get(name=signing_name)
     if candidates:
-        return services.get(*candidates.pop())
+        candidate = candidates.pop()
+        return services.get(candidate.name, candidate.protocol)
     return None

localstack/aws/protocol/validate.py

@@ -159,7 +159,7 @@ class ParamValidator(BotocoreParamValidator):
             if required_member in params and params[required_member] is None:
                 params.pop(required_member)
 
-        super(ParamValidator, self)._validate_structure(params, shape, errors, name)
+        super()._validate_structure(params, shape, errors, name)
 
 
 def validate_request(operation: OperationModel, request: ServiceRequest) -> ValidationErrors:

localstack/aws/scaffold.py

@@ -221,7 +221,7 @@ class ShapeNode:
 
     def _print_as_typed_dict(self, output, doc=True, quote_types=False):
         name = to_valid_python_name(self.shape.name)
-        output.write('%s = TypedDict("%s", {\n' % (name, name))
+        output.write(f'{name} = TypedDict("{name}", {{\n')
         for k, v in self.shape.members.items():
             member_name = to_valid_python_name(v.name)
             # check if the member name is the same as the type name (recursive types need to use forward references)

localstack/aws/skeleton.py

@@ -130,14 +130,16 @@ class Skeleton:
             self.dispatch_table = create_dispatch_table(implementation)
 
     def invoke(self, context: RequestContext) -> Response:
-        serializer = create_serializer(context.service)
+        serializer = create_serializer(context.service, context.protocol)
 
         if context.operation and context.service_request:
             # if the parsed request is already set in the context, re-use them
             operation, instance = context.operation, context.service_request
         else:
             # otherwise, parse the incoming HTTPRequest
-            operation, instance = create_parser(context.service).parse(context.request)
+            operation, instance = create_parser(context.service, context.protocol).parse(
+                context.request
+            )
             context.operation = operation
 
         try:

localstack/aws/spec-patches.json

@@ -1329,6 +1329,26 @@
         "documentation": "<p>The Content-MD5 you specified did not match what we received.</p>",
         "exception": true
       }
+    },
+    {
+      "op": "add",
+      "path": "/shapes/AuthorizationHeaderMalformed",
+      "value": {
+        "type": "structure",
+        "members": {
+          "Region": {
+            "shape": "BucketRegion"
+          },
+          "HostId": {
+            "shape": "HostId"
+          }
+        },
+        "error": {
+          "httpStatusCode": 400
+        },
+        "documentation": "<p>The authorization header is malformed.</p>",
+        "exception": true
+      }
     }
   ],
   "apigatewayv2/2018-11-29/service-2": [
@@ -1352,5 +1372,43 @@
       "path": "/operations/CreateApiMapping/http/responseCode",
       "value": 200
     }
+  ],
+  "cloudwatch/2010-08-01/service-2": [
+    {
+      "op": "add",
+      "path": "/metadata/awsQueryCompatible",
+      "value": {}
+    },
+    {
+      "op": "add",
+      "path": "/metadata/jsonVersion",
+      "value": "1.0"
+    },
+    {
+      "op": "add",
+      "path": "/metadata/targetPrefix",
+      "value": "GraniteServiceVersion20100801"
+    },
+    {
+      "op": "replace",
+      "path": "/metadata/protocol",
+      "value": "smithy-rpc-v2-cbor"
+    },
+    {
+      "op": "replace",
+      "path": "/metadata/protocols",
+      "value": [
+        "smithy-rpc-v2-cbor",
+        "json",
+        "query"
+      ]
+    },
+    {
+      "op": "add",
+      "path": "/shapes/ConflictException/error",
+      "value": {
+        "httpStatusCode": 409
+      }
+    }
   ]
 }

localstack/aws/spec.py

@@ -21,7 +21,7 @@ from localstack.utils.objects import singleton_factory
 LOG = logging.getLogger(__name__)
 
 ServiceName = str
-ProtocolName = Literal["query", "json", "rest-json", "rest-xml", "ec2"]
+ProtocolName = Literal["query", "json", "rest-json", "rest-xml", "ec2", "smithy-rpc-v2-cbor"]
 
 
 class ServiceModelIdentifier(NamedTuple):
@@ -33,6 +33,7 @@ class ServiceModelIdentifier(NamedTuple):
 
     name: ServiceName
     protocol: ProtocolName | None = None
+    protocols: tuple[ProtocolName] | None = None
 
 
 spec_patches_json = os.path.join(os.path.dirname(__file__), "spec-patches.json")
@@ -69,7 +70,7 @@ class PatchingLoader(Loader):
 
     @instance_cache
     def load_data(self, name: str):
-        result = super(PatchingLoader, self).load_data(name)
+        result = super().load_data(name)
 
         if patches := self.patches.get(name):
             return jsonpatch.apply_patch(result, patches)
@@ -114,9 +115,13 @@ def load_service(
     :raises: UnknownServiceProtocolError if the specific protocol of the service cannot be found
     """
     service_description = loader.load_service_model(service, "service-2", version)
+    service_metadata = service_description.get("metadata", {})
+    service_protocols = {service_metadata.get("protocol")}
+    if protocols := service_metadata.get("protocols"):
+        service_protocols.update(protocols)
 
     # check if the protocol is defined, and if so, if the loaded service defines this protocol
-    if protocol is not None and protocol != service_description.get("metadata", {}).get("protocol"):
+    if protocol is not None and protocol not in service_protocols:
         # if the protocol is defined, but not the one of the currently loaded service,
         # check if we already loaded the custom spec based on the naming convention (<service>-<protocol>),
         # f.e. "sqs-query"
@@ -132,7 +137,7 @@ def load_service(
 
     # remove potential protocol names from the service name
     # FIXME add more protocols here if we have to internalize more than just sqs-query
-    # TODO this should not contain specific internalized serivce names
+    # TODO this should not contain specific internalized service names
     service = {"sqs-query": "sqs"}.get(service, service)
     return ServiceModel(service_description, service)
 
@@ -149,6 +154,27 @@ def iterate_service_operations() -> Generator[tuple[ServiceModel, OperationModel
             yield service, service.operation_model(op_name)
 
 
+def is_protocol_in_service_model_identifier(
+    protocol: ProtocolName, service_model_identifier: ServiceModelIdentifier
+) -> bool:
+    """
+    :param protocol: the protocol name to check
+    :param service_model_identifier:
+    :return: boolean to indicate if the protocol is available for that service
+    """
+    protocols = service_model_identifier.protocols or []
+    return protocol in protocols or protocol == service_model_identifier.protocol
+
+
+def get_service_model_identifier(service_model: ServiceModel) -> ServiceModelIdentifier:
+    protocols = service_model.metadata.get("protocols")
+    return ServiceModelIdentifier(
+        name=service_model.service_name,
+        protocol=service_model.protocol,
+        protocols=tuple(protocols) if protocols else None,
+    )
+
+
 @dataclasses.dataclass
 class ServiceCatalogIndex:
     """
@@ -178,9 +204,7 @@ class LazyServiceCatalogIndex:
             for service_model in service_models:
                 target_prefix = service_model.metadata.get("targetPrefix")
                 if target_prefix:
-                    result[target_prefix].append(
-                        ServiceModelIdentifier(service_model.service_name, service_model.protocol)
-                    )
+                    result[target_prefix].append(get_service_model_identifier(service_model))
         return dict(result)
 
     @cached_property
@@ -189,7 +213,7 @@ class LazyServiceCatalogIndex:
         for service_models in self._services.values():
             for service_model in service_models:
                 result[service_model.signing_name].append(
-                    ServiceModelIdentifier(service_model.service_name, service_model.protocol)
+                    get_service_model_identifier(service_model)
                 )
         return dict(result)
 
@@ -201,11 +225,7 @@ class LazyServiceCatalogIndex:
                 operations = service_model.operation_names
                 if operations:
                     for operation in operations:
-                        result[operation].append(
-                            ServiceModelIdentifier(
-                                service_model.service_name, service_model.protocol
-                            )
-                        )
+                        result[operation].append(get_service_model_identifier(service_model))
         return dict(result)
 
     @cached_property
@@ -214,7 +234,7 @@ class LazyServiceCatalogIndex:
         for service_models in self._services.values():
             for service_model in service_models:
                 result[service_model.endpoint_prefix].append(
-                    ServiceModelIdentifier(service_model.service_name, service_model.protocol)
+                    get_service_model_identifier(service_model)
                 )
         return dict(result)
 
@@ -343,8 +363,9 @@ def get_service_catalog() -> ServiceCatalog:
                 index = build_service_index_cache(cache_catalog_file)
         return ServiceCatalog(index)
     except Exception:
-        LOG.exception(
-            "error while processing service catalog index cache, falling back to lazy-loaded index"
+        LOG.error(
+            "error while processing service catalog index cache, falling back to lazy-loaded index",
+            exc_info=LOG.isEnabledFor(logging.DEBUG),
         )
         return ServiceCatalog()
 

localstack/cli/exceptions.py

@@ -12,7 +12,7 @@ class CLIError(ClickException):
     def format_message(self) -> str:
         return click.style(f"❌ Error: {self.message}", fg="red")
 
-    def show(self, file: t.Optional[t.IO[t.Any]] = None) -> None:
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
         if file is None:
             file = get_text_stderr()
 

localstack/cli/localstack.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 import traceback
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 import click
 import requests
@@ -47,7 +47,7 @@ class LocalStackCliGroup(click.Group):
 
     def invoke(self, ctx: click.Context):
         try:
-            return super(LocalStackCliGroup, self).invoke(ctx)
+            return super().invoke(ctx)
         except click.exceptions.Exit:
             # raise Exit exceptions unmodified (e.g., raised on --help)
             raise
@@ -320,8 +320,8 @@ class DockerStatus(TypedDict, total=False):
     image_tag: str
     image_id: str
     image_created: str
-    container_name: Optional[str]
-    container_ip: Optional[str]
+    container_name: str | None
+    container_ip: str | None
 
 
 def _print_docker_status(format_: str) -> None:
@@ -602,7 +602,7 @@ def cmd_stop() -> None:
 
     try:
         DOCKER_CLIENT.stop_container(container_name)
-        console.print("container stopped: %s" % container_name)
+        console.print(f"container stopped: {container_name}")
     except NoSuchContainer:
         raise CLIError(
             f'Expected a running LocalStack container named "{container_name}", but found none'
@@ -699,7 +699,7 @@ def cmd_logs(follow: bool, tail: int) -> None:
     metavar="N",
 )
 @publish_invocation
-def cmd_wait(timeout: Optional[float] = None) -> None:
+def cmd_wait(timeout: float | None = None) -> None:
     """
     Wait for the LocalStack runtime to be up and running.
 

localstack/cli/lpm.py

@@ -1,7 +1,6 @@
 import itertools
 import logging
 from multiprocessing.pool import ThreadPool
-from typing import Optional
 
 import click
 from rich.console import Console
@@ -75,9 +74,9 @@ def _do_install_package(package: Package, version: str = None, target: InstallTa
 )
 def install(
     package: list[str],
-    parallel: Optional[int] = 1,
-    version: Optional[str] = None,
-    target: Optional[str] = None,
+    parallel: int | None = 1,
+    version: str | None = None,
+    target: str | None = None,
 ):
     """Install one or more packages."""
     try:

localstack/cli/plugins.py

@@ -56,7 +56,7 @@ def find(where, exclude, include, output):
     elif output == "dict":
         rprint(dict(plugins))
     else:
-        raise CLIError("unknown output format %s" % output)
+        raise CLIError(f"unknown output format {output}")
 
 
 @cli.command("list")

localstack/cli/profiles.py

@@ -1,7 +1,6 @@
 import argparse
 import os
 import sys
-from typing import Optional
 
 # important: this needs to be free of localstack imports
 
@@ -44,7 +43,7 @@ def set_and_remove_profile_from_sys_argv():
         os.environ["CONFIG_PROFILE"] = profile.strip()
 
 
-def parse_p_argument(args) -> Optional[str]:
+def parse_p_argument(args) -> str | None:
     """
     Lightweight arg parsing to find the first occurrence of ``-p <config>``, or ``-p=<config>`` and return the value of
     ``<config>`` from the given arguments.