localstack-core 4.10.1.dev7__py3-none-any.whl → 4.11.2.dev14__py3-none-any.whl

localstack/utils/catalog/catalog_loader.py

@@ -1,11 +1,119 @@
 import json
+import logging
+from json import JSONDecodeError
+from pathlib import Path
 
+import requests
+from pydantic import BaseModel
+
+from localstack import config, constants
 from localstack.utils.catalog.common import AwsRemoteCatalog
+from localstack.utils.http import get_proxies
+from localstack.utils.json import FileMappedDocument
+
+LOG = logging.getLogger(__name__)
+
+AWS_CATALOG_FILE_NAME = "aws_catalog.json"
+
 
-LICENSE_CATALOG_PATH = ""
+class RemoteCatalogVersionResponse(BaseModel):
+    emulator_type: str
+    version: str
+
+
+class AwsCatalogLoaderException(Exception):
+    def __init__(self, msg: str, *args):
+        super().__init__(msg, *args)
 
 
 class RemoteCatalogLoader:
+    supported_schema_version = "v1"
+    api_endpoint_catalog = f"{constants.API_ENDPOINT}/license/catalog"
+    catalog_file_path = Path(config.dirs.cache) / AWS_CATALOG_FILE_NAME
+
     def get_remote_catalog(self) -> AwsRemoteCatalog:
-        with open(LICENSE_CATALOG_PATH) as f:
-            return AwsRemoteCatalog(**json.load(f))
+        catalog_doc = FileMappedDocument(self.catalog_file_path)
+        cached_catalog = AwsRemoteCatalog(**catalog_doc) if catalog_doc else None
+        if cached_catalog:
+            cached_catalog_version = cached_catalog.localstack.version
+            if not self._should_update_cached_catalog(cached_catalog_version):
+                return cached_catalog
+        catalog = self._get_catalog_from_platform()
+        self._save_catalog_to_cache(catalog_doc, catalog)
+        return catalog
+
+    def _get_latest_localstack_version(self) -> str:
+        try:
+            proxies = get_proxies()
+            response = requests.get(
+                f"{self.api_endpoint_catalog}/aws/version",
+                verify=not config.is_env_true("SSL_NO_VERIFY"),
+                proxies=proxies,
+            )
+            if response.ok:
+                return RemoteCatalogVersionResponse.model_validate(response.content).version
+            self._raise_server_error(response)
+        except requests.exceptions.RequestException as e:
+            raise AwsCatalogLoaderException(
+                f"An unexpected network error occurred when trying to fetch latest localstack version: {e}"
+            ) from e
+
+    def _should_update_cached_catalog(self, current_catalog_version: str) -> bool:
+        try:
+            latest_version = self._get_latest_localstack_version()
+            return latest_version != current_catalog_version
+        except Exception as e:
+            LOG.warning(
+                "Failed to retrieve the latest catalog version, cached catalog update skipped: %s",
+                e,
+            )
+            return False
+
+    def _save_catalog_to_cache(self, catalog_doc: FileMappedDocument, catalog: AwsRemoteCatalog):
+        catalog_doc.clear()
+        catalog_doc.update(catalog.model_dump())
+        catalog_doc.save()
+
+    def _get_catalog_from_platform(self) -> AwsRemoteCatalog:
+        try:
+            proxies = get_proxies()
+            response = requests.post(
+                self.api_endpoint_catalog,
+                verify=not config.is_env_true("SSL_NO_VERIFY"),
+                proxies=proxies,
+            )
+
+            if response.ok:
+                return self._parse_catalog(response.content)
+            self._raise_server_error(response)
+        except requests.exceptions.RequestException as e:
+            raise AwsCatalogLoaderException(
+                f"An unexpected network error occurred when trying to fetch remote catalog: {e}"
+            ) from e
+
+    def _parse_catalog(self, document: bytes) -> AwsRemoteCatalog | None:
+        try:
+            catalog_json = json.loads(document)
+        except JSONDecodeError as e:
+            raise AwsCatalogLoaderException(f"Could not de-serialize json catalog: {e}") from e
+        remote_catalog = AwsRemoteCatalog.model_validate(catalog_json)
+        if remote_catalog.schema_version != self.supported_schema_version:
+            raise AwsCatalogLoaderException(
+                f"Unsupported schema version: '{remote_catalog.schema_version}'. Only '{self.supported_schema_version}' is supported"
+            )
+        return remote_catalog
+
+    def _raise_server_error(self, response: requests.Response):
+        try:
+            server_error = response.json()
+            if error_message := server_error.get("message"):
+                raise AwsCatalogLoaderException(
+                    f"Unexpected AWS catalog server error: {response.text}"
+                )
+            raise AwsCatalogLoaderException(
+                f"A server error occurred while calling remote catalog API (HTTP {response.status_code}): {error_message}"
+            )
+        except Exception:
+            raise AwsCatalogLoaderException(
+                f"An unexpected server error occurred while calling remote catalog API (HTTP {response.status_code}): {response.text}"
+            )

localstack/utils/collections.py

@@ -5,7 +5,7 @@ and manipulate python collection (dicts, list, sets).
 
 import logging
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sized
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sized
 from typing import (
     Any,
     Optional,
@@ -24,6 +24,9 @@ LOG = logging.getLogger(__name__)
 # default regex to match an item in a comma-separated list string
 DEFAULT_REGEX_LIST_ITEM = r"[\w-]+"
 
+_E = TypeVar("_E")
+"""TypeVar var used internally for container type parameters."""
+
 
 class AccessTrackingDict(dict):
     """
@@ -101,21 +104,18 @@ class HashableJsonDict(ImmutableDict):
         return hash(canonical_json(self._dict))
 
 
-_ListType = TypeVar("_ListType")
-
-
-class PaginatedList(list[_ListType]):
+class PaginatedList(list[_E]):
     """List which can be paginated and filtered. For usage in AWS APIs with paginated responses"""
 
     DEFAULT_PAGE_SIZE = 50
 
     def get_page(
         self,
-        token_generator: Callable[[_ListType], str],
+        token_generator: Callable[[_E], str],
         next_token: str = None,
         page_size: int = None,
-        filter_function: Callable[[_ListType], bool] = None,
-    ) -> tuple[list[_ListType], str | None]:
+        filter_function: Callable[[_E], bool] = None,
+    ) -> tuple[list[_E], str | None]:
         if filter_function is not None:
             result_list = list(filter(filter_function, self))
         else:
@@ -528,9 +528,6 @@ def is_comma_delimited_list(string: str, item_regex: str | None = None) -> bool:
     return True
 
 
-_E = TypeVar("_E")
-
-
 def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     """
     Given an iterable, either create a list out of the entire iterable (if `condition` is `True`), or return the empty list.
@@ -540,3 +537,18 @@ def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     []
     """
     return list(filter(lambda _: condition, items))
+
+
+def iter_chunks(items: list[_E], chunk_size: int) -> Generator[list[_E], None, None]:
+    """
+    Split a list into smaller chunks of a specified size and iterate over them.
+
+    It is implemented as a generator and yields each chunk as needed, making it memory-efficient for large lists.
+
+    :param items:  A list of elements to be divided into chunks.
+    :param chunk_size: The maximum number of elements that a single chunk can contain.
+    :return: A generator that yields chunks (sublists) of the original list. Each chunk contains up to `chunk_size`
+    elements.
+    """
+    for i in range(0, len(items), chunk_size):
+        yield items[i : i + chunk_size]

localstack/utils/crypto.py

@@ -4,7 +4,13 @@ import os
 import re
 import threading
 
+from asn1crypto import algos, cms, core
+from asn1crypto import x509 as asn1_x509
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import padding as sym_padding
+from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
 from .files import TMP_FILES, file_exists_not_empty, load_file, new_tmp_file, save_file
@@ -26,6 +32,11 @@ PEM_CERT_END = "-----END CERTIFICATE-----"
 PEM_KEY_START_REGEX = r"-----BEGIN(.*)PRIVATE KEY-----"
 PEM_KEY_END_REGEX = r"-----END(.*)PRIVATE KEY-----"
 
+OID_AES256_CBC = "2.16.840.1.101.3.4.1.42"
+OID_MGF1 = "1.2.840.113549.1.1.8"
+OID_RSAES_OAEP = "1.2.840.113549.1.1.7"
+OID_SHA256 = "2.16.840.1.101.3.4.2.1"
+
 
 @synchronized(lock=SSL_CERT_LOCK)
 def generate_ssl_cert(
@@ -183,3 +194,101 @@ def decrypt(
     decrypted = decryptor.update(encrypted) + decryptor.finalize()
     decrypted = unpad(decrypted)
     return decrypted
+
+
+def pkcs7_envelope_encrypt(plaintext: bytes, recipient_pubkey: RSAPublicKey) -> bytes:
+    """
+    Create a PKCS7 wrapper of some plaintext decryptable by recipient_pubkey.  Uses RSA-OAEP with SHA-256
+    to encrypt the AES-256-CBC content key.  Hazmat's PKCS7EnvelopeBuilder doesn't support RSA-OAEP with SHA-256,
+    so we need to build the pieces manually and then put them together in an envelope with asn1crypto.
+    """
+
+    # Encrypt the plaintext with an AES session key, then encrypt the session key to the recipient_pubkey
+    session_key = os.urandom(32)
+    iv = os.urandom(16)
+    encrypted_session_key = recipient_pubkey.encrypt(
+        session_key,
+        padding.OAEP(
+            mgf=padding.MGF1(algorithm=hashes.SHA256()), algorithm=hashes.SHA256(), label=None
+        ),
+    )
+    cipher = Cipher(algorithms.AES(session_key), modes.CBC(iv), backend=default_backend())
+    encryptor = cipher.encryptor()
+    padder = sym_padding.PKCS7(algorithms.AES.block_size).padder()
+    padded_plaintext = padder.update(plaintext) + padder.finalize()
+    encrypted_content = encryptor.update(padded_plaintext) + encryptor.finalize()
+
+    # Now put together the envelope.
+    # Add the recipient with their copy of the session key
+    recipient_identifier = cms.RecipientIdentifier(
+        name="issuer_and_serial_number",
+        value=cms.IssuerAndSerialNumber(
+            {
+                "issuer": asn1_x509.Name.build({"common_name": "recipient"}),
+                "serial_number": 1,
+            }
+        ),
+    )
+    key_enc_algorithm = cms.KeyEncryptionAlgorithm(
+        {
+            "algorithm": OID_RSAES_OAEP,
+            "parameters": algos.RSAESOAEPParams(
+                {
+                    "hash_algorithm": algos.DigestAlgorithm(
+                        {
+                            "algorithm": OID_SHA256,
+                        }
+                    ),
+                    "mask_gen_algorithm": algos.MaskGenAlgorithm(
+                        {
+                            "algorithm": OID_MGF1,
+                            "parameters": algos.DigestAlgorithm(
+                                {
+                                    "algorithm": OID_SHA256,
+                                }
+                            ),
+                        }
+                    ),
+                }
+            ),
+        }
+    )
+    recipient_info = cms.KeyTransRecipientInfo(
+        {
+            "version": "v0",
+            "rid": recipient_identifier,
+            "key_encryption_algorithm": key_enc_algorithm,
+            "encrypted_key": encrypted_session_key,
+        }
+    )
+
+    # Add the encrypted content
+    content_enc_algorithm = cms.EncryptionAlgorithm(
+        {
+            "algorithm": OID_AES256_CBC,
+            "parameters": core.OctetString(iv),
+        }
+    )
+    encrypted_content_info = cms.EncryptedContentInfo(
+        {
+            "content_type": "data",
+            "content_encryption_algorithm": content_enc_algorithm,
+            "encrypted_content": encrypted_content,
+        }
+    )
+    enveloped_data = cms.EnvelopedData(
+        {
+            "version": "v0",
+            "recipient_infos": [recipient_info],
+            "encrypted_content_info": encrypted_content_info,
+        }
+    )
+
+    # Finally add a wrapper and return its bytes
+    content_info = cms.ContentInfo(
+        {
+            "content_type": "enveloped_data",
+            "content": enveloped_data,
+        }
+    )
+    return content_info.dump()

localstack/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '4.10.1.dev7'
-__version_tuple__ = version_tuple = (4, 10, 1, 'dev7')
+__version__ = version = '4.11.2.dev14'
+__version_tuple__ = version_tuple = (4, 11, 2, 'dev14')
 
 __commit_id__ = commit_id = None

{localstack_core-4.10.1.dev7.dist-info → localstack_core-4.11.2.dev14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localstack-core
-Version: 4.10.1.dev7
+Version: 4.11.2.dev14
 Summary: The core library and runtime of LocalStack
 Author-email: LocalStack Contributors <info@localstack.cloud>
 License-Expression: Apache-2.0
@@ -15,7 +15,8 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: System :: Emulators
 Requires-Python: >=3.10
 License-File: LICENSE.txt
-Requires-Dist: click>=7.1
+Requires-Dist: asn1crypto>=1.5.1
+Requires-Dist: click>=8.2.0
 Requires-Dist: cachetools>=5.0
 Requires-Dist: cryptography
 Requires-Dist: dill==0.3.6
@@ -29,8 +30,8 @@ Requires-Dist: rich>=12.3.0
 Requires-Dist: requests>=2.20.0
 Requires-Dist: semver>=2.10
 Provides-Extra: base-runtime
-Requires-Dist: boto3==1.40.64; extra == "base-runtime"
-Requires-Dist: botocore==1.40.64; extra == "base-runtime"
+Requires-Dist: boto3==1.42.0; extra == "base-runtime"
+Requires-Dist: botocore==1.41.6; extra == "base-runtime"
 Requires-Dist: awscrt!=0.27.1,>=0.13.14; extra == "base-runtime"
 Requires-Dist: cbor2>=5.5.0; extra == "base-runtime"
 Requires-Dist: dnspython>=1.16.0; extra == "base-runtime"
@@ -52,7 +53,7 @@ Requires-Dist: xmltodict>=0.13.0; extra == "base-runtime"
 Requires-Dist: rolo>=0.7; extra == "base-runtime"
 Provides-Extra: runtime
 Requires-Dist: localstack-core[base-runtime]; extra == "runtime"
-Requires-Dist: awscli==1.42.64; extra == "runtime"
+Requires-Dist: awscli==1.43.6; extra == "runtime"
 Requires-Dist: airspeed-ext>=0.6.3; extra == "runtime"
 Requires-Dist: antlr4-python3-runtime==4.13.2; extra == "runtime"
 Requires-Dist: apispec>=5.1.1; extra == "runtime"
@@ -99,5 +100,5 @@ Requires-Dist: rstr>=3.2.0; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Provides-Extra: typehint
 Requires-Dist: localstack-core[dev]; extra == "typehint"
-Requires-Dist: boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codebuild,codecommit,codeconnections,codedeploy,codepipeline,codestar-connections,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pinpoint,pipes,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,verifiedpermissions,wafv2,xray]; extra == "typehint"
+Requires-Dist: boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codebuild,codecommit,codeconnections,codedeploy,codepipeline,codestar-connections,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pinpoint,pipes,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,verifiedpermissions,wafv2,xray]; extra == "typehint"
 Dynamic: license-file