localstack-core 4.11.2.dev14__py3-none-any.whl → 4.12.1.dev18__py3-none-any.whl

localstack/dev/kubernetes/__main__.py

@@ -1,5 +1,7 @@
 import dataclasses
 import os
+import shlex
+import subprocess as sp
 from typing import Literal
 
 import click
@@ -315,12 +317,11 @@ def generate_k8s_helm_overrides(
     return overrides
 
 
-def write_file(content: dict, output_path: str, file_name: str):
-    path = os.path.join(output_path, file_name)
-    with open(path, "w") as f:
+def write_file(content: dict, output_path: str):
+    with open(output_path, "w") as f:
         f.write(yaml.dump(content))
         f.close()
-        print(f"Generated file at {path}")
+        print(f"Generated file at {output_path}")
 
 
 def print_file(content: dict, file_name: str):
@@ -330,6 +331,22 @@ def print_file(content: dict, file_name: str):
     print("=====================================")
 
 
+def generate_k3d_command(config_file_path: str) -> str:
+    return f"k3d cluster create --config {config_file_path}"
+
+
+def generate_helm_command(overrides_file_path: str) -> str:
+    return f"helm upgrade --install localstack localstack/localstack -f {overrides_file_path}"
+
+
+def execute_deployment(config_file_path: str, overrides_file_path: str):
+    """
+    Use the k3d and helm commands to create a cluster and deploy LocalStack in one command
+    """
+    sp.check_call(shlex.split(generate_k3d_command(config_file_path)))
+    sp.check_call(shlex.split(generate_helm_command(overrides_file_path)))
+
+
 @click.command("run")
 @click.option(
     "--pro", is_flag=True, default=None, help="Mount the localstack-pro code into the cluster."
@@ -386,6 +403,13 @@ def print_file(content: dict, file_name: str):
     help="DNS port to expose from the kubernetes node. It is applied only if --expose-dns is set.",
     type=click.IntRange(0, 65535),
 )
+@click.option(
+    "--execute",
+    "-x",
+    is_flag=True,
+    default=False,
+    help="Execute deployment from generated config files. Implies -w/--write.",
+)
 @click.argument("command", nargs=-1, required=False)
 def run(
     pro: bool = None,
@@ -400,6 +424,7 @@ def run(
     port: int = None,
     expose_dns: bool = False,
     dns_port: int = 53,
+    execute: bool = False,
 ):
     """
     A tool for localstack developers to generate the kubernetes cluster configuration file and the overrides to mount the localstack code into the cluster.
@@ -416,25 +441,25 @@ def run(
     overrides_file = overrides_file or "overrides.yml"
     config_file = config_file or "configuration.yml"
 
-    if write:
-        write_file(config, output_dir, config_file)
-        write_file(overrides, output_dir, overrides_file)
+    overrides_file_path = os.path.join(output_dir, overrides_file)
+    config_file_path = os.path.join(output_dir, config_file)
+
+    if write or execute:
+        write_file(config, config_file_path)
+        write_file(overrides, overrides_file_path)
+        if execute:
+            execute_deployment(config_file, overrides_file)
     else:
         print_file(config, config_file)
         print_file(overrides, overrides_file)
 
-    overrides_file_path = os.path.join(output_dir, overrides_file)
-    config_file_path = os.path.join(output_dir, config_file)
-
     print("\nTo create a k3d cluster with the generated configuration, follow these steps:")
     print("1. Run the following command to create the cluster:")
-    print(f"\n    k3d cluster create --config {config_file_path}\n")
+    print(f"\n    {generate_k3d_command(config_file_path)}\n")
 
     print("2. Once the cluster is created, start LocalStack with the generated overrides:")
     print("\n    helm repo add localstack https://localstack.github.io/helm-charts # (if required)")
-    print(
-        f"\n    helm upgrade --install localstack localstack/localstack -f {overrides_file_path}\n"
-    )
+    print(f"\n    {generate_helm_command(overrides_file_path)}\n")
 
 
 def main():

localstack/runtime/analytics.py

@@ -7,10 +7,14 @@ from localstack.utils.analytics import log
 
 LOG = logging.getLogger(__name__)
 
+# Config options for which both usage and values are reported in analytics.
+# Important: This list must only contain options whose values do not contain PII or sensitive data.
 TRACKED_ENV_VAR = [
     "ACTIVATE_PRO",
     "ALLOW_NONSTANDARD_REGIONS",
     "BEDROCK_PREWARM",
+    "CFN_IGNORE_UNSUPPORTED_TYPE_CREATE",
+    "CFN_IGNORE_UNSUPPORTED_TYPE_UPDATE",
     "CFN_IGNORE_UNSUPPORTED_RESOURCE_TYPES",
     "CLOUDFRONT_LAMBDA_EDGE",
     "CONTAINER_RUNTIME",
@@ -26,6 +30,7 @@ TRACKED_ENV_VAR = [
     "DYNAMODB_IN_MEMORY",
     "DYNAMODB_REMOVE_EXPIRED_ITEMS",
     "EAGER_SERVICE_LOADING",
+    "EC2_DOCKER_INIT",
     "EC2_VM_MANAGER",
     "ECS_TASK_EXECUTOR",
     "EDGE_PORT",
@@ -71,9 +76,15 @@ TRACKED_ENV_VAR = [
     "USE_SSL",
 ]
 
+# Config options for which only the usage is reported in analytics.
+# Use this for options which may hold sensitive data or PII.
 PRESENCE_ENV_VAR = [
     "DATA_DIR",
     "EDGE_FORWARD_URL",  # Not functional; deprecated in 1.4.0, removed in 3.0.0
+    "EC2_HYPERVISOR_URI",
+    "EC2_REFERENCE_DOMAIN",
+    "EC2_LIBVIRT_NETWORK",
+    "EC2_LIBVIRT_POOL",
     "GATEWAY_LISTEN",
     "HOSTNAME",
     "HOSTNAME_EXTERNAL",

localstack/services/acm/provider.py

@@ -17,6 +17,16 @@ from localstack.utils.patch import patch
 moto_settings.ACM_VALIDATION_WAIT = min(10, moto_settings.ACM_VALIDATION_WAIT)
 
 
+@patch(acm_models.AWSCertificateManagerBackend.list_certificates)
+def list_certificates(list_certificates_orig, self, statuses, includes):
+    # Normalize keyTypes filter to match our describe() output format (hyphens)
+    if includes and "keyTypes" in includes:
+        includes["keyTypes"] = [
+            kt.replace("RSA_", "RSA-").replace("EC_", "EC-") for kt in includes["keyTypes"]
+        ]
+    return list_certificates_orig(self, statuses, includes)
+
+
 @patch(acm_models.CertBundle.describe)
 def describe(describe_orig, self):
     # TODO fix! Terrible hack (for parity). Moto adds certain required fields only if status is PENDING_VALIDATION.
@@ -71,8 +81,10 @@ def describe(describe_orig, self):
             cert[key] = value
     cert["Serial"] = str(cert.get("Serial") or "")
 
-    if cert.get("KeyAlgorithm") in ["RSA_1024", "RSA_2048"]:
+    if cert.get("KeyAlgorithm") in ["RSA_1024", "RSA_2048", "RSA_3072", "RSA_4096"]:
         cert["KeyAlgorithm"] = cert["KeyAlgorithm"].replace("RSA_", "RSA-")
+    if cert.get("KeyAlgorithm") in ["EC_prime256v1", "EC_secp384r1", "EC_secp521r1"]:
+        cert["KeyAlgorithm"] = cert["KeyAlgorithm"].replace("EC_", "EC-")
 
     # add subject alternative names
     if cert["DomainName"] not in sans:

localstack/services/cloudformation/engine/v2/change_set_model.py

@@ -1169,6 +1169,15 @@ class ChangeSetModel:
                 fn_transform,
             ],
         )
+
+        # special case of where either the before or after state does not specify properties but
+        # the resource was in the previous template
+        if (
+            terminal_value_type.change_type == ChangeType.UNCHANGED
+            and properties.change_type != ChangeType.UNCHANGED
+        ):
+            change_type = ChangeType.MODIFIED
+
         requires_replacement = self._resolve_requires_replacement(
             node_properties=properties, resource_type=terminal_value_type
         )

localstack/services/cloudformation/engine/v2/change_set_model_preproc.py

@@ -1069,7 +1069,9 @@ class ChangeSetModelPreproc(ChangeSetModelVisitor):
 
         def _resolve_parameter_type(value: str, type_: str) -> Any:
             match type_:
-                case "List<String>" | "CommaDelimitedList":
+                case s if re.match(r"List<[^>]+>", s):
+                    return [item.strip() for item in value.split(",")]
+                case "CommaDelimitedList":
                     return [item.strip() for item in value.split(",")]
                 case "Number":
                     # TODO: validate the parameter type at template parse time (or whatever is in parity with AWS) so we know this cannot fail

localstack/services/cloudformation/engine/v2/change_set_resource_support_checker.py

@@ -0,0 +1,114 @@
+from localstack.services.cloudformation.engine.v2.change_set_model import (
+    NodeResource,
+)
+from localstack.services.cloudformation.engine.v2.change_set_model_visitor import (
+    ChangeSetModelVisitor,
+)
+from localstack.services.cloudformation.resources import AWS_AVAILABLE_CFN_RESOURCES
+from localstack.utils.catalog.catalog import (
+    AwsServicesSupportStatus,
+    CatalogPlugin,
+    CfnResourceSupportStatus,
+)
+from localstack.utils.catalog.common import (
+    AwsServicesSupportInLatest,
+    AwsServiceSupportAtRuntime,
+    CloudFormationResourcesSupportAtRuntime,
+    CloudFormationResourcesSupportInLatest,
+)
+from localstack.utils.catalog.plugins import get_aws_catalog
+
+
+# TODO handle all available resource types
+def _get_service_name(resource_type: str) -> str | None:
+    parts = resource_type.split("::")
+    if len(parts) == 1:
+        return None
+
+    match parts:
+        case _ if "Cognito::IdentityPool" in resource_type:
+            return "cognito-identity"
+        case [*_, "Cognito", "UserPool"]:
+            return "cognito-idp"
+        case [*_, "Cognito", _]:
+            return "cognito-idp"
+        case [*_, "Elasticsearch", _]:
+            return "es"
+        case [*_, "OpenSearchService", _]:
+            return "opensearch"
+        case [*_, "KinesisFirehose", _]:
+            return "firehose"
+        case [*_, "ResourceGroups", _]:
+            return "resource-groups"
+        case [*_, "CertificateManager", _]:
+            return "acm"
+        case _ if "ElasticLoadBalancing::" in resource_type:
+            return "elb"
+        case _ if "ElasticLoadBalancingV2::" in resource_type:
+            return "elbv2"
+        case _ if "ApplicationAutoScaling::" in resource_type:
+            return "application-autoscaling"
+        case _ if "MSK::" in resource_type:
+            return "kafka"
+        case _ if "Timestream::" in resource_type:
+            return "timestream-write"
+        case [_, service, *_]:
+            return service.lower()
+
+
+def _build_resource_failure_message(
+    resource_type: str, status: AwsServicesSupportStatus | CfnResourceSupportStatus
+) -> str:
+    service_name = _get_service_name(resource_type) or "malformed"
+    template = "Sorry, the {resource} resource in the {service} service is not supported."
+    match status:
+        case CloudFormationResourcesSupportAtRuntime.NOT_IMPLEMENTED:
+            template = "Sorry, the {resource} resource (from the {service} service) is not supported by this version of LocalStack, but is available in the latest version."
+        case CloudFormationResourcesSupportInLatest.NOT_SUPPORTED:
+            template = "Sorry, the {resource} resource (from the {service} service) is not currently supported by LocalStack."
+        case AwsServiceSupportAtRuntime.AVAILABLE_WITH_LICENSE_UPGRADE:
+            template = "Sorry, the {service} service (for the {resource} resource) is not included within your LocalStack license, but is available in an upgraded license."
+        case AwsServiceSupportAtRuntime.NOT_IMPLEMENTED:
+            template = "The API for service {service} (for the {resource} resource) is either not included in your current license plan or has not yet been emulated by LocalStack."
+        case AwsServicesSupportInLatest.NOT_SUPPORTED:
+            template = "Sorry, the {service} (for the {resource} resource) service is not currently supported by LocalStack."
+        case AwsServicesSupportInLatest.SUPPORTED_WITH_LICENSE_UPGRADE:
+            template = "Sorry, the {service} service (for the {resource} resource) is not supported by this version of LocalStack, but is available in the latest version if you upgrade to the latest stable version."
+    return template.format(
+        resource=resource_type,
+        service=service_name,
+    )
+
+
+class ChangeSetResourceSupportChecker(ChangeSetModelVisitor):
+    catalog: CatalogPlugin
+
+    TITLE_MESSAGE = "Unsupported resources detected:"
+
+    def __init__(self):
+        self._resource_failure_messages: dict[str, str] = {}
+        self.catalog = get_aws_catalog()
+
+    def visit_node_resource(self, node_resource: NodeResource):
+        resource_type = node_resource.type_.value
+        if resource_type not in self._resource_failure_messages:
+            if resource_type not in AWS_AVAILABLE_CFN_RESOURCES:
+                # Ignore non-AWS resources
+                pass
+            support_status = self._resource_support_status(resource_type)
+            if support_status == CloudFormationResourcesSupportAtRuntime.AVAILABLE:
+                pass
+            else:
+                failure_message = _build_resource_failure_message(resource_type, support_status)
+                self._resource_failure_messages[resource_type] = failure_message
+        super().visit_node_resource(node_resource)
+
+    def _resource_support_status(
+        self, resource_type: str
+    ) -> AwsServicesSupportStatus | CfnResourceSupportStatus:
+        service_name = _get_service_name(resource_type)
+        return self.catalog.get_cloudformation_resource_status(resource_type, service_name, True)
+
+    @property
+    def failure_messages(self) -> list[str]:
+        return list(self._resource_failure_messages.values())

localstack/services/cloudformation/provider.py

@@ -5,6 +5,7 @@ import re
 from collections import defaultdict
 from copy import deepcopy
 
+from localstack import config
 from localstack.aws.api import CommonServiceException, RequestContext, handler
 from localstack.aws.api.cloudformation import (
     AlreadyExistsException,
@@ -120,6 +121,7 @@ from localstack.services.cloudformation.stores import (
     find_stack_by_id,
     get_cloudformation_store,
 )
+from localstack.services.plugins import ServiceLifecycleHook
 from localstack.state import StateVisitor
 from localstack.utils.collections import (
     remove_attributes,
@@ -177,7 +179,30 @@ class InternalFailure(CommonServiceException):
         super().__init__("InternalFailure", status_code=500, message=message, sender_fault=False)
 
 
-class CloudformationProvider(CloudformationApi):
+class CloudformationProvider(CloudformationApi, ServiceLifecycleHook):
+    def on_before_start(self):
+        self._validate_config()
+
+    def _validate_config(self):
+        no_wait_value: int = 5
+        try:
+            no_wait_value = int(config.CFN_NO_WAIT_ITERATIONS or 5)
+        except (TypeError, ValueError):
+            LOG.warning(
+                "You have set CFN_NO_WAIT_ITERATIONS to an invalid value: '%s'. It must be an integer greater or equal to 0. Using the default of 5",
+                config.CFN_NO_WAIT_ITERATIONS,
+            )
+
+        if no_wait_value < 0:
+            LOG.warning(
+                "You have set CFN_NO_WAIT_ITERATIONS to an invalid value: '%s'. It must be an integer greater or equal to 0. Using the default of 5",
+                config.CFN_NO_WAIT_ITERATIONS,
+            )
+            no_wait_value = 5
+
+        # Set the configuration back
+        config.CFN_NO_WAIT_ITERATIONS = no_wait_value
+
     def _stack_status_is_active(self, stack_status: str) -> bool:
         return stack_status not in [StackStatus.DELETE_COMPLETE]
 

localstack/services/cloudformation/provider_utils.py

@@ -275,6 +275,26 @@ def convert_values_to_numbers(input_dict: dict, keys_to_skip: list[str] | None =
     return recursive_convert(input_dict)
 
 
+def resource_tags_to_remove_or_update(
+    prev_tags: list[dict], new_tags: list[dict]
+) -> tuple[list[str], dict[str, str]]:
+    """
+    When updating resources that have tags, we need to determine which tags to remove and which to add/update,
+    as these are typically done in separate API calls. The format of prev_tags and new_tags is expected to
+    be [{ "Key": tagName, "Value": tagValue }, ...]. The return value will be a tuple of (tags_to_remove, tags_to_update),
+    where:
+    - tags_to_remove is a list of tag keys that are present in prev_tags but not in new_tags.
+    - tags_to_update is a dict of tags to add or update, with the format: { tagName: tagValue, ... }.
+    """
+    prev_tag_keys = [tag["Key"] for tag in prev_tags]
+    new_tag_keys = [tag["Key"] for tag in new_tags]
+    tags_to_remove = list(set(prev_tag_keys) - set(new_tag_keys))
+
+    # convert from list of dicts, to a single dict because that's what tag_queue APIs expect.
+    tags_to_update = {tag["Key"]: tag["Value"] for tag in new_tags}
+    return (tags_to_remove, tags_to_update)
+
+
 #  LocalStack specific utilities
 def get_schema_path(file_path: Path) -> dict:
     file_name_base = file_path.name.removesuffix(".py").removesuffix(".py.enc")

localstack/services/cloudformation/resource_provider.py

@@ -436,11 +436,11 @@ class ResourceProviderExecutor:
         resource: dict,
         raw_payload: ResourceProviderPayload,
         max_timeout: int = config.CFN_PER_RESOURCE_TIMEOUT,
-        sleep_time: float = 5,
+        sleep_time: float = 1,
     ) -> ProgressEvent[Properties]:
         payload = copy.deepcopy(raw_payload)
 
-        max_iterations = max(ceil(max_timeout / sleep_time), 2)
+        max_iterations = max(ceil(max_timeout / sleep_time), 10)
 
         for current_iteration in range(max_iterations):
             resource_type = get_resource_type({"Type": raw_payload["resourceType"]})
@@ -486,10 +486,11 @@ class ResourceProviderExecutor:
                     payload["requestData"]["resourceProperties"] = event.resource_model
                     resource["Properties"] = event.resource_model
 
-                    if current_iteration == 0:
-                        time.sleep(0)
+                    if current_iteration < config.CFN_NO_WAIT_ITERATIONS:
+                        pass
                     else:
                         time.sleep(sleep_time)
+
                 case OperationStatus.PENDING:
                     # come back to this resource in another iteration
                     return event

localstack/services/cloudformation/scaffolding/__main__.py

@@ -10,6 +10,7 @@ from functools import reduce
 from pathlib import Path
 from typing import Any, Literal, TypedDict, TypeVar
 
+import boto3
 import click
 from jinja2 import Environment, FileSystemLoader
 from yaml import safe_dump
@@ -140,14 +141,76 @@ class SchemaProvider:
             ) from e
 
 
+class LiveSchemaProvider:
+    """
+    Provides CloudFormation resource schemas by fetching them from the live AWS CloudFormation service, rather than
+    a local zip file.
+    """
+
+    def __init__(self, cfn_client):
+        self.cfn_client = cfn_client
+
+    def available_schemas(self, pattern: str) -> list[str]:
+        """
+        Return the names of available CloudFormation resource types. `pattern` should be something like
+        AWS::S3::Bucket or AWS::S3::*, depending on whether you want all resources for a service or a specific one.
+        The result is a list of matching resource type names (e.g. [AWS::S3::Bucket, AWS::S3::Object, ...])
+        """
+
+        is_wildcard = pattern.endswith("*")
+        pattern = pattern[:-1] if is_wildcard else pattern
+        matching_names = []
+
+        params = {
+            "Visibility": "PUBLIC",
+            "Type": "RESOURCE",
+            "DeprecatedStatus": "LIVE",
+            "Filters": {"Category": "AWS_TYPES", "TypeNamePrefix": pattern},
+        }
+        next_token: str | None = None
+
+        # Note: pagination is necessary since list_types requires multiple calls even to get a single result.
+        while True:
+            if next_token:
+                params["NextToken"] = next_token
+            response = self.cfn_client.list_types(**params)
+
+            # collect any matching type names (if wildcard, all; else exact match only)
+            matching_names.extend(
+                [
+                    type_summary["TypeName"]
+                    for type_summary in response.get("TypeSummaries", [])
+                    if (is_wildcard or type_summary["TypeName"] == pattern)
+                ]
+            )
+
+            next_token = response.get("NextToken")
+            if not next_token:
+                break
+
+        return matching_names
+
+    def schema(self, type_name: ResourceName) -> ResourceSchema:
+        """
+        Given a CloudFormation ResourceName (representing something like "AWS::S3::Bucket"), return the resource
+        schema as dict.
+        """
+        response = self.cfn_client.describe_type(
+            Type="RESOURCE",
+            TypeName=type_name.full_name,
+        )
+        schema_str = response.get("Schema")
+        if not schema_str:
+            raise click.ClickException(
+                f"Could not fetch schema for CloudFormation resource type: {type_name}"
+            )
+        return json.loads(schema_str)
+
+
 LOCALSTACK_ROOT_DIR = Path(__file__).parent.joinpath("../../../../..").resolve()
 LOCALSTACK_PRO_ROOT_DIR = LOCALSTACK_ROOT_DIR.joinpath("../localstack-pro").resolve()
-TESTS_ROOT_DIR = LOCALSTACK_ROOT_DIR.joinpath(
-    "tests/aws/services/cloudformation/resource_providers"
-)
-TESTS_PRO_ROOT_DIR = LOCALSTACK_PRO_ROOT_DIR.joinpath(
-    "localstack-pro-core/tests/aws/services/cloudformation/resource_providers"
-)
+TESTS_ROOT_DIR = LOCALSTACK_ROOT_DIR.joinpath("tests/aws/services")
+TESTS_PRO_ROOT_DIR = LOCALSTACK_PRO_ROOT_DIR.joinpath("localstack-pro-core/tests/aws/services")
 
 assert LOCALSTACK_ROOT_DIR.is_dir(), f"{LOCALSTACK_ROOT_DIR} does not exist"
 assert LOCALSTACK_PRO_ROOT_DIR.is_dir(), f"{LOCALSTACK_PRO_ROOT_DIR} does not exist"
@@ -193,7 +256,7 @@ def template_path(
     output_path = (
         tests_root_dir(pro)
         .joinpath(
-            f"{resource_name.python_compatible_service_name.lower()}/{resource_name.path_compatible_full_name()}/templates/{stub}"
+            f"{resource_name.python_compatible_service_name.lower()}/resource_providers/templates/{stub}"
         )
         .resolve()
     )
@@ -202,7 +265,7 @@ def template_path(
         test_path = (
             root_dir(pro)
             .joinpath(
-                f"tests/aws/cloudformation/resource_providers/{resource_name.python_compatible_service_name.lower()}/{resource_name.path_compatible_full_name()}"
+                f"tests/aws/{resource_name.python_compatible_service_name.lower()}/resource_providers/templates"
             )
             .resolve()
         )
@@ -276,7 +339,7 @@ class TemplateRenderer:
         # e.g. .../resource_providers/aws_iam_role/test_X.py vs. .../resource_providers/iam/test_X.py
         # add extra parameters
         tests_output_path = root_dir(self.pro).joinpath(
-            f"tests/aws/cloudformation/resource_providers/{resource_name.python_compatible_service_name.lower()}/{resource_name.full_name.lower()}"
+            f"tests/aws/{resource_name.python_compatible_service_name.lower()}/resource_providers/templates"
         )
         match file_type:
             case FileType.getatt_test:
@@ -284,7 +347,9 @@ class TemplateRenderer:
                 kwargs["service"] = resource_name.service.lower()
                 kwargs["resource"] = resource_name.resource.lower()
                 kwargs["template_path"] = str(
-                    template_path(resource_name, FileType.attribute_template, tests_output_path)
+                    template_path(
+                        resource_name, FileType.attribute_template, tests_output_path, pro=self.pro
+                    )
                 )
             case FileType.provider:
                 property_ir = generate_ir_for_type(
@@ -318,17 +383,25 @@ class TemplateRenderer:
                 kwargs["pro"] = self.pro
             case FileType.integration_test:
                 kwargs["black_box_template_path"] = str(
-                    template_path(resource_name, FileType.minimal_template, tests_output_path)
+                    template_path(
+                        resource_name, FileType.minimal_template, tests_output_path, pro=self.pro
+                    )
                 )
                 kwargs["update_template_path"] = str(
                     template_path(
                         resource_name,
                         FileType.update_without_replacement_template,
                         tests_output_path,
+                        pro=self.pro,
                     )
                 )
                 kwargs["autogenerated_template_path"] = str(
-                    template_path(resource_name, FileType.autogenerated_template, tests_output_path)
+                    template_path(
+                        resource_name,
+                        FileType.autogenerated_template,
+                        tests_output_path,
+                        pro=self.pro,
+                    )
                 )
             # case FileType.cloudcontrol_test:
             case FileType.parity_test:
@@ -531,20 +604,24 @@ class FileWriter:
             ),
             FileType.integration_test: tests_root_dir(self.pro).joinpath(
                 self.resource_name.python_compatible_service_name.lower(),
+                "resource_providers",
                 self.resource_name.path_compatible_full_name(),
                 "test_basic.py",
             ),
             FileType.getatt_test: tests_root_dir(self.pro).joinpath(
                 self.resource_name.python_compatible_service_name.lower(),
+                "resource_providers",
                 self.resource_name.path_compatible_full_name(),
                 "test_exploration.py",
             ),
             # FileType.cloudcontrol_test: tests_root_dir(self.pro).joinpath(
             #     self.resource_name.python_compatible_service_name.lower(),
+            #     "resource_providers",
             #     f"test_aws_{self.resource_name.service.lower()}_{self.resource_name.resource.lower()}_cloudcontrol.py",
             # ),
             FileType.parity_test: tests_root_dir(self.pro).joinpath(
                 self.resource_name.python_compatible_service_name.lower(),
+                "resource_providers",
                 self.resource_name.path_compatible_full_name(),
                 "test_parity.py",
             ),
@@ -558,7 +635,9 @@ class FileWriter:
             FileType.autogenerated_template,
         ]
         for template_type in templates:
-            self.destination_files[template_type] = template_path(self.resource_name, template_type)
+            self.destination_files[template_type] = template_path(
+                self.resource_name, template_type, pro=self.pro
+            )
 
     def write(self, file_type: FileType, contents: str):
         file_destination = self.destination_files[file_type]
@@ -763,21 +842,14 @@ def generate(
     console = Console()
     console.rule(title=resource_type)
 
-    schema_provider = SchemaProvider(
-        zipfile_path=Path(__file__).parent.joinpath("CloudformationSchema.zip")
-    )
+    schema_provider = LiveSchemaProvider(boto3.client("cloudformation"))
 
     template_root = Path(__file__).parent.joinpath("templates")
     env = Environment(
         loader=FileSystemLoader(template_root),
     )
 
-    parts = resource_type.rpartition("::")
-    if parts[-1] == "*":
-        # generate all resource types for that service
-        matching_resources = [x for x in schema_provider.schemas.keys() if x.startswith(parts[0])]
-    else:
-        matching_resources = [resource_type]
+    matching_resources = schema_provider.available_schemas(resource_type)
 
     for matching_resource in matching_resources:
         console.rule(title=matching_resource)