dbt-platform-helper 12.5.1__py3-none-any.whl → 13.0.0__py3-none-any.whl

dbt_platform_helper/domain/pipelines.py

@@ -5,13 +5,14 @@ from shutil import rmtree
 
 from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
-from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
+from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.ecr import ECRProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
 from dbt_platform_helper.utils.application import get_application_name
-from dbt_platform_helper.utils.aws import get_account_details
-from dbt_platform_helper.utils.aws import get_public_repository_arn
-from dbt_platform_helper.utils.files import generate_override_files_from_template
 from dbt_platform_helper.utils.template import setup_templates
 from dbt_platform_helper.utils.versioning import (
     get_required_terraform_platform_modules_version,
@@ -22,192 +23,116 @@ class Pipelines:
     def __init__(
         self,
         config_provider: ConfigProvider,
-        echo: Callable[[str], str],
-        abort: Callable[[str], None],
+        terraform_manifest_provider: TerraformManifestProvider,
+        ecr_provider: ECRProvider,
         get_git_remote: Callable[[], str],
         get_codestar_arn: Callable[[str], str],
+        io: ClickIOProvider = ClickIOProvider(),
+        file_provider: FileProvider = FileProvider(),
     ):
         self.config_provider = config_provider
-        self.echo = echo
-        self.abort = abort
         self.get_git_remote = get_git_remote
         self.get_codestar_arn = get_codestar_arn
+        self.terraform_manifest_provider = terraform_manifest_provider
+        self.ecr_provider = ecr_provider
+        self.io = io
+        self.file_provider = file_provider
 
-    def generate(self, terraform_platform_modules_version, deploy_branch):
-        pipeline_config = self.config_provider.load_and_validate_platform_config()
+    def generate(self, cli_terraform_platform_modules_version: str, deploy_branch: str):
+        platform_config = self.config_provider.load_and_validate_platform_config()
 
-        has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
-        has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
+        has_codebase_pipelines = CODEBASE_PIPELINES_KEY in platform_config
+        has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in platform_config
 
         if not (has_codebase_pipelines or has_environment_pipelines):
-            self.echo("No pipelines defined: nothing to do.", err=True, fg="yellow")
+            self.io.warn("No pipelines defined: nothing to do.")
             return
 
-        platform_config_terraform_modules_default_version = pipeline_config.get(
+        platform_config_terraform_modules_default_version = platform_config.get(
             "default_versions", {}
         ).get("terraform-platform-modules", "")
 
-        templates = setup_templates()
         app_name = get_application_name()
 
         git_repo = self.get_git_remote()
         if not git_repo:
-            self.abort("The current directory is not a git repository")
+            self.io.abort_with_error("The current directory is not a git repository")
 
         codestar_connection_arn = self.get_codestar_arn(app_name)
         if codestar_connection_arn is None:
-            self.abort(f'There is no CodeStar Connection named "{app_name}" to use')
+            self.io.abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
 
         base_path = Path(".")
         copilot_pipelines_dir = base_path / f"copilot/pipelines"
 
         self._clean_pipeline_config(copilot_pipelines_dir)
 
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
         if has_environment_pipelines:
-            environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
+            environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
+            accounts = {
+                config.get("account")
+                for config in environment_pipelines.values()
+                if "account" in config
+            }
 
-            for config in environment_pipelines.values():
-                aws_account = config.get("account")
+            for account in accounts:
                 self._generate_terraform_environment_pipeline_manifest(
-                    pipeline_config["application"],
-                    aws_account,
+                    platform_config["application"],
+                    account,
                     terraform_platform_modules_version,
-                    platform_config_terraform_modules_default_version,
                     deploy_branch,
                 )
 
         if has_codebase_pipelines:
-            account_id, _ = get_account_details()
-
-            for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
-                self._generate_codebase_pipeline(
-                    account_id,
-                    app_name,
-                    codestar_connection_arn,
-                    git_repo,
-                    codebase,
-                    base_path,
-                    copilot_pipelines_dir,
-                    templates,
-                )
+            codebase_pipelines = platform_config[CODEBASE_PIPELINES_KEY]
+            ecrs_to_be_managed = {
+                codebase: f"{platform_config['application']}/{codebase}"
+                for codebase in codebase_pipelines.keys()
+            }
+            ecrs_already_provisioned = set(self.ecr_provider.get_ecr_repo_names())
+            ecrs_that_need_importing = {
+                codebase: repo
+                for codebase, repo in ecrs_to_be_managed.items()
+                if repo in ecrs_already_provisioned
+            }
 
-    def _clean_pipeline_config(self, pipelines_dir):
+            self.terraform_manifest_provider.generate_codebase_pipeline_config(
+                platform_config, terraform_platform_modules_version, ecrs_that_need_importing
+            )
+
+    def _clean_pipeline_config(self, pipelines_dir: Path):
         if pipelines_dir.exists():
-            self.echo("Deleting copilot/pipelines directory.")
+            self.io.info("Deleting copilot/pipelines directory.")
             rmtree(pipelines_dir)
 
-    def _generate_codebase_pipeline(
-        self,
-        account_id,
-        app_name,
-        codestar_connection_arn,
-        git_repo,
-        codebase,
-        base_path,
-        pipelines_dir,
-        templates,
-    ):
-        makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
-        environments = []
-        for pipelines in codebase["pipelines"]:
-            environments += pipelines[ENVIRONMENTS_KEY]
-
-        additional_ecr = codebase.get("additional_ecr_repository", None)
-        add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
-        additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
-
-        template_data = {
-            "account_id": account_id,
-            "app_name": app_name,
-            "deploy_repo": git_repo,
-            "codebase": codebase,
-            ENVIRONMENTS_KEY: environments,
-            "codestar_connection_arn": codestar_connection_arn,
-            "codestar_connection_id": codestar_connection_arn.split("/")[-1],
-            "additional_ecr_arn": additional_ecr_arn,
-        }
-
-        self._create_file_from_template(
-            base_path,
-            f"{codebase['name']}/manifest.yml",
-            pipelines_dir,
-            template_data,
-            templates,
-            "codebase/manifest.yml",
-        )
-
-        overrides_path = Path(__file__).parent.parent.joinpath(
-            "templates/pipelines/codebase/overrides"
-        )
-        generate_override_files_from_template(
-            base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
-        )
-
-    def _create_file_from_template(
-        self, base_path, file_name, pipelines_dir, template_data, templates, template_name=None
-    ):
-        contents = templates.get_template(
-            f"pipelines/{file_name if template_name is None else template_name}"
-        ).render(template_data)
-        message = FileProvider.mkfile(
-            base_path, pipelines_dir / file_name, contents, overwrite=True
-        )
-        self.echo(message)
-
     def _generate_terraform_environment_pipeline_manifest(
         self,
-        application,
-        aws_account,
-        cli_terraform_platform_modules_version,
-        platform_config_terraform_modules_default_version,
-        deploy_branch,
+        application: str,
+        aws_account: str,
+        terraform_platform_modules_version: str,
+        deploy_branch: str,
     ):
         env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
 
-        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
-            cli_terraform_platform_modules_version,
-            platform_config_terraform_modules_default_version,
-        )
-
         contents = env_pipeline_template.render(
             {
                 "application": application,
                 "aws_account": aws_account,
                 "terraform_platform_modules_version": terraform_platform_modules_version,
                 "deploy_branch": deploy_branch,
+                "terraform_version": SUPPORTED_TERRAFORM_VERSION,
+                "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,
             }
         )
 
         dir_path = f"terraform/environment-pipelines/{aws_account}"
         makedirs(dir_path, exist_ok=True)
 
-        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
-
-    def generate_terraform_codebase_pipeline_manifest(
-        self,
-        application,
-        aws_account,
-        cli_terraform_platform_modules_version,
-        platform_config_terraform_modules_default_version,
-        deploy_branch,
-    ):
-        env_pipeline_template = setup_templates().get_template("codebase-pipelines/main.tf")
-
-        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
-            cli_terraform_platform_modules_version,
-            platform_config_terraform_modules_default_version,
-        )
-
-        contents = env_pipeline_template.render(
-            {
-                "application": application,
-                "aws_account": aws_account,
-                "terraform_platform_modules_version": terraform_platform_modules_version,
-                "deploy_branch": deploy_branch,
-            }
+        self.io.info(
+            self.file_provider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True)
         )
-
-        dir_path = f"terraform/environment-pipelines/{aws_account}"
-        makedirs(dir_path, exist_ok=True)
-
-        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))

dbt_platform_helper/domain/terraform_environment.py

@@ -1,6 +1,8 @@
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
+from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.template import setup_templates
@@ -31,6 +33,8 @@ class PlatformTerraformManifestGenerator:
                 "environment": environment_name,
                 "config": environment_config,
                 "terraform_platform_modules_version": terraform_platform_modules_version,
+                "terraform_version": SUPPORTED_TERRAFORM_VERSION,
+                "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,
             }
         )
 
@@ -56,9 +60,9 @@ class TerraformEnvironment:
         self,
         config_provider,
         manifest_generator: PlatformTerraformManifestGenerator = None,
-        echo_fn=click.echo,
+        echo=click.echo,
     ):
-        self.echo = echo_fn
+        self.echo = echo
         self.config_provider = config_provider
         self.manifest_generator = manifest_generator or PlatformTerraformManifestGenerator(
             FileProvider()
@@ -69,7 +73,7 @@ class TerraformEnvironment:
 
         if environment_name not in config.get("environments").keys():
             raise EnvironmentNotFoundException(
-                f"Error: cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+                f"cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
             )
 
         manifest = self.manifest_generator.generate_manifest(

dbt_platform_helper/providers/aws.py

@@ -19,6 +19,11 @@ class ImageNotFoundException(AWSException):
         )
 
 
+class RepositoryNotFoundException(AWSException):
+    def __init__(self, repository: str):
+        super().__init__(f"""The ECR repository "{repository}" could not be found.""")
+
+
 class LogGroupNotFoundException(AWSException):
     def __init__(self, log_group_name: str):
         super().__init__(f"""No log group called "{log_group_name}".""")

dbt_platform_helper/providers/cloudformation.py

@@ -8,7 +8,8 @@ from dbt_platform_helper.platform_exception import PlatformException
 
 
 class CloudFormation:
-    def __init__(self, cloudformation_client, iam_client, ssm_client):
+    # TODO add handling for optional client parameters to handle case of calling boto API with None
+    def __init__(self, cloudformation_client, iam_client=None, ssm_client=None):
         self.cloudformation_client = cloudformation_client
         self.iam_client = iam_client
         self.ssm_client = ssm_client
@@ -125,6 +126,16 @@ class CloudFormation:
                     stack_name, f"Error while waiting for stack status: {str(err)}"
                 )
 
+    def get_cloudformation_exports_for_environment(self, environment_name):
+        exports = []
+
+        for page in self.cloudformation_client.get_paginator("list_exports").paginate():
+            for export in page["Exports"]:
+                if f"-{environment_name}-" in export["Name"]:
+                    exports.append(export)
+
+        return exports
+
 
 class CloudFormationException(PlatformException):
     def __init__(self, stack_name: str, current_status: str):

dbt_platform_helper/providers/config.py

@@ -1,35 +1,34 @@
 from copy import deepcopy
 from pathlib import Path
 
-import click
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
 from dbt_platform_helper.providers.yaml_file import FileNotFoundException
 from dbt_platform_helper.providers.yaml_file import FileProviderException
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
-from dbt_platform_helper.utils.messages import abort_with_error
 
 
 class ConfigProvider:
     def __init__(
         self,
-        config_validator: ConfigValidator,
+        config_validator: ConfigValidator = None,
         file_provider: YamlFileProvider = None,
-        echo=click.secho,
+        io: ClickIOProvider = None,
     ):
         self.config = {}
-        self.validator = config_validator
-        self.echo = echo
+        self.validator = config_validator or ConfigValidator()
+        self.io = io or ClickIOProvider()
         self.file_provider = file_provider or YamlFileProvider
 
     # TODO refactor so that apply_environment_defaults isn't set, discarded and set again
     def get_enriched_config(self):
         return self.apply_environment_defaults(self.load_and_validate_platform_config())
 
-    def validate_platform_config(self):
+    def _validate_platform_config(self):
         PlatformConfigSchema.schema().validate(self.config)
 
         # TODO= logically this isn't validation but loading + parsing, to move.
@@ -42,24 +41,23 @@ class ConfigProvider:
         try:
             self.config = self.file_provider.load(path)
         except FileNotFoundException as e:
-            abort_with_error(
+            self.io.abort_with_error(
                 f"{e} Please check it exists and you are in the root directory of your deployment project."
             )
         except FileProviderException as e:
-            abort_with_error(f"Error loading configuration from {path}: {e}")
+            self.io.abort_with_error(f"Error loading configuration from {path}: {e}")
 
         try:
-            self.validate_platform_config()
+            self._validate_platform_config()
         except SchemaError as e:
-            abort_with_error(f"Schema error in {path}. {e}")
+            self.io.abort_with_error(f"Schema error in {path}. {e}")
 
         return self.config
 
-    @staticmethod
     # TODO this general function should be moved out of ConfigProvider
-    def config_file_check(path=PLATFORM_CONFIG_FILE):
+    def config_file_check(self, path=PLATFORM_CONFIG_FILE):
         if not Path(path).exists():
-            abort_with_error(
+            self.io.abort_with_error(
                 f"`{path}` is missing. "
                 "Please check it exists and you are in the root directory of your deployment project."
             )

dbt_platform_helper/providers/ecr.py

@@ -0,0 +1,20 @@
+from boto3 import Session
+
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+
+
+class ECRProvider:
+    def __init__(self, session: Session = None):
+        self.session = session
+        self.client = None
+
+    def _get_client(self):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client("ecr")
+
+    def get_ecr_repo_names(self) -> list[str]:
+        out = []
+        for page in self._get_client().get_paginator("describe_repositories").paginate():
+            out.extend([repo["repositoryName"] for repo in page.get("repositories", {})])
+        return out

dbt_platform_helper/providers/files.py

@@ -8,7 +8,7 @@ class FileProvider:
         pass
 
     @staticmethod
-    def mkfile(base_path: str, file_path: str, contents, overwrite=False) -> str:
+    def mkfile(base_path: str, file_path: str, contents: str, overwrite=False) -> str:
         file_path = Path(file_path)
         file = Path(base_path).joinpath(file_path)
         file_exists = file.exists()

dbt_platform_helper/providers/io.py

@@ -0,0 +1,31 @@
+import click
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ClickIOProvider:
+    def warn(self, message: str):
+        click.secho(message, fg="magenta")
+
+    def error(self, message: str):
+        click.secho(f"Error: {message}", fg="red")
+
+    def info(self, message: str):
+        click.secho(message)
+
+    def input(self, message: str) -> str:
+        return click.prompt(message)
+
+    def confirm(self, message: str) -> bool:
+        try:
+            return click.confirm(message)
+        except click.Abort:
+            raise ClickIOProviderException(message + " [y/N]: Error: invalid input")
+
+    def abort_with_error(self, message: str):
+        click.secho(f"Error: {message}", err=True, fg="red")
+        exit(1)
+
+
+class ClickIOProviderException(PlatformException):
+    pass

dbt_platform_helper/providers/load_balancers.py

@@ -2,8 +2,11 @@ import boto3
 
 from dbt_platform_helper.platform_exception import PlatformException
 
+# TODO - a good candidate for a dataclass when this is refactored into a class.
+# Below methods should also really be refactored to not be so tightly coupled with eachother.
 
-def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
+
+def get_load_balancer_for_application(session: boto3.Session, app: str, env: str) -> str:
     lb_client = session.client("elbv2")
 
     describe_response = lb_client.describe_load_balancers()
@@ -23,8 +26,8 @@ def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
     return load_balancer_arn
 
 
-def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
-    load_balancer_arn = find_load_balancer(session, app, env)
+def get_https_listener_for_application(session: boto3.Session, app: str, env: str) -> str:
+    load_balancer_arn = get_load_balancer_for_application(session, app, env)
     lb_client = session.client("elbv2")
     listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
 
@@ -41,6 +44,22 @@ def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
     return listener_arn
 
 
+def get_https_certificate_for_application(session: boto3.Session, app: str, env: str) -> str:
+
+    listener_arn = get_https_listener_for_application(session, app, env)
+    cert_client = session.client("elbv2")
+    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
+        "Certificates"
+    ]
+
+    try:
+        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+    except StopIteration:
+        raise CertificateNotFoundException(env)
+
+    return certificate_arn
+
+
 class LoadBalancerException(PlatformException):
     pass
 
@@ -55,3 +74,10 @@ class ListenerNotFoundException(LoadBalancerException):
 
 class ListenerRuleNotFoundException(LoadBalancerException):
     pass
+
+
+class CertificateNotFoundException(PlatformException):
+    def __init__(self, environment_name: str):
+        super().__init__(
+            f"""No certificate found with domain name matching environment {environment_name}."."""
+        )

dbt_platform_helper/providers/platform_config_schema.py

@@ -16,9 +16,7 @@ class PlatformConfigSchema:
             {
                 # The following line is for the AWS Copilot version, will be removed under DBTP-1002
                 "application": str,
-                Optional("legacy_project", default=False): bool,
                 Optional("default_versions"): PlatformConfigSchema.__default_versions_schema(),
-                Optional("accounts"): list[str],
                 Optional("environments"): PlatformConfigSchema.__environments_schema(),
                 Optional("codebase_pipelines"): PlatformConfigSchema.__codebase_pipelines_schema(),
                 Optional(
@@ -80,7 +78,7 @@ class PlatformConfigSchema:
                 "cache": str,
                 "request": str,
             },
-            Optional("additional"): list[
+            Optional("additional"): [
                 {
                     "path": str,
                     "cache": str,
@@ -94,12 +92,12 @@ class PlatformConfigSchema:
             Optional("environments"): {
                 PlatformConfigSchema.__valid_environment_name(): Or(
                     {
-                        Optional("additional_address_list"): list,
-                        Optional("allowed_methods"): list,
-                        Optional("cached_methods"): list,
+                        Optional("additional_address_list"): [str],
+                        Optional("allowed_methods"): [str],
+                        Optional("cached_methods"): [str],
                         Optional("cdn_compress"): bool,
                         Optional("cdn_domains_list"): dict,
-                        Optional("cdn_geo_locations"): list,
+                        Optional("cdn_geo_locations"): [str],
                         Optional("cdn_geo_restriction_type"): str,
                         Optional("cdn_logging_bucket"): str,
                         Optional("cdn_logging_bucket_prefix"): str,
@@ -109,10 +107,10 @@ class PlatformConfigSchema:
                         Optional("enable_logging"): bool,
                         Optional("env_root"): str,
                         Optional("forwarded_values_forward"): str,
-                        Optional("forwarded_values_headers"): list,
+                        Optional("forwarded_values_headers"): [str],
                         Optional("forwarded_values_query_string"): bool,
                         Optional("origin_protocol_policy"): str,
-                        Optional("origin_ssl_protocols"): list,
+                        Optional("origin_ssl_protocols"): [str],
                         Optional("slack_alert_channel_alb_secret_rotation"): str,
                         Optional("viewer_certificate_minimum_protocol_version"): str,
                         Optional("viewer_certificate_ssl_support_method"): str,
@@ -127,14 +125,15 @@ class PlatformConfigSchema:
         }
 
     @staticmethod
-    def __codebase_pipelines_schema() -> list[dict]:
-        return [
-            {
-                "name": str,
+    def __codebase_pipelines_schema() -> dict:
+        return {
+            str: {
                 "repository": str,
+                Optional("slack_channel"): str,
+                Optional("requires_image_build"): bool,
                 Optional("additional_ecr_repository"): str,
                 Optional("deploy_repository_branch"): str,
-                "services": list[str],
+                "services": [{str: [str]}],
                 "pipelines": [
                     Or(
                         {
@@ -160,7 +159,7 @@ class PlatformConfigSchema:
                     ),
                 ],
             },
-        ]
+        }
 
     @staticmethod
     def __default_versions_schema() -> dict:
@@ -443,14 +442,17 @@ class PlatformConfigSchema:
                 error=f"{key} must contain a valid ARN for an S3 bucket",
             )
 
+        _single_import_config = {
+            Optional("source_kms_key_arn"): PlatformConfigSchema.__valid_kms_key_arn(
+                "source_kms_key_arn"
+            ),
+            "source_bucket_arn": _valid_s3_bucket_arn("source_bucket_arn"),
+            "worker_role_arn": PlatformConfigSchema.__valid_iam_role_arn("worker_role_arn"),
+        }
+
         _valid_s3_data_migration = {
-            "import": {
-                Optional("source_kms_key_arn"): PlatformConfigSchema.__valid_kms_key_arn(
-                    "source_kms_key_arn"
-                ),
-                "source_bucket_arn": _valid_s3_bucket_arn("source_bucket_arn"),
-                "worker_role_arn": PlatformConfigSchema.__valid_iam_role_arn("worker_role_arn"),
-            },
+            Optional("import"): _single_import_config,
+            Optional("import_sources"): [_single_import_config],
         }
 
         _valid_s3_bucket_retention_policy = Or(