dbt-platform-helper 12.5.1__py3-none-any.whl → 12.6.0__py3-none-any.whl

dbt_platform_helper/COMMANDS.md

@@ -1,40 +1,40 @@
 # Commands Reference
 
 - [platform-helper](#platform-helper)
-- [platform-helper application](#platform-helper-application)
-- [platform-helper application container-stats](#platform-helper-application-container-stats)
-- [platform-helper application task-stats](#platform-helper-application-task-stats)
-- [platform-helper codebase](#platform-helper-codebase)
-- [platform-helper codebase prepare](#platform-helper-codebase-prepare)
-- [platform-helper codebase list](#platform-helper-codebase-list)
-- [platform-helper codebase build](#platform-helper-codebase-build)
-- [platform-helper codebase deploy](#platform-helper-codebase-deploy)
-- [platform-helper conduit](#platform-helper-conduit)
-- [platform-helper config](#platform-helper-config)
-- [platform-helper config validate](#platform-helper-config-validate)
-- [platform-helper config aws](#platform-helper-config-aws)
-- [platform-helper copilot](#platform-helper-copilot)
-- [platform-helper copilot make-addons](#platform-helper-copilot-make-addons)
-- [platform-helper environment](#platform-helper-environment)
-- [platform-helper environment offline](#platform-helper-environment-offline)
-- [platform-helper environment online](#platform-helper-environment-online)
-- [platform-helper environment generate](#platform-helper-environment-generate)
-- [platform-helper environment generate-terraform](#platform-helper-environment-generate-terraform)
-- [platform-helper generate](#platform-helper-generate)
-- [platform-helper pipeline](#platform-helper-pipeline)
-- [platform-helper pipeline generate](#platform-helper-pipeline-generate)
-- [platform-helper secrets](#platform-helper-secrets)
-- [platform-helper secrets copy](#platform-helper-secrets-copy)
-- [platform-helper secrets list](#platform-helper-secrets-list)
-- [platform-helper notify](#platform-helper-notify)
-- [platform-helper notify environment-progress](#platform-helper-notify-environment-progress)
-- [platform-helper notify add-comment](#platform-helper-notify-add-comment)
-- [platform-helper database](#platform-helper-database)
-- [platform-helper database dump](#platform-helper-database-dump)
-- [platform-helper database load](#platform-helper-database-load)
-- [platform-helper database copy](#platform-helper-database-copy)
-- [platform-helper version](#platform-helper-version)
-- [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
+    - [platform-helper application](#platform-helper-application)
+        - [platform-helper application container-stats](#platform-helper-application-container-stats)
+        - [platform-helper application task-stats](#platform-helper-application-task-stats)
+    - [platform-helper codebase](#platform-helper-codebase)
+        - [platform-helper codebase prepare](#platform-helper-codebase-prepare)
+        - [platform-helper codebase list](#platform-helper-codebase-list)
+        - [platform-helper codebase build](#platform-helper-codebase-build)
+        - [platform-helper codebase deploy](#platform-helper-codebase-deploy)
+    - [platform-helper conduit](#platform-helper-conduit)
+    - [platform-helper config](#platform-helper-config)
+        - [platform-helper config validate](#platform-helper-config-validate)
+        - [platform-helper config aws](#platform-helper-config-aws)
+    - [platform-helper copilot](#platform-helper-copilot)
+        - [platform-helper copilot make-addons](#platform-helper-copilot-make-addons)
+    - [platform-helper environment](#platform-helper-environment)
+        - [platform-helper environment offline](#platform-helper-environment-offline)
+        - [platform-helper environment online](#platform-helper-environment-online)
+        - [platform-helper environment generate](#platform-helper-environment-generate)
+        - [platform-helper environment generate-terraform](#platform-helper-environment-generate-terraform)
+    - [platform-helper generate](#platform-helper-generate)
+    - [platform-helper pipeline](#platform-helper-pipeline)
+        - [platform-helper pipeline generate](#platform-helper-pipeline-generate)
+    - [platform-helper secrets](#platform-helper-secrets)
+        - [platform-helper secrets copy](#platform-helper-secrets-copy)
+        - [platform-helper secrets list](#platform-helper-secrets-list)
+    - [platform-helper notify](#platform-helper-notify)
+        - [platform-helper notify environment-progress](#platform-helper-notify-environment-progress)
+        - [platform-helper notify add-comment](#platform-helper-notify-add-comment)
+    - [platform-helper database](#platform-helper-database)
+        - [platform-helper database dump](#platform-helper-database-dump)
+        - [platform-helper database load](#platform-helper-database-load)
+        - [platform-helper database copy](#platform-helper-database-copy)
+    - [platform-helper version](#platform-helper-version)
+        - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
 
 # platform-helper
 
@@ -457,6 +457,9 @@ platform-helper environment online --app <application> --env <environment>
 
 [↩ Parent](#platform-helper-environment)
 
+    Gathers various IDs and ARNs from AWS and generates the AWS Copilot
+    environment manifest at copilot/environments/<environment>/manifest.yml.
+
 ## Usage
 
 ```
@@ -467,7 +470,7 @@ platform-helper environment generate --name <name>
 
 - `--name
 -n <text>`
-
+  - The name of the environment to generate a copilot manifest for.
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 

dbt_platform_helper/commands/codebase.py

@@ -2,6 +2,7 @@ import click
 
 from dbt_platform_helper.domain.codebase import Codebase
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
@@ -20,8 +21,7 @@ def prepare():
     try:
         Codebase().prepare()
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @codebase.command()
@@ -37,8 +37,7 @@ def list(app, with_images):
     try:
         Codebase().list(app, with_images)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @codebase.command()
@@ -54,8 +53,7 @@ def build(app, codebase, commit):
     try:
         Codebase().build(app, codebase, commit)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @codebase.command()
@@ -71,5 +69,4 @@ def deploy(app, env, codebase, commit):
     try:
         Codebase().deploy(app, env, codebase, commit)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))

dbt_platform_helper/commands/conduit.py

@@ -4,6 +4,7 @@ from dbt_platform_helper.domain.conduit import Conduit
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.cloudformation import CloudFormation
 from dbt_platform_helper.providers.ecs import ECS
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.secrets import Secrets
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.click import ClickDocOptCommand
@@ -54,5 +55,4 @@ def conduit(addon_name: str, app: str, env: str, access: str):
             env, addon_name, access
         )
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))

dbt_platform_helper/commands/config.py

@@ -98,7 +98,7 @@ def deployment():
 
     recommendations = {}
 
-    ConfigProvider.config_file_check()
+    ConfigProvider().config_file_check()
 
     for template_file in addons_templates:
         generated_with_version = maybe

dbt_platform_helper/commands/environment.py

@@ -1,14 +1,17 @@
 import click
-from schema import SchemaError
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
-from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
 from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.cloudformation import CloudFormation
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.vpc import VpcProvider
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
@@ -36,11 +39,12 @@ def environment():
 @click.option("--vpc", type=str)
 def offline(app, env, svc, template, vpc):
     """Take load-balanced web services offline with a maintenance page."""
+
     try:
-        MaintenancePage().activate(app, env, svc, template, vpc)
+        application = load_application(app)
+        MaintenancePage(application).activate(env, svc, template, vpc)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @environment.command()
@@ -48,26 +52,37 @@ def offline(app, env, svc, template, vpc):
 @click.option("--env", type=str, required=True)
 def online(app, env):
     """Remove a maintenance page from an environment."""
+
     try:
-        MaintenancePage().deactivate(app, env)
+        application = load_application(app)
+        MaintenancePage(application).deactivate(env)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @environment.command()
-@click.option("--name", "-n", required=True)
+@click.option(
+    "--name",
+    "-n",
+    required=True,
+    help="The name of the environment to generate a copilot manifest for.",
+)
 def generate(name):
+    """Gathers various IDs and ARNs from AWS and generates the AWS Copilot
+    environment manifest at copilot/environments/<environment>/manifest.yml."""
+
+    click_io = ClickIOProvider()
     try:
+        session = get_aws_session_or_abort()
         config_provider = ConfigProvider(ConfigValidator())
-        CopilotEnvironment(config_provider).generate(name)
-    # TODO this exception will never be caught as the config provider catches schema errors and aborts
-    except SchemaError as ex:
-        click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
-        raise click.Abort
+        vpc_provider = VpcProvider(session)
+        cloudformation_provider = CloudFormation(session.client("cloudformation"))
+
+        CopilotEnvironment(
+            config_provider, vpc_provider, cloudformation_provider, session
+        ).generate(name)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        click_io.abort_with_error(str(err))
 
 
 @environment.command(help="Generate terraform manifest for the specified environment.")
@@ -84,5 +99,4 @@ def generate_terraform(name, terraform_platform_modules_version):
         config_provider = ConfigProvider(ConfigValidator())
         TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
     except PlatformException as err:
-        click.secho(str(err), fg="red")
-        raise click.Abort
+        ClickIOProvider().abort_with_error(str(err))

dbt_platform_helper/commands/pipeline.py

@@ -9,7 +9,6 @@ from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.aws import get_codestar_connection_arn
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.git import git_remote
-from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
@@ -52,8 +51,6 @@ def generate(terraform_platform_modules_version, deploy_branch):
     """
     pipelines = Pipelines(
         ConfigProvider(ConfigValidator()),
-        click.secho,
-        abort_with_error,
         git_remote,
         get_codestar_connection_arn,
     )

dbt_platform_helper/domain/codebase.py

@@ -4,13 +4,13 @@ import subprocess
 from collections.abc import Callable
 from pathlib import Path
 
-import click
 import requests
 import yaml
 from boto3 import Session
 
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationException
 from dbt_platform_helper.utils.application import load_application
@@ -27,9 +27,7 @@ from dbt_platform_helper.utils.template import setup_templates
 class Codebase:
     def __init__(
         self,
-        input: Callable[[str], str] = click.prompt,
-        echo: Callable[[str], str] = click.secho,
-        confirm: Callable[[str], bool] = click.confirm,
+        io: ClickIOProvider = ClickIOProvider(),
         load_application: Callable[[str], Application] = load_application,
         get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
         check_codebase_exists: Callable[[str], str] = check_codebase_exists,
@@ -40,9 +38,7 @@ class Codebase:
         check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
         run_subprocess: Callable[[str], str] = subprocess.run,
     ):
-        self.input = input
-        self.echo = echo
-        self.confirm = confirm
+        self.io = io
         self.load_application = load_application
         self.get_aws_session_or_abort = get_aws_session_or_abort
         self.check_codebase_exists = check_codebase_exists
@@ -91,7 +87,7 @@ class Codebase:
         config_contents = templates.get_template(f".copilot/config.yml").render(
             repository=repository, builder_version=builder_version
         )
-        self.echo(
+        self.io.info(
             FileProvider.mkfile(
                 Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
             )
@@ -100,14 +96,14 @@ class Codebase:
         image_build_run_file = Path(".copilot/image_build_run.sh")
         image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
 
-        self.echo(
+        self.io.info(
             FileProvider.mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True)
         )
 
         for phase in ["build", "install", "post_build", "pre_build"]:
             phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
 
-            self.echo(
+            self.io.info(
                 FileProvider.mkfile(
                     Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True
                 )
@@ -122,7 +118,6 @@ class Codebase:
 
         codebuild_client = session.client("codebuild")
         build_url = self.__start_build_with_confirmation(
-            self.confirm,
             codebuild_client,
             self.get_build_url_from_arn,
             f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
@@ -134,7 +129,7 @@ class Codebase:
         )
 
         if build_url:
-            return self.echo(
+            return self.io.info(
                 f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
             )
 
@@ -154,7 +149,6 @@ class Codebase:
 
         codebuild_client = session.client("codebuild")
         build_url = self.__start_build_with_confirmation(
-            self.confirm,
             codebuild_client,
             self.get_build_url_from_arn,
             f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
@@ -170,7 +164,7 @@ class Codebase:
         )
 
         if build_url:
-            return self.echo(
+            return self.io.info(
                 "Your deployment has been triggered. Check your build progress in the AWS Console: "
                 f"{build_url}",
             )
@@ -185,19 +179,19 @@ class Codebase:
         ecr_client = session.client("ecr")
         codebases = self.__get_codebases(application, ssm_client)
 
-        self.echo("The following codebases are available:")
+        self.io.info("The following codebases are available:")
 
         for codebase in codebases:
-            self.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
+            self.io.info(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
             if with_images:
                 self.list_latest_images(
                     ecr_client,
                     f"{application.name}/{codebase['name']}",
                     codebase["repository"],
-                    self.echo,
+                    self.io.info,
                 )
 
-        self.echo("")
+        self.io.info("")
 
     def __get_codebases(self, application, ssm_client):
         parameters = ssm_client.get_parameters_by_path(
@@ -213,13 +207,12 @@ class Codebase:
 
     def __start_build_with_confirmation(
         self,
-        confirm,
         codebuild_client,
         get_build_url_from_arn,
         confirmation_message,
         build_options,
     ):
-        if confirm(confirmation_message):
+        if self.io.confirm(confirmation_message):
             build_arn = self.start_build_extraction(codebuild_client, build_options)
             return get_build_url_from_arn(build_arn)
         return None

dbt_platform_helper/domain/conduit.py

@@ -1,12 +1,10 @@
 import subprocess
-from collections.abc import Callable
-
-import click
 
 from dbt_platform_helper.providers.cloudformation import CloudFormation
 from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
 from dbt_platform_helper.providers.copilot import create_addon_client_task
 from dbt_platform_helper.providers.ecs import ECS
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.secrets import Secrets
 from dbt_platform_helper.utils.application import Application
 
@@ -18,7 +16,7 @@ class Conduit:
         secrets_provider: Secrets,
         cloudformation_provider: CloudFormation,
         ecs_provider: ECS,
-        echo: Callable[[str], str] = click.secho,
+        io: ClickIOProvider = ClickIOProvider(),
         subprocess: subprocess = subprocess,
         connect_to_addon_client_task=connect_to_addon_client_task,
         create_addon_client_task=create_addon_client_task,
@@ -29,7 +27,7 @@ class Conduit:
         self.cloudformation_provider = cloudformation_provider
         self.ecs_provider = ecs_provider
         self.subprocess = subprocess
-        self.echo = echo
+        self.io = io
         self.connect_to_addon_client_task = connect_to_addon_client_task
         self.create_addon_client_task = create_addon_client_task
 
@@ -39,10 +37,10 @@ class Conduit:
             addon_name, access
         )
 
-        self.echo(f"Checking if a conduit task is already running for {addon_type}")
+        self.io.info(f"Checking if a conduit task is already running for {addon_type}")
         task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
         if not task_arns:
-            self.echo("Creating conduit task")
+            self.io.info("Creating conduit task")
             self.create_addon_client_task(
                 clients["iam"],
                 clients["ssm"],
@@ -55,7 +53,7 @@ class Conduit:
                 access,
             )
 
-            self.echo("Updating conduit task")
+            self.io.info("Updating conduit task")
             self._update_stack_resources(
                 self.application.name,
                 env,
@@ -69,13 +67,13 @@ class Conduit:
             task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
 
         else:
-            self.echo("Conduit task already running")
+            self.io.info("Conduit task already running")
 
-        self.echo(f"Checking if exec is available for conduit task...")
+        self.io.info(f"Checking if exec is available for conduit task...")
 
         self.ecs_provider.ecs_exec_is_available(cluster_arn, task_arns)
 
-        self.echo("Connecting to conduit task")
+        self.io.info("Connecting to conduit task")
         self.connect_to_addon_client_task(
             clients["ecs"], self.subprocess, self.application.name, env, cluster_arn, task_name
         )
@@ -115,7 +113,7 @@ class Conduit:
             parameter_name,
             access,
         )
-        self.echo("Waiting for conduit task update to complete...")
+        self.io.info("Waiting for conduit task update to complete...")
         self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
             "stack_update_complete", stack_name
         )

dbt_platform_helper/domain/config_validator.py

@@ -1,11 +1,11 @@
 from typing import Callable
 
 import boto3
-import click
 
 from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENTS_KEY
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.opensearch import OpensearchProvider
 from dbt_platform_helper.providers.redis import RedisProvider
 from dbt_platform_helper.utils.messages import abort_with_error
@@ -13,7 +13,9 @@ from dbt_platform_helper.utils.messages import abort_with_error
 
 class ConfigValidator:
 
-    def __init__(self, validations: Callable[[dict], None] = None):
+    def __init__(
+        self, validations: Callable[[dict], None] = None, io: ClickIOProvider = ClickIOProvider()
+    ):
         self.validations = validations or [
             self.validate_supported_redis_versions,
             self.validate_supported_opensearch_versions,
@@ -21,7 +23,9 @@ class ConfigValidator:
             self.validate_codebase_pipelines,
             self.validate_environment_pipelines_triggers,
             self.validate_database_copy_section,
+            self.validate_database_migration_input_sources,
         ]
+        self.io = io
 
     def run_validations(self, config: dict):
         for validation in self.validations:
@@ -48,9 +52,8 @@ class ConfigValidator:
             environments = extension.get("environments", {})
 
             if not isinstance(environments, dict):
-                click.secho(
-                    f"Error: {extension_type} extension definition is invalid type, expected dictionary",
-                    fg="red",
+                self.io.error(
+                    f"{extension_type} extension definition is invalid type, expected dictionary",
                 )
                 continue
             for environment, env_config in environments.items():
@@ -64,9 +67,8 @@ class ConfigValidator:
                     )
 
         for version_failure in extensions_with_invalid_version:
-            click.secho(
+            self.io.error(
                 f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
-                fg="red",
             )
 
     def validate_supported_redis_versions(self, config):
@@ -240,3 +242,34 @@ class ConfigValidator:
 
         if errors:
             abort_with_error("\n".join(errors))
+
+    def validate_database_migration_input_sources(self, config):
+        extensions = config.get("extensions", {})
+        if not extensions:
+            return
+
+        s3_extensions = {
+            key: ext for key, ext in extensions.items() if ext.get("type", None) == "s3"
+        }
+
+        if not s3_extensions:
+            return
+
+        errors = []
+
+        for extension_name, extension in s3_extensions.items():
+            for env, env_config in extension.get("environments", {}).items():
+                if "data_migration" not in env_config:
+                    continue
+                data_migration = env_config.get("data_migration", {})
+                if "import" in data_migration and "import_sources" in data_migration:
+                    errors.append(
+                        f"Error in '{extension_name}.environments.{env}.data_migration': only the 'import_sources' property is required - 'import' is deprecated."
+                    )
+                if "import" not in data_migration and "import_sources" not in data_migration:
+                    errors.append(
+                        f"Error in '{extension_name}.environments.{env}.data_migration': 'import_sources' property is missing."
+                    )
+
+        if errors:
+            abort_with_error("\n".join(errors))