dbt-platform-helper 12.5.0__py3-none-any.whl → 12.5.1__py3-none-any.whl

dbt_platform_helper/COMMANDS.md

@@ -460,13 +460,11 @@ platform-helper environment online --app <application> --env <environment>
 ## Usage
 
 ```
-platform-helper environment generate --name <name> [--vpc-name <vpc_name>] 
+platform-helper environment generate --name <name> 
 ```
 
 ## Options
 
-- `--vpc-name <text>`
-
 - `--name
 -n <text>`
 

dbt_platform_helper/commands/copilot.py

@@ -13,12 +13,12 @@ from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.files import generate_override_files
-from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
 from dbt_platform_helper.utils.template import camel_case
@@ -379,7 +379,9 @@ def _generate_service_addons(
 
             (output_dir / service_path).mkdir(parents=True, exist_ok=True)
             click.echo(
-                mkfile(output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True)
+                FileProvider.mkfile(
+                    output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
+                )
             )
 
 

dbt_platform_helper/commands/environment.py

@@ -5,7 +5,7 @@ from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VER
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
-from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.config import ConfigProvider
@@ -37,7 +37,7 @@ def environment():
 def offline(app, env, svc, template, vpc):
     """Take load-balanced web services offline with a maintenance page."""
     try:
-        MaintenancePageProvider().activate(app, env, svc, template, vpc)
+        MaintenancePage().activate(app, env, svc, template, vpc)
     except PlatformException as err:
         click.secho(str(err), fg="red")
         raise click.Abort
@@ -49,28 +49,25 @@ def offline(app, env, svc, template, vpc):
 def online(app, env):
     """Remove a maintenance page from an environment."""
     try:
-        MaintenancePageProvider().deactivate(app, env)
+        MaintenancePage().deactivate(app, env)
     except PlatformException as err:
         click.secho(str(err), fg="red")
         raise click.Abort
 
 
 @environment.command()
-@click.option("--vpc-name", hidden=True)
 @click.option("--name", "-n", required=True)
-def generate(name, vpc_name):
-    if vpc_name:
-        click.secho(
-            f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
-            fg="red",
-        )
-        raise click.Abort
+def generate(name):
     try:
         config_provider = ConfigProvider(ConfigValidator())
         CopilotEnvironment(config_provider).generate(name)
+    # TODO this exception will never be caught as the config provider catches schema errors and aborts
     except SchemaError as ex:
         click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
         raise click.Abort
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
 
 
 @environment.command(help="Generate terraform manifest for the specified environment.")
@@ -82,5 +79,10 @@ def generate(name, vpc_name):
     help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
 )
 def generate_terraform(name, terraform_platform_modules_version):
-    config_provider = ConfigProvider(ConfigValidator())
-    TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
+
+    try:
+        config_provider = ConfigProvider(ConfigValidator())
+        TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort

dbt_platform_helper/commands/pipeline.py

@@ -1,31 +1,19 @@
 #!/usr/bin/env python
-from os import makedirs
-from pathlib import Path
-from shutil import rmtree
 
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.pipelines import Pipelines
 from dbt_platform_helper.providers.config import ConfigProvider
-from dbt_platform_helper.utils.application import get_application_name
-from dbt_platform_helper.utils.aws import get_account_details
 from dbt_platform_helper.utils.aws import get_codestar_connection_arn
-from dbt_platform_helper.utils.aws import get_public_repository_arn
 from dbt_platform_helper.utils.click import ClickDocOptGroup
-from dbt_platform_helper.utils.files import generate_override_files_from_template
-from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.git import git_remote
 from dbt_platform_helper.utils.messages import abort_with_error
-from dbt_platform_helper.utils.template import setup_templates
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
 
-CODEBASE_PIPELINES_KEY = "codebase_pipelines"
-ENVIRONMENTS_KEY = "environments"
-ENVIRONMENT_PIPELINES_KEY = "environment_pipelines"
-
 
 @click.group(chain=True, cls=ClickDocOptGroup)
 def pipeline():
@@ -62,161 +50,11 @@ def generate(terraform_platform_modules_version, deploy_branch):
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
     """
-    config_provider = ConfigProvider(ConfigValidator())
-    pipeline_config = config_provider.load_and_validate_platform_config()
-
-    has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
-    has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
-
-    if not (has_codebase_pipelines or has_environment_pipelines):
-        click.secho("No pipelines defined: nothing to do.", err=True, fg="yellow")
-        return
-
-    platform_config_terraform_modules_default_version = pipeline_config.get(
-        "default_versions", {}
-    ).get("terraform-platform-modules", "")
-
-    templates = setup_templates()
-    app_name = get_application_name()
-
-    git_repo = git_remote()
-    if not git_repo:
-        abort_with_error("The current directory is not a git repository")
-
-    codestar_connection_arn = get_codestar_connection_arn(app_name)
-    if codestar_connection_arn is None:
-        abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
-
-    base_path = Path(".")
-    copilot_pipelines_dir = base_path / f"copilot/pipelines"
-
-    _clean_pipeline_config(copilot_pipelines_dir)
-
-    if has_environment_pipelines:
-        environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
-
-        for config in environment_pipelines.values():
-            aws_account = config.get("account")
-            _generate_terraform_environment_pipeline_manifest(
-                pipeline_config["application"],
-                aws_account,
-                terraform_platform_modules_version,
-                platform_config_terraform_modules_default_version,
-                deploy_branch,
-            )
-
-    if has_codebase_pipelines:
-        account_id, _ = get_account_details()
-
-        for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
-            _generate_codebase_pipeline(
-                account_id,
-                app_name,
-                codestar_connection_arn,
-                git_repo,
-                codebase,
-                base_path,
-                copilot_pipelines_dir,
-                templates,
-            )
-
-
-def _clean_pipeline_config(pipelines_dir):
-    if pipelines_dir.exists():
-        click.echo("Deleting copilot/pipelines directory.")
-        rmtree(pipelines_dir)
-
-
-def _generate_codebase_pipeline(
-    account_id,
-    app_name,
-    codestar_connection_arn,
-    git_repo,
-    codebase,
-    base_path,
-    pipelines_dir,
-    templates,
-):
-    makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
-    environments = []
-    for pipelines in codebase["pipelines"]:
-        environments += pipelines[ENVIRONMENTS_KEY]
-
-    additional_ecr = codebase.get("additional_ecr_repository", None)
-    add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
-    additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
-
-    template_data = {
-        "account_id": account_id,
-        "app_name": app_name,
-        "deploy_repo": git_repo,
-        "codebase": codebase,
-        ENVIRONMENTS_KEY: environments,
-        "codestar_connection_arn": codestar_connection_arn,
-        "codestar_connection_id": codestar_connection_arn.split("/")[-1],
-        "additional_ecr_arn": additional_ecr_arn,
-    }
-
-    _create_file_from_template(
-        base_path,
-        f"{codebase['name']}/manifest.yml",
-        pipelines_dir,
-        template_data,
-        templates,
-        "codebase/manifest.yml",
+    pipelines = Pipelines(
+        ConfigProvider(ConfigValidator()),
+        click.secho,
+        abort_with_error,
+        git_remote,
+        get_codestar_connection_arn,
     )
-
-    overrides_path = Path(__file__).parent.parent.joinpath("templates/pipelines/codebase/overrides")
-    generate_override_files_from_template(
-        base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
-    )
-
-
-def _create_file_from_template(
-    base_path, file_name, pipelines_dir, template_data, templates, template_name=None
-):
-    contents = templates.get_template(
-        f"pipelines/{file_name if template_name is None else template_name}"
-    ).render(template_data)
-    message = mkfile(base_path, pipelines_dir / file_name, contents, overwrite=True)
-    click.echo(message)
-
-
-def _generate_terraform_environment_pipeline_manifest(
-    application,
-    aws_account,
-    cli_terraform_platform_modules_version,
-    platform_config_terraform_modules_default_version,
-    deploy_branch,
-):
-    env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
-
-    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
-        cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
-    )
-
-    contents = env_pipeline_template.render(
-        {
-            "application": application,
-            "aws_account": aws_account,
-            "terraform_platform_modules_version": terraform_platform_modules_version,
-            "deploy_branch": deploy_branch,
-        }
-    )
-
-    dir_path = f"terraform/environment-pipelines/{aws_account}"
-    makedirs(dir_path, exist_ok=True)
-
-    click.echo(mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
-
-
-def _determine_terraform_platform_modules_version(
-    cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
-):
-
-    version_preference_order = [
-        cli_terraform_platform_modules_version,
-        platform_config_terraform_modules_default_version,
-        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
-    ]
-    return [version for version in version_preference_order if version][0]
+    pipelines.generate(terraform_platform_modules_version, deploy_branch)

dbt_platform_helper/constants.py

@@ -8,6 +8,7 @@ DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
 # Keys
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"
 ENVIRONMENTS_KEY = "environments"
+ENVIRONMENT_PIPELINES_KEY = "environment_pipelines"
 
 # Conduit
 CONDUIT_ADDON_TYPES = [

dbt_platform_helper/domain/codebase.py

@@ -10,6 +10,7 @@ import yaml
 from boto3 import Session
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationException
 from dbt_platform_helper.utils.application import load_application
@@ -19,7 +20,6 @@ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_build_url_from_arn
 from dbt_platform_helper.utils.aws import list_latest_images
 from dbt_platform_helper.utils.aws import start_build_extraction
-from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.git import check_if_commit_exists
 from dbt_platform_helper.utils.template import setup_templates
 
@@ -92,7 +92,7 @@ class Codebase:
             repository=repository, builder_version=builder_version
         )
         self.echo(
-            mkfile(
+            FileProvider.mkfile(
                 Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
             )
         )
@@ -100,13 +100,17 @@ class Codebase:
         image_build_run_file = Path(".copilot/image_build_run.sh")
         image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
 
-        self.echo(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
+        self.echo(
+            FileProvider.mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True)
+        )
 
         for phase in ["build", "install", "post_build", "pre_build"]:
             phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
 
             self.echo(
-                mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
+                FileProvider.mkfile(
+                    Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True
+                )
             )
 
     def build(self, app: str, codebase: str, commit: str):

dbt_platform_helper/domain/copilot_environment.py

@@ -5,9 +5,9 @@ import boto3
 import click
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.load_balancers import find_https_listener
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
-from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.template import S3_CROSS_ACCOUNT_POLICY
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
@@ -111,7 +111,7 @@ def _generate_copilot_environment_manifests(
         }
     )
     click.echo(
-        mkfile(
+        FileProvider.mkfile(
             ".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
         )
     )
@@ -155,7 +155,7 @@ class CopilotEnvironment:
 
 
 class CopilotTemplating:
-    def __init__(self, mkfile_fn=mkfile):
+    def __init__(self, mkfile_fn=FileProvider.mkfile):
         self.mkfile_fn = mkfile_fn
 
     def generate_cross_account_s3_policies(self, environments: dict, extensions):

dbt_platform_helper/domain/database_copy.py

@@ -8,15 +8,15 @@ from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
-from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.providers.aws import AWSException
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.vpc import Vpc
+from dbt_platform_helper.providers.vpc import VpcProvider
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationNotFoundException
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import Vpc
 from dbt_platform_helper.utils.aws import get_connection_string
-from dbt_platform_helper.utils.aws import get_vpc_info_by_name
 from dbt_platform_helper.utils.aws import wait_for_log_group_to_exist
 from dbt_platform_helper.utils.messages import abort_with_error
 
@@ -28,13 +28,14 @@ class DatabaseCopy:
         database: str,
         auto_approve: bool = False,
         load_application: Callable[[str], Application] = load_application,
-        vpc_config: Callable[[Session, str, str, str], Vpc] = get_vpc_info_by_name,
+        # TODO We inject VpcProvider as a callable here so that it can be instantiated within the method.  To be improved
+        vpc_provider: Callable[[Session], VpcProvider] = VpcProvider,
         db_connection_string: Callable[
             [Session, str, str, str, Callable], str
         ] = get_connection_string,
         maintenance_page_provider: Callable[
             [str, str, list[str], str, str], None
-        ] = MaintenancePageProvider(),
+        ] = MaintenancePage(),
         input: Callable[[str], str] = click.prompt,
         echo: Callable[[str], str] = click.secho,
         abort: Callable[[str], None] = abort_with_error,
@@ -43,7 +44,7 @@ class DatabaseCopy:
         self.app = app
         self.database = database
         self.auto_approve = auto_approve
-        self.vpc_config = vpc_config
+        self.vpc_provider = vpc_provider
         self.db_connection_string = db_connection_string
         self.maintenance_page_provider = maintenance_page_provider
         self.input = input
@@ -76,7 +77,8 @@ class DatabaseCopy:
         env_session = environment.session
 
         try:
-            vpc_config = self.vpc_config(env_session, self.app, env, vpc_name)
+            vpc_provider = self.vpc_provider(env_session)
+            vpc_config = vpc_provider.get_vpc_info_by_name(self.app, env, vpc_name)
         except AWSException as ex:
             self.abort(str(ex))
 

dbt_platform_helper/domain/maintenance_page.py

@@ -3,38 +3,62 @@ import random
 import re
 import string
 from pathlib import Path
+from typing import Callable
 from typing import List
 from typing import Union
 
 import boto3
 import click
 
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
 from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
 from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
 from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import Environment
 from dbt_platform_helper.utils.application import Service
 from dbt_platform_helper.utils.application import load_application
 
 
-class MaintenancePageProvider:
-    def activate(self, app, env, svc, template, vpc):
-        application = load_application(app)
-        application_environment = get_app_environment(app, env)
+class MaintenancePageException(PlatformException):
+    pass
+
+
+class LoadBalancedWebServiceNotFoundException(MaintenancePageException):
+    pass
+
+
+class MaintenancePage:
+    def __init__(
+        self,
+        user_prompt_callback: Callable[[str], bool] = click.confirm,
+        echo: Callable[[str], str] = click.secho,
+    ):
+        self.user_prompt_callback = user_prompt_callback
+        self.echo = echo
 
+    def _get_deployed_load_balanced_web_services(self, app: Application, svc: str):
         if "*" in svc:
-            services = [
-                s for s in application.services.values() if s.kind == "Load Balanced Web Service"
-            ]
+            services = [s for s in app.services.values() if s.kind == "Load Balanced Web Service"]
         else:
-            all_services = [get_app_service(app, s) for s in list(svc)]
+            all_services = [get_app_service(app.name, s) for s in list(svc)]
             services = [s for s in all_services if s.kind == "Load Balanced Web Service"]
-
         if not services:
-            click.secho(f"No services deployed yet to {app} environment {env}", fg="red")
+            raise LoadBalancedWebServiceNotFoundException
+        return services
+
+    def activate(self, app, env, svc, template, vpc):
+        try:
+            services = self._get_deployed_load_balanced_web_services(load_application(app), svc)
+        except LoadBalancedWebServiceNotFoundException:
+            # TODO DBTP-1643 - this bit of logic does not depend on env, so env shouldn't really be in the exception
+            # message
+            # Exception should be propagated to command and caught there.
+            self.echo(f"No services deployed yet to {app} environment {env}", fg="red")
             raise click.Abort
 
+        application_environment = get_app_environment(app, env)
         try:
             https_listener = find_https_listener(application_environment.session, app, env)
             current_maintenance_page = get_maintenance_page(
@@ -42,7 +66,7 @@ class MaintenancePageProvider:
             )
             remove_current_maintenance_page = False
             if current_maintenance_page:
-                remove_current_maintenance_page = click.confirm(
+                remove_current_maintenance_page = self.user_prompt_callback(
                     f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
                     f"environment in {app}.\nWould you like to replace it with a '{template}' "
                     f"maintenance page?"
@@ -50,7 +74,7 @@ class MaintenancePageProvider:
                 if not remove_current_maintenance_page:
                     raise click.Abort
 
-            if remove_current_maintenance_page or click.confirm(
+            if remove_current_maintenance_page or self.user_prompt_callback(
                 f"You are about to enable the '{template}' maintenance page for the {env} "
                 f"environment in {app}.\nWould you like to continue?"
             ):
@@ -68,7 +92,7 @@ class MaintenancePageProvider:
                     allowed_ips,
                     template,
                 )
-                click.secho(
+                self.echo(
                     f"Maintenance page '{template}' added for environment {env} in application {app}",
                     fg="green",
                 )
@@ -76,13 +100,13 @@ class MaintenancePageProvider:
                 raise click.Abort
 
         except LoadBalancerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No load balancer found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
 
         except ListenerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
@@ -96,28 +120,28 @@ class MaintenancePageProvider:
                 application_environment.session, https_listener
             )
             if not current_maintenance_page:
-                click.secho("There is no current maintenance page to remove", fg="red")
+                self.echo("There is no current maintenance page to remove", fg="red")
                 raise click.Abort
 
-            if not click.confirm(
+            if not self.user_prompt_callback(
                 f"There is currently a '{current_maintenance_page}' maintenance page, "
                 f"would you like to remove it?"
             ):
                 raise click.Abort
 
             remove_maintenance_page(application_environment.session, https_listener)
-            click.secho(
+            self.echo(
                 f"Maintenance page removed from environment {env} in application {app}", fg="green"
             )
 
         except LoadBalancerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No load balancer found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
 
         except ListenerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort