dbt-platform-helper 13.4.0__py3-none-any.whl → 14.0.0__py3-none-any.whl

dbt_platform_helper/COMMANDS.md

@@ -12,6 +12,7 @@
     - [platform-helper conduit](#platform-helper-conduit)
     - [platform-helper config](#platform-helper-config)
         - [platform-helper config validate](#platform-helper-config-validate)
+        - [platform-helper config migrate](#platform-helper-config-migrate)
         - [platform-helper config aws](#platform-helper-config-aws)
     - [platform-helper copilot](#platform-helper-copilot)
         - [platform-helper copilot make-addons](#platform-helper-copilot-make-addons)
@@ -33,8 +34,6 @@
         - [platform-helper database dump](#platform-helper-database-dump)
         - [platform-helper database load](#platform-helper-database-load)
         - [platform-helper database copy](#platform-helper-database-copy)
-    - [platform-helper version](#platform-helper-version)
-        - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
 
 # platform-helper
 
@@ -64,7 +63,6 @@ platform-helper <command> [--version]
 - [`notify` ↪](#platform-helper-notify)
 - [`pipeline` ↪](#platform-helper-pipeline)
 - [`secrets` ↪](#platform-helper-secrets)
-- [`version` ↪](#platform-helper-version)
 
 # platform-helper application
 
@@ -295,7 +293,7 @@ platform-helper conduit <addon_name>
 ## Usage
 
 ```
-platform-helper config (validate|aws) 
+platform-helper config (validate|migrate|aws) 
 ```
 
 ## Options
@@ -306,6 +304,7 @@ platform-helper config (validate|aws)
 ## Commands
 
 - [`aws` ↪](#platform-helper-config-aws)
+- [`migrate` ↪](#platform-helper-config-migrate)
 - [`validate` ↪](#platform-helper-config-validate)
 
 # platform-helper config validate
@@ -322,6 +321,23 @@ platform-helper config validate
 
 ## Options
 
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
+# platform-helper config migrate
+
+[↩ Parent](#platform-helper-config)
+
+    Update configuration to match current schema.
+
+## Usage
+
+```
+platform-helper config migrate 
+```
+
+## Options
+
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -488,7 +504,7 @@ platform-helper environment generate --name <name>
 ## Usage
 
 ```
-platform-helper environment generate-terraform --name <name> [--terraform-platform-modules-version <terraform_platform_modules_version>] 
+platform-helper environment generate-terraform --name <name> 
 ```
 
 ## Options
@@ -496,8 +512,6 @@ platform-helper environment generate-terraform --name <name> [--terraform-platfo
 - `--name
 -n <text>`
   - The name of the environment to generate a manifest for.
-- `--terraform-platform-modules-version <text>`
-  - Override the default version of terraform-platform-modules. (Default version is '7').
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -550,27 +564,23 @@ platform-helper pipeline generate
     deployment pipelines.
 
     This command does the following in relation to the environment pipelines:
-    - Reads contents of `platform-config.yml/environment-pipelines` configuration.
+    - Reads contents of `platform-config.yml/environment_pipelines` configuration.
       The `terraform/environment-pipelines/<aws_account>/main.tf` file is generated using this configuration.
       The `main.tf` file is then used to generate Terraform for creating an environment pipeline resource.
 
     This command does the following in relation to the codebase pipelines:
-    - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
+    - Reads contents of `platform-config.yml/codebase_pipelines` configuration.
+      The `terraform/codebase-pipelines/main.tf.json` file is generated using this configuration.
+      The `main.tf.json` file is then used to generate Terraform for creating a codebase pipeline resource.
 
 ## Usage
 
 ```
-platform-helper pipeline generate [--terraform-platform-modules-version <terraform_platform_modules_version>] 
-                                  [--deploy-branch <deploy_branch>] 
+platform-helper pipeline generate [--deploy-branch <deploy_branch>] 
 ```
 
 ## Options
 
-- `--terraform-platform-modules-version <text>`
-  - Override the default version of terraform-platform-modules with a specific version or branch. 
-Precedence of version used is version supplied via CLI, then the version found in 
-platform-config.yml/default_versions/terraform-platform-modules. 
-In absence of these inputs, defaults to version '7'.
 - `--deploy-branch <text>`
   - Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource. 
 This is generated from the terraform/environments-pipeline/<aws_account>/main.tf file. 
@@ -856,44 +866,3 @@ platform-helper database copy --from <from_env> --to <to_env> --database <databa
 
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
-
-# platform-helper version
-
-[↩ Parent](#platform-helper)
-
-    Contains subcommands for getting version information about the current
-    project.
-
-## Usage
-
-```
-platform-helper version get-platform-helper-for-project 
-```
-
-## Options
-
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-## Commands
-
-- [`get-platform-helper-for-project` ↪](#platform-helper-version-get-platform-helper-for-project)
-
-# platform-helper version get-platform-helper-for-project
-
-[↩ Parent](#platform-helper-version)
-
-    Print the version of platform-tools required by the current project
-
-## Usage
-
-```
-platform-helper version get-platform-helper-for-project [--pipeline <pipeline>] 
-```
-
-## Options
-
-- `--pipeline <text>`
-  - Take into account platform-tools version overrides in the specified pipeline
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.

dbt_platform_helper/commands/config.py

@@ -22,6 +22,15 @@ def validate():
         ClickIOProvider().abort_with_error(str(err))
 
 
+@config.command()
+def migrate():
+    """Update configuration to match current schema."""
+    try:
+        Config().migrate()
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))
+
+
 @config.command()
 @click.option("--file-path", "-fp", default="~/.aws/config")
 def aws(file_path):

dbt_platform_helper/commands/environment.py

@@ -1,6 +1,5 @@
 import click
 
-from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
 from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
@@ -87,15 +86,12 @@ def generate(name):
 @click.option(
     "--name", "-n", required=True, help="The name of the environment to generate a manifest for."
 )
-@click.option(
-    "--terraform-platform-modules-version",
-    help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
-)
-def generate_terraform(name, terraform_platform_modules_version):
+def generate_terraform(name):
     click_io = ClickIOProvider()
     try:
         session = get_aws_session_or_abort()
         config_provider = ConfigProvider(ConfigValidator(session=session))
-        TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
+        TerraformEnvironment(config_provider).generate(name)
+
     except PlatformException as err:
         click_io.abort_with_error(str(err))

dbt_platform_helper/commands/notify.py

@@ -1,9 +1,10 @@
 import click
-from slack_sdk import WebClient
-from slack_sdk.models import blocks
 
+from dbt_platform_helper.domain.notify import Notify
 from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
-from dbt_platform_helper.utils.arn_parser import ARN
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.slack_channel_notifier import SlackChannelNotifier
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 
 
@@ -31,56 +32,20 @@ def environment_progress(
     commit_sha: str,
     slack_ref: str,
 ):
-    args = _get_slack_args(build_arn, commit_sha, message, repository, slack_channel_id)
-    slack = _get_slack_client(slack_token)
-
-    if slack_ref:
-        response = slack.chat_update(ts=slack_ref, **args)
-    else:
-        response = slack.chat_postMessage(ts=slack_ref, **args)
-
-    print(response["ts"])
-
-
-def _get_slack_args(
-    build_arn: str, commit_sha: str, message: str, repository: str, slack_channel_id: str
-):
-    context_elements = []
-    if repository:
-        context_elements.append(f"*Repository*: <https://github.com/{repository}|{repository}>")
-        if commit_sha:
-            context_elements.append(
-                f"*Revision*: <https://github.com/{repository}/commit/{commit_sha}|{commit_sha}>"
-            )
-    if build_arn:
-        context_elements.append(f"<{get_build_url(build_arn)}|Build Logs>")
-    message_blocks = [
-        blocks.SectionBlock(
-            text=blocks.TextObject(type="mrkdwn", text=message),
-        ),
-    ]
-
-    if context_elements:
-        message_blocks.append(
-            blocks.ContextBlock(
-                elements=[
-                    blocks.TextObject(type="mrkdwn", text=element) for element in context_elements
-                ]
-            )
+    try:
+        io = ClickIOProvider()
+        slack_notifier = SlackChannelNotifier(slack_token, slack_channel_id)
+        result = Notify(slack_notifier).environment_progress(
+            original_message_ref=slack_ref,
+            message=message,
+            build_arn=build_arn,
+            repository=repository,
+            commit_sha=commit_sha,
         )
 
-    args = {
-        "channel": slack_channel_id,
-        "blocks": message_blocks,
-        "text": message,
-        "unfurl_links": False,
-        "unfurl_media": False,
-    }
-    return args
-
-
-def _get_slack_client(token: str):
-    return WebClient(token=token)
+        io.info(result)
+    except PlatformException as err:
+        io.abort_with_error(str(err))
 
 
 @notify.command(help="Add a comment to an existing Slack message")
@@ -98,31 +63,13 @@ def add_comment(
     title: str,
     send_to_main_channel: bool,
 ):
-    slack = _get_slack_client(slack_token)
-
-    slack.chat_postMessage(
-        channel=slack_channel_id,
-        blocks=[blocks.SectionBlock(text=blocks.TextObject(type="mrkdwn", text=message))],
-        text=title if title else message,
-        reply_broadcast=send_to_main_channel,
-        unfurl_links=False,
-        unfurl_media=False,
-        thread_ts=slack_ref,
-    )
-
-
-def get_build_url(build_arn: str):
     try:
-        arn = ARN(build_arn)
-        url = (
-            "https://{region}.console.aws.amazon.com/codesuite/codebuild/{account}/projects/{"
-            "project}/build/{project}%3A{build_id}"
-        )
-        return url.format(
-            region=arn.region,
-            account=arn.account_id,
-            project=arn.project.replace("build/", ""),
-            build_id=arn.build_id,
+        slack_notifier = SlackChannelNotifier(slack_token, slack_channel_id)
+        Notify(slack_notifier).add_comment(
+            message=message,
+            title=title,
+            reply_broadcast=send_to_main_channel,
+            original_message_ref=slack_ref,
         )
-    except ValueError:
-        return ""
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))

dbt_platform_helper/commands/pipeline.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import click
 
-from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.domain.pipelines import Pipelines
 from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
 from dbt_platform_helper.providers.config import ConfigProvider
@@ -21,13 +20,6 @@ def pipeline():
 
 
 @pipeline.command()
-@click.option(
-    "--terraform-platform-modules-version",
-    help=f"""Override the default version of terraform-platform-modules with a specific version or branch. 
-    Precedence of version used is version supplied via CLI, then the version found in 
-    platform-config.yml/default_versions/terraform-platform-modules. 
-    In absence of these inputs, defaults to version '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}'.""",
-)
 @click.option(
     "--deploy-branch",
     help="""Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource. 
@@ -36,18 +28,20 @@ def pipeline():
     <application>-deploy/platform-config.yml/environment_pipelines/<environment-pipeline>/branch).""",
     default=None,
 )
-def generate(terraform_platform_modules_version: str, deploy_branch: str):
+def generate(deploy_branch: str):
     """
     Given a platform-config.yml file, generate environment and service
     deployment pipelines.
 
     This command does the following in relation to the environment pipelines:
-    - Reads contents of `platform-config.yml/environment-pipelines` configuration.
+    - Reads contents of `platform-config.yml/environment_pipelines` configuration.
       The `terraform/environment-pipelines/<aws_account>/main.tf` file is generated using this configuration.
       The `main.tf` file is then used to generate Terraform for creating an environment pipeline resource.
 
     This command does the following in relation to the codebase pipelines:
-    - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
+    - Reads contents of `platform-config.yml/codebase_pipelines` configuration.
+      The `terraform/codebase-pipelines/main.tf.json` file is generated using this configuration.
+      The `main.tf.json` file is then used to generate Terraform for creating a codebase pipeline resource.
     """
     io = ClickIOProvider()
     try:
@@ -59,6 +53,6 @@ def generate(terraform_platform_modules_version: str, deploy_branch: str):
             get_codestar_connection_arn,
             io,
         )
-        pipelines.generate(terraform_platform_modules_version, deploy_branch)
+        pipelines.generate(deploy_branch)
     except Exception as exc:
         io.abort_with_error(str(exc))

dbt_platform_helper/commands/secrets.py

@@ -111,7 +111,7 @@ def list(app, env):
         else:
             break
 
-    # Todo: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
+    # TODO: DBTP-1953: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
     print("\n".join(sorted(secrets)))
 
 

dbt_platform_helper/constants.py

@@ -1,11 +1,13 @@
-# Todo: Move to Config provider
+# TODO: DBTP-1888: Move to Config provider
 PLATFORM_CONFIG_FILE = "platform-config.yml"
-# Todo: Can we get rid of this yet?
-PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
-# Todo: Move to ???
-DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "7"
+PLATFORM_HELPER_VERSION_OVERRIDE_KEY = "PLATFORM_HELPER_VERSION_OVERRIDE"
+PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
 SUPPORTED_TERRAFORM_VERSION = "~> 1.8"
 SUPPORTED_AWS_PROVIDER_VERSION = "~> 5"
+FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR_VERSION = 13
+
+MERGED_TPM_PLATFORM_HELPER_VERSION = 14
+PLATFORM_CONFIG_SCHEMA_VERSION = 1
 
 # Keys
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"

dbt_platform_helper/domain/codebase.py

@@ -27,7 +27,6 @@ from dbt_platform_helper.utils.aws import get_manual_release_pipeline
 from dbt_platform_helper.utils.aws import list_latest_images
 from dbt_platform_helper.utils.aws import start_build_extraction
 from dbt_platform_helper.utils.aws import start_pipeline_and_return_execution_id
-from dbt_platform_helper.utils.git import check_if_commit_exists
 from dbt_platform_helper.utils.template import setup_templates
 
 
@@ -50,7 +49,6 @@ class Codebase:
         start_pipeline_and_return_execution_id: Callable[
             [str], str
         ] = start_pipeline_and_return_execution_id,
-        check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
         run_subprocess: Callable[[str], str] = subprocess.run,
     ):
         self.parameter_provider = parameter_provider
@@ -65,7 +63,6 @@ class Codebase:
         self.list_latest_images = list_latest_images
         self.start_build_extraction = start_build_extraction
         self.start_pipeline_and_return_execution_id = start_pipeline_and_return_execution_id
-        self.check_if_commit_exists = check_if_commit_exists
         self.run_subprocess = run_subprocess
 
     def prepare(self):
@@ -133,8 +130,6 @@ class Codebase:
         session = self.get_aws_session_or_abort()
         self.load_application(app, default_session=session)
 
-        self.check_if_commit_exists(commit)
-
         codebuild_client = session.client("codebuild")
         project_name = self.get_image_build_project(codebuild_client, app, codebase)
         build_url = self.__start_build_with_confirmation(

dbt_platform_helper/domain/config.py

@@ -12,13 +12,15 @@ from dbt_platform_helper.domain.versioning import CopilotVersioning
 from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.aws.sso_auth import SSOAuthProvider
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.schema_migrator import ALL_MIGRATIONS
+from dbt_platform_helper.providers.schema_migrator import Migrator
 from dbt_platform_helper.providers.semantic_version import (
     IncompatibleMajorVersionException,
 )
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
 from dbt_platform_helper.providers.validation import ValidationException
-from dbt_platform_helper.providers.version_status import PlatformHelperVersionStatus
 from dbt_platform_helper.providers.version_status import VersionStatus
 
 yes = "\033[92m✔\033[0m"
@@ -61,7 +63,7 @@ class NoDeploymentRepoConfigException(PlatformException):
         super().__init__("Could not find a deployment repository, no checks to run.")
 
 
-# TODO move to generic location so it can be reused
+# TODO: DBTP-1993: move to generic location so it can be reused
 class NoPlatformConfigException(PlatformException):
     def __init__(self):
         super().__init__(
@@ -71,7 +73,6 @@ class NoPlatformConfigException(PlatformException):
 
 
 class Config:
-
     def __init__(
         self,
         io: ClickIOProvider = ClickIOProvider(),
@@ -79,6 +80,8 @@ class Config:
         aws_versioning: AWSVersioning = AWSVersioning(),
         copilot_versioning: CopilotVersioning = CopilotVersioning(),
         sso: SSOAuthProvider = None,
+        config_provider: ConfigProvider = ConfigProvider(),
+        migrator: Migrator = Migrator(ALL_MIGRATIONS, io_provider=ClickIOProvider()),
     ):
         self.oidc_app = None
         self.io = io
@@ -87,6 +90,8 @@ class Config:
         self.copilot_versioning = copilot_versioning
         self.sso = sso or SSOAuthProvider()
         self.SSO_START_URL = "https://uktrade.awsapps.com/start"
+        self.config_provider = config_provider
+        self.migrator = migrator
 
     def validate(self):
         if not Path("copilot").exists():
@@ -95,10 +100,7 @@ class Config:
             raise NoPlatformConfigException()
 
         self.io.debug("\nDetected a deployment repository\n")
-        platform_helper_version_status = self.platform_helper_versioning._get_version_status(
-            include_project_versions=True
-        )
-        self.io.process_messages(platform_helper_version_status.validate())
+        platform_helper_version_status = self.platform_helper_versioning.get_version_status()
         aws_version_status = self.aws_versioning.get_version_status()
         copilot_version_status = self.copilot_versioning.get_version_status()
 
@@ -110,6 +112,11 @@ class Config:
 
         exit(0 if compatible else 1)
 
+    def migrate(self):
+        platform_config = self.config_provider.load_unvalidated_config_file()
+        new_platform_config = self.migrator.migrate(platform_config)
+        self.config_provider.write_platform_config(new_platform_config)
+
     def generate_aws(self, file_path):
         self.oidc_app = self._create_oidc_application()
         verification_url, device_code = self._get_device_code(self.oidc_app)
@@ -184,7 +191,7 @@ class Config:
 
     def _check_tool_versions(
         self,
-        platform_helper_version_status: PlatformHelperVersionStatus,
+        platform_helper_version_status: VersionStatus,
         aws_version_status: VersionStatus,
         copilot_version_status: VersionStatus,
     ):
@@ -237,7 +244,7 @@ class Config:
 
         self._render_recommendations(recommendations)
 
-    def _check_addon_versions(self, platform_helper_versions: PlatformHelperVersionStatus) -> bool:
+    def _check_addon_versions(self, platform_helper_versions: VersionStatus) -> bool:
 
         self.io.debug("Checking addons templates versions...")
 

dbt_platform_helper/domain/copilot_environment.py

@@ -25,7 +25,7 @@ class CopilotEnvironment:
         config_provider: ConfigProvider,
         vpc_provider: VpcProvider = None,
         cloudformation_provider: CloudFormation = None,
-        session: Session = None,  # TODO - this is a temporary fix, will fall away once _get_environment_vpc is updated.
+        session: Session = None,  # TODO: DBTP-1954: - this is a temporary fix, will fall away once _get_environment_vpc is updated.
         copilot_templating=None,
         io: ClickIOProvider = ClickIOProvider(),
         load_balancer_provider: LoadBalancerProvider = LoadBalancerProvider,
@@ -77,7 +77,7 @@ class CopilotEnvironment:
             )
         )
 
-    # TODO: There should always be a vpc_name as defaults have been applied to the config.  This function can
+    # TODO: DBTP-1954: There should always be a vpc_name as defaults have been applied to the config.  This function can
     # probably fall away. We shouldn't need to check 3 different names (vpc_name, session.profile_name, {session.profile_name}-{env_name})
     # To be checked.
     def _get_environment_vpc(self, session: Session, app_name, env_name: str, vpc_name: str) -> Vpc:
@@ -134,7 +134,7 @@ class CopilotTemplating:
         self,
         file_provider: FileProvider = FileProvider(),
         io: ClickIOProvider = ClickIOProvider(),
-        # TODO file_provider can be moved up a layer.  File writing can be the responsibility of CopilotEnvironment generate
+        # TODO: DBTP-1958: file_provider can be moved up a layer.  File writing can be the responsibility of CopilotEnvironment generate
         # Or we align with PlatformTerraformManifestGenerator and rename from Templating to reflect the file writing responsibility
     ):
         self.file_provider = file_provider

dbt_platform_helper/domain/database_copy.py

@@ -28,7 +28,7 @@ class DatabaseCopy:
         database: str,
         auto_approve: bool = False,
         load_application: Callable[[str], Application] = load_application,
-        # TODO We inject VpcProvider as a callable here so that it can be instantiated within the method.  To be improved
+        # TODO: DBTP-1960: We inject VpcProvider as a callable here so that it can be instantiated within the method.  To be improved
         vpc_provider: Callable[[Session], VpcProvider] = VpcProvider,
         db_connection_string: Callable[
             [Session, str, str, str, Callable], str