dbt-platform-helper 13.2.0__py3-none-any.whl → 13.4.0__py3-none-any.whl

dbt_platform_helper/domain/copilot.py

@@ -5,19 +5,19 @@ import json
 from pathlib import Path
 from pathlib import PosixPath
 
-import click
-import yaml
-from schema import SchemaError
+import botocore
+import botocore.errorfactory
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.kms import KMSProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import get_aws_session_or_abort
-from dbt_platform_helper.utils.files import generate_override_files
 from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
@@ -28,9 +28,6 @@ class Copilot:
 
     PACKAGE_DIR = Path(__file__).resolve().parent.parent
 
-    # TODO Remove and test
-    WAF_ACL_ARN_KEY = "waf-acl-arn"
-
     SERVICE_TYPES = [
         "Load Balanced Web Service",
         "Backend Service",
@@ -42,36 +39,115 @@ class Copilot:
     def __init__(
         self,
         config_provider: ConfigProvider,
+        parameter_provider: ParameterStore,
         file_provider: FileProvider,
         copilot_templating: CopilotTemplating,
         kms_provider: KMSProvider,
+        session,
+        io: ClickIOProvider = ClickIOProvider(),
+        yaml_file_provider: YamlFileProvider = YamlFileProvider,
     ):
         self.config_provider = config_provider
+        self.parameter_provider = parameter_provider
         self.file_provider = file_provider
         self.copilot_templating = copilot_templating
         self.kms_provider = kms_provider
+        self.io = io
+        self.yaml_file_provider = yaml_file_provider
+        self.session = session
+
+    def make_addons(self):
+        config = self.config_provider.load_and_validate_platform_config()
+
+        templates = setup_templates()
+        extensions = self._get_extensions()
+        application_name = get_application_name()
+
+        self.io.info("\n>>> Generating Terraform compatible addons CloudFormation\n")
+
+        output_dir = Path(".").absolute()
+        env_path = Path(f"copilot/environments/")
+        env_addons_path = env_path / "addons"
+        env_overrides_path = env_path / "overrides"
+
+        self._cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
+        self._generate_env_overrides(output_dir)
+
+        svc_names = self._list_copilot_local_services()
+        base_path = Path(".")
+        for svc_name in svc_names:
+            self._generate_svc_overrides(base_path, templates, svc_name)
 
-    def list_copilot_local_environments(self):
+        services = []
+        for ext_name, ext_data in extensions.items():
+            extension = {**ext_data}
+            addon_type = extension.pop("type")
+            environments = extension.pop("environments")
+            environment_addon_config = {
+                "addon_type": addon_type,
+                "environments": environments,
+                "name": extension.get("name", None) or ext_name,
+                "prefix": camel_case(ext_name),
+                "secret_name": ext_name.upper().replace("-", "_"),
+                **extension,
+            }
+
+            services.append(environment_addon_config)
+
+            service_addon_config = {
+                "application_name": application_name,
+                "name": extension.get("name", None) or ext_name,
+                "prefix": camel_case(ext_name),
+                "environments": environments,
+                **extension,
+            }
+
+            log_destination_arns = self._get_log_destination_arn()
+
+            if addon_type in ["s3", "s3-policy"]:
+                if extensions[ext_name].get("serve_static_content"):
+                    continue
+
+                s3_kms_arns = self._get_s3_kms_alias_arns(application_name, environments)
+                for environment_name in environments:
+                    environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
+                        environment_name, "kms-key-not-found"
+                    )
+
+            self._generate_service_addons(
+                extension,
+                ext_name,
+                addon_type,
+                output_dir,
+                service_addon_config,
+                templates,
+                log_destination_arns,
+            )
+
+        environments = self.config_provider.apply_environment_defaults(config)["environments"]
+
+        self.copilot_templating.generate_cross_account_s3_policies(environments, extensions)
+
+        self.io.info(templates.get_template("addon-instructions.txt").render(services=services))
+
+    def _list_copilot_local_environments(self):
         return [
             path.parent.parts[-1] for path in Path("./copilot/environments/").glob("*/manifest.yml")
         ]
 
-    def is_service(self, path: PosixPath) -> bool:
-        with open(path) as manifest_file:
-            data = yaml.safe_load(manifest_file)
-            if not data or not data.get("type"):
-                click.echo(
-                    click.style(f"No type defined in manifest file {str(path)}; exiting", fg="red")
-                )
-                exit(1)
+    def _is_service(self, path: PosixPath) -> bool:
+
+        manifest_file = self.yaml_file_provider.load(path)
+        if not manifest_file or not manifest_file.get("type"):
+            self.io.abort_with_error(f"No type defined in manifest file {str(path)}; exiting")
 
-            return data.get("type") in self.SERVICE_TYPES
+        return manifest_file.get("type") in self.SERVICE_TYPES
 
-    def list_copilot_local_services(self):
+    def _list_copilot_local_services(self):
         return [
             path.parent.parts[-1]
             for path in Path("./copilot/").glob("*/manifest.yml")
-            if self.is_service(path)
+            if self._is_service(path)
         ]
 
     def _validate_and_normalise_extensions_config(self, config_file, key_in_config_file=None):
@@ -93,12 +169,10 @@ class Copilot:
         def _normalise_keys(source: dict):
             return {k.replace("-", "_"): v for k, v in source.items()}
 
-        with open(self.PACKAGE_DIR / "addon-plans.yml", "r") as fd:
-            addon_plans = yaml.safe_load(fd)
+        addon_plans = self.yaml_file_provider.load(self.PACKAGE_DIR / "addon-plans.yml")
 
         # load and validate config
-        with open(config_file, "r") as fd:
-            config = yaml.safe_load(fd)
+        config = self.yaml_file_provider.load(config_file)
 
         if config and key_in_config_file:
             config = config[key_in_config_file]
@@ -110,23 +184,19 @@ class Copilot:
         errors = validate_addons(config)
 
         if errors:
-            click.echo(click.style(f"Errors found in {config_file}:", fg="red"))
+            self.io.error(f"Errors found in {config_file}:")
             for addon, error in errors.items():
-                click.echo(click.style(f"Addon '{addon}': {error}", fg="red"))
-            exit(1)
+                self.io.error(f"Addon '{addon}': {error}")
+            self.io.abort_with_error("Invalid platform-config.yml provided, see above warnings")
 
-        env_names = self.list_copilot_local_environments()
-        svc_names = self.list_copilot_local_services()
+        env_names = self._list_copilot_local_environments()
+        svc_names = self._list_copilot_local_services()
 
         if not env_names:
-            click.echo(
-                click.style(f"No environments found in ./copilot/environments; exiting", fg="red")
-            )
-            exit(1)
+            self.io.abort_with_error("No environments found in ./copilot/environments; exiting")
 
         if not svc_names:
-            click.echo(click.style(f"No services found in ./copilot/; exiting", fg="red"))
-            exit(1)
+            self.io.abort_with_error("No services found in ./copilot/; exiting")
 
         normalised_config = {}
         config_has_errors = False
@@ -139,11 +209,8 @@ class Copilot:
                     normalised_config[addon_name]["services"] = svc_names
 
                 if not set(normalised_config[addon_name]["services"]).issubset(set(svc_names)):
-                    click.echo(
-                        click.style(
-                            f"Services listed in {addon_name}.services do not exist in ./copilot/",
-                            fg="red",
-                        ),
+                    self.io.error(
+                        f"Services listed in {addon_name}.services do not exist in ./copilot/"
                     )
                     config_has_errors = True
 
@@ -154,18 +221,10 @@ class Copilot:
 
             missing_envs = set(environments.keys()) - set(env_names)
             if missing_envs:
-                click.echo(
-                    click.style(
-                        f"Environment keys listed in {addon_name} do not match those defined in ./copilot/environments.",
-                        fg="red",
-                    )
-                ),
-                click.echo(
-                    click.style(
-                        f"  Missing environments: {', '.join(sorted(missing_envs))}",
-                        fg="white",
-                    ),
+                self.io.error(
+                    f"Environment keys listed in {addon_name} do not match those defined in ./copilot/environments"
                 )
+                self.io.error(f"  Missing environments: {', '.join(sorted(missing_envs))}")
                 config_has_errors = True
 
             if config_has_errors:
@@ -186,41 +245,41 @@ class Copilot:
             normalised_config[addon_name]["environments"] = normalised_environments
 
         if config_has_errors:
-            exit(1)
+            self.io.abort_with_error("Configuration has errors. Exiting.")
 
         return normalised_config
 
-    def get_log_destination_arn(self):
+    def _get_log_destination_arn(self):
         """Get destination arns stored in param store in projects aws
         account."""
-        session = get_aws_session_or_abort()
-        client = session.client("ssm", region_name="eu-west-2")
-        response = client.get_parameters(Names=["/copilot/tools/central_log_groups"])
-
-        if not response["Parameters"]:
-            click.echo(
-                click.style(
-                    "No aws central log group defined in Parameter Store at location /copilot/tools/central_log_groups; exiting",
-                    fg="red",
-                )
+
+        try:
+            destination_arns = self.parameter_provider.get_ssm_parameter_by_name(
+                "/copilot/tools/central_log_groups"
+            )
+        except botocore.errorfactory.ParameterNotFound:
+            self.io.abort_with_error(
+                "No aws central log group defined in Parameter Store at location /copilot/tools/central_log_groups; exiting"
             )
-            exit(1)
 
-        destination_arns = json.loads(response["Parameters"][0]["Value"])
-        return destination_arns
+        return json.loads(destination_arns["Value"])
 
     def _generate_svc_overrides(self, base_path, templates, name):
-        click.echo(f"\n>>> Generating service overrides for {name}\n")
+        self.io.info(f"\n>>> Generating service overrides for {name}\n")
         overrides_path = base_path.joinpath(f"copilot/{name}/overrides")
         overrides_path.mkdir(parents=True, exist_ok=True)
         overrides_file = overrides_path.joinpath("cfn.patches.yml")
         overrides_file.write_text(templates.get_template("svc/overrides/cfn.patches.yml").render())
 
-    def _get_s3_kms_alias_arns(self, session, application_name, config):
-        application = load_application(application_name, session)
+    def _get_s3_kms_alias_arns(self, application_name, config):
+        application = load_application(application_name, self.session)
         arns = {}
 
         for environment_name in application.environments:
+            kms_provider = self.kms_provider(
+                application.environments[environment_name].session.client("kms")
+            )
+
             if environment_name not in config:
                 continue
 
@@ -228,99 +287,20 @@ class Copilot:
                 continue
 
             bucket_name = config[environment_name]["bucket_name"]
-            kms_client = application.environments[environment_name].session.client("kms")
             alias_name = f"alias/{application_name}-{environment_name}-{bucket_name}-key"
 
             try:
-                response = kms_client.describe_key(KeyId=alias_name)
-            except kms_client.exceptions.NotFoundException:
-                pass
+                response = kms_provider.describe_key(alias_name)
+
+            # Boto3 classifies all AWS service errors and exceptions as ClientError exceptions
+            except botocore.exceptions.ClientError as error:
+                if error.response["Error"]["Code"] == "NotFoundException":
+                    pass
             else:
                 arns[environment_name] = response["KeyMetadata"]["Arn"]
 
         return arns
 
-    def make_addons(self):
-        self.config_provider.config_file_check()
-        try:
-            config = self.config_provider.load_and_validate_platform_config()
-        except SchemaError as ex:
-            click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
-            raise click.Abort
-
-        templates = setup_templates()
-        extensions = self._get_extensions()
-        session = get_aws_session_or_abort()
-
-        application_name = get_application_name()
-
-        click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
-
-        output_dir = Path(".").absolute()
-        env_path = Path(f"copilot/environments/")
-        env_addons_path = env_path / "addons"
-        env_overrides_path = env_path / "overrides"
-
-        self._cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
-        self._generate_env_overrides(output_dir)
-
-        svc_names = self.list_copilot_local_services()
-        base_path = Path(".")
-        for svc_name in svc_names:
-            self._generate_svc_overrides(base_path, templates, svc_name)
-
-        services = []
-        for ext_name, ext_data in extensions.items():
-            extension = {**ext_data}
-            addon_type = extension.pop("type")
-            environments = extension.pop("environments")
-            environment_addon_config = {
-                "addon_type": addon_type,
-                "environments": environments,
-                "name": extension.get("name", None) or ext_name,
-                "prefix": camel_case(ext_name),
-                "secret_name": ext_name.upper().replace("-", "_"),
-                **extension,
-            }
-
-            services.append(environment_addon_config)
-
-            service_addon_config = {
-                "application_name": application_name,
-                "name": extension.get("name", None) or ext_name,
-                "prefix": camel_case(ext_name),
-                "environments": environments,
-                **extension,
-            }
-
-            log_destination_arns = self.get_log_destination_arn()
-
-            if addon_type in ["s3", "s3-policy"]:
-                if extensions[ext_name].get("serve_static_content"):
-                    continue
-
-                s3_kms_arns = self._get_s3_kms_alias_arns(session, application_name, environments)
-                for environment_name in environments:
-                    environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
-                        environment_name, "kms-key-not-found"
-                    )
-
-            self._generate_service_addons(
-                extension,
-                ext_name,
-                addon_type,
-                output_dir,
-                service_addon_config,
-                templates,
-                log_destination_arns,
-            )
-
-        environments = self.config_provider.apply_environment_defaults(config)["environments"]
-
-        self.copilot_templating.generate_cross_account_s3_policies(environments, extensions)
-
-        click.echo(templates.get_template("addon-instructions.txt").render(services=services))
-
     def _get_extensions(self):
         config = self._validate_and_normalise_extensions_config(
             self.PACKAGE_DIR / "default-extensions.yml"
@@ -331,13 +311,31 @@ class Copilot:
         config.update(project_config)
         return config
 
+    def _generate_override_files(self, base_path, file_path, output_dir):
+        def generate_files_for_dir(pattern):
+            for file in file_path.glob(pattern):
+                if file.is_file():
+                    contents = file.read_text()
+                    file_name = str(file).removeprefix(f"{file_path}/")
+                    self.io.info(
+                        self.file_provider.mkfile(
+                            base_path,
+                            output_dir / file_name,
+                            contents,
+                            overwrite=True,
+                        )
+                    )
+
+        generate_files_for_dir("*")
+        generate_files_for_dir("bin/*")
+
     def _generate_env_overrides(self, output_dir):
         path = "templates/env/terraform-overrides"
-        click.echo("\n>>> Generating Environment overrides\n")
+        self.io.info("\n>>> Generating Environment overrides\n")
         overrides_path = output_dir.joinpath(f"copilot/environments/overrides")
         overrides_path.mkdir(parents=True, exist_ok=True)
         template_overrides_path = Path(__file__).parent.parent.joinpath(path)
-        generate_override_files(Path("."), template_overrides_path, overrides_path)
+        self._generate_override_files(Path("."), template_overrides_path, overrides_path)
 
     def _generate_service_addons(
         self,
@@ -364,7 +362,7 @@ class Copilot:
                 )
 
                 (output_dir / service_path).mkdir(parents=True, exist_ok=True)
-                click.echo(
+                self.io.info(
                     self.file_provider.mkfile(
                         output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
                     )

dbt_platform_helper/domain/versioning.py

@@ -9,11 +9,16 @@ from dbt_platform_helper.providers.semantic_version import (
 from dbt_platform_helper.providers.semantic_version import (
     IncompatibleMinorVersionException,
 )
-from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.version import AWSCLIInstalledVersionProvider
+from dbt_platform_helper.providers.version import CopilotInstalledVersionProvider
 from dbt_platform_helper.providers.version import DeprecatedVersionFileVersionProvider
+from dbt_platform_helper.providers.version import GithubLatestVersionProvider
 from dbt_platform_helper.providers.version import InstalledVersionProvider
-from dbt_platform_helper.providers.version import PyPiVersionProvider
+from dbt_platform_helper.providers.version import PyPiLatestVersionProvider
+from dbt_platform_helper.providers.version import VersionProvider
+from dbt_platform_helper.providers.version_status import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.version_status import VersionStatus
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
@@ -38,14 +43,14 @@ class PlatformHelperVersioning:
             YamlFileProvider
         ),
         config_provider: ConfigProvider = ConfigProvider(),
-        pypi_provider: PyPiVersionProvider = PyPiVersionProvider,
+        latest_version_provider: VersionProvider = PyPiLatestVersionProvider,
         installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
         skip_versioning_checks: bool = None,
     ):
         self.io = io
         self.version_file_version_provider = version_file_version_provider
         self.config_provider = config_provider
-        self.pypi_provider = pypi_provider
+        self.latest_version_provider = latest_version_provider
         self.installed_version_provider = installed_version_provider
         self.skip_versioning_checks = (
             skip_versioning_checks if skip_versioning_checks is not None else skip_version_checks()
@@ -100,11 +105,11 @@ class PlatformHelperVersioning:
         self,
         include_project_versions: bool = True,
     ) -> PlatformHelperVersionStatus:
-        locally_installed_version = self.installed_version_provider.get_installed_tool_version(
+        locally_installed_version = self.installed_version_provider.get_semantic_version(
             "dbt-platform-helper"
         )
 
-        latest_release = self.pypi_provider.get_latest_version("dbt-platform-helper")
+        latest_release = self.latest_version_provider.get_semantic_version("dbt-platform-helper")
 
         if not include_project_versions:
             return PlatformHelperVersionStatus(
@@ -129,7 +134,7 @@ class PlatformHelperVersioning:
         out = PlatformHelperVersionStatus(
             installed=locally_installed_version,
             latest=latest_release,
-            deprecated_version_file=self.version_file_version_provider.get_required_version(),
+            deprecated_version_file=self.version_file_version_provider.get_semantic_version(),
             platform_config_default=platform_config_default,
             pipeline_overrides=pipeline_overrides,
         )
@@ -155,3 +160,39 @@ class PlatformHelperVersioning:
             raise PlatformHelperVersionNotFoundException
 
         return out
+
+
+class AWSVersioning:
+    def __init__(
+        self,
+        latest_version_provider: VersionProvider = None,
+        installed_version_provider: VersionProvider = None,
+    ):
+        self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
+        self.installed_version_provider = (
+            installed_version_provider or AWSCLIInstalledVersionProvider
+        )
+
+    def get_version_status(self) -> VersionStatus:
+        return VersionStatus(
+            self.installed_version_provider.get_semantic_version(),
+            self.latest_version_provider.get_semantic_version("aws/aws-cli", True),
+        )
+
+
+class CopilotVersioning:
+    def __init__(
+        self,
+        latest_version_provider: VersionProvider = None,
+        installed_version_provider: VersionProvider = None,
+    ):
+        self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
+        self.installed_version_provider = (
+            installed_version_provider or CopilotInstalledVersionProvider
+        )
+
+    def get_version_status(self) -> VersionStatus:
+        return VersionStatus(
+            self.installed_version_provider.get_semantic_version(),
+            self.latest_version_provider.get_semantic_version("aws/copilot-cli"),
+        )

dbt_platform_helper/providers/aws/exceptions.py

@@ -13,9 +13,9 @@ class CreateTaskTimeoutException(AWSException):
 
 
 class ImageNotFoundException(AWSException):
-    def __init__(self, commit: str):
+    def __init__(self, image_ref: str):
         super().__init__(
-            f"""The commit hash "{commit}" has not been built into an image, try the `platform-helper codebase build` command first."""
+            f"""An image labelled "{image_ref}" could not be found in your image repository. Try the `platform-helper codebase build` command first."""
         )
 
 
@@ -35,3 +35,13 @@ class CopilotCodebaseNotFoundException(PlatformException):
         super().__init__(
             f"""The codebase "{codebase}" either does not exist or has not been deployed."""
         )
+
+
+class CreateAccessTokenException(AWSException):
+    def __init__(self, client_id: str):
+        super().__init__(f"""Failed to create access token for Client "{client_id}".""")
+
+
+class UnableToRetrieveSSOAccountList(AWSException):
+    def __init__(self):
+        super().__init__("Unable to retrieve AWS SSO account list")

dbt_platform_helper/providers/aws/sso_auth.py

@@ -0,0 +1,61 @@
+import botocore
+from boto3 import Session
+
+from dbt_platform_helper.providers.aws.exceptions import CreateAccessTokenException
+from dbt_platform_helper.providers.aws.exceptions import UnableToRetrieveSSOAccountList
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+
+
+class SSOAuthProvider:
+    def __init__(self, session: Session = None):
+        self.session = session
+        self.sso_oidc = self._get_client("sso-oidc")
+        self.sso = self._get_client("sso")
+
+    def register(self, client_name, client_type):
+        client = self.sso_oidc.register_client(clientName=client_name, clientType=client_type)
+        client_id = client.get("clientId")
+        client_secret = client.get("clientSecret")
+
+        return client_id, client_secret
+
+    def start_device_authorization(self, client_id, client_secret, start_url):
+        authz = self.sso_oidc.start_device_authorization(
+            clientId=client_id,
+            clientSecret=client_secret,
+            startUrl=start_url,
+        )
+        url = authz.get("verificationUriComplete")
+        deviceCode = authz.get("deviceCode")
+
+        return url, deviceCode
+
+    def create_access_token(self, client_id, client_secret, device_code):
+        try:
+            response = self.sso_oidc.create_token(
+                clientId=client_id,
+                clientSecret=client_secret,
+                grantType="urn:ietf:params:oauth:grant-type:device_code",
+                deviceCode=device_code,
+            )
+
+            return response.get("accessToken")
+
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] != "AuthorizationPendingException":
+                raise CreateAccessTokenException(client_id)
+
+    def list_accounts(self, access_token, max_results=100):
+        aws_accounts_response = self.sso.list_accounts(
+            accessToken=access_token,
+            maxResults=max_results,
+        )
+
+        if len(aws_accounts_response.get("accountList", [])) == 0:
+            raise UnableToRetrieveSSOAccountList()
+        return aws_accounts_response.get("accountList")
+
+    def _get_client(self, client: str):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client(client)

dbt_platform_helper/providers/config.py

@@ -64,7 +64,8 @@ class ConfigProvider:
         except FileProviderException:
             return {}
 
-    # TODO this general function should be moved out of ConfigProvider
+    # TODO remove function and push logic to where this is called.
+    # removed usage from config domain, code is very generic and doesn't require the overhead of a function
     def config_file_check(self, path=PLATFORM_CONFIG_FILE):
         if not Path(path).exists():
             self.io.abort_with_error(