dbt-platform-helper 13.0.0__py3-none-any.whl → 13.0.2__py3-none-any.whl

dbt_platform_helper/providers/files.py

@@ -1,4 +1,3 @@
-from os import makedirs
 from pathlib import Path
 
 
@@ -8,19 +7,21 @@ class FileProvider:
         pass
 
     @staticmethod
-    def mkfile(base_path: str, file_path: str, contents: str, overwrite=False) -> str:
-        file_path = Path(file_path)
-        file = Path(base_path).joinpath(file_path)
-        file_exists = file.exists()
-
-        if not file_path.parent.exists():
-            makedirs(file_path.parent)
-
+    def mkfile(base_path: str, file_name: str, contents: str, overwrite=False) -> str:
+        file_path = Path(base_path).joinpath(file_name)
+        file_exists = file_path.exists()
         if file_exists and not overwrite:
             return f"File {file_path} exists; doing nothing"
 
-        action = "overwritten" if file_exists and overwrite else "created"
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(contents)
 
-        file.write_text(contents)
+        action = "overwritten" if file_exists and overwrite else "created"
+        return f"File {file_name} {action}"
 
-        return f"File {file_path} {action}"
+    @staticmethod
+    def delete_file(base_path: str, file_name: str):
+        file_path = Path(base_path) / file_name
+        if file_path.exists():
+            file_path.unlink()
+            return f"{str(file_path)} has been deleted"

dbt_platform_helper/providers/load_balancers.py

@@ -21,7 +21,9 @@ def get_load_balancer_for_application(session: boto3.Session, app: str, env: str
             load_balancer_arn = lb["ResourceArn"]
 
     if not load_balancer_arn:
-        raise LoadBalancerNotFoundException()
+        raise LoadBalancerNotFoundException(
+            f"No load balancer found for {app} in the {env} environment"
+        )
 
     return load_balancer_arn
 
@@ -39,7 +41,7 @@ def get_https_listener_for_application(session: boto3.Session, app: str, env: st
         pass
 
     if not listener_arn:
-        raise ListenerNotFoundException()
+        raise ListenerNotFoundException(f"No HTTPS listener for {app} in the {env} environment")
 
     return listener_arn
 

dbt_platform_helper/providers/semantic_version.py

@@ -0,0 +1,126 @@
+import re
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Dict
+from typing import Optional
+from typing import Union
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.providers.validation import ValidationException
+
+
+class IncompatibleMajorVersionException(ValidationException):
+    def __init__(self, app_version: str, check_version: str):
+        super().__init__()
+        self.app_version = app_version
+        self.check_version = check_version
+
+
+class IncompatibleMinorVersionException(ValidationException):
+    def __init__(self, app_version: str, check_version: str):
+        super().__init__()
+        self.app_version = app_version
+        self.check_version = check_version
+
+
+class SemanticVersion:
+    def __init__(self, major, minor, patch):
+        self.major = major
+        self.minor = minor
+        self.patch = patch
+
+    def __str__(self) -> str:
+        if self.major is None:
+            return "unknown"
+        return ".".join([str(s) for s in [self.major, self.minor, self.patch]])
+
+    def __lt__(self, other) -> bool:
+        return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
+
+    def __eq__(self, other) -> bool:
+        return (self.major, self.minor, self.patch) == (other.major, other.minor, other.patch)
+
+    def validate_compatibility_with(self, other):
+        if (self.major == 0 and other.major == 0) and (
+            self.minor != other.minor or self.patch != other.patch
+        ):
+            raise IncompatibleMajorVersionException(str(self), str(other))
+
+        if self.major != other.major:
+            raise IncompatibleMajorVersionException(str(self), str(other))
+
+        if self.minor != other.minor:
+            raise IncompatibleMinorVersionException(str(self), str(other))
+
+    @staticmethod
+    def from_string(version_string: Union[str, None]):
+        if version_string is None:
+            return None
+
+        version_plain = version_string.replace("v", "")
+        version_segments = re.split(r"[.\-]", version_plain)
+
+        if len(version_segments) != 3:
+            return None
+
+        output_version = [0, 0, 0]
+        for index, segment in enumerate(version_segments):
+            try:
+                output_version[index] = int(segment)
+            except ValueError:
+                output_version[index] = -1
+
+        return SemanticVersion(output_version[0], output_version[1], output_version[2])
+
+
+class VersionStatus:
+    def __init__(
+        self, local_version: SemanticVersion = None, latest_release: SemanticVersion = None
+    ):
+        self.local = local_version
+        self.latest = latest_release
+
+    def is_outdated(self):
+        return self.local != self.latest
+
+    def warn(self):
+        pass
+
+
+@dataclass
+class PlatformHelperVersionStatus(VersionStatus):
+    local: Optional[SemanticVersion] = None
+    latest: Optional[SemanticVersion] = None
+    deprecated_version_file: Optional[SemanticVersion] = None
+    platform_config_default: Optional[SemanticVersion] = None
+    pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
+
+    def warn(self) -> dict:
+        if self.platform_config_default and not self.deprecated_version_file:
+            return {}
+
+        warnings = []
+        errors = []
+
+        missing_default_version_message = f"Create a section in the root of '{PLATFORM_CONFIG_FILE}':\n\ndefault_versions:\n  platform-helper: "
+        deprecation_message = (
+            f"Please delete '{PLATFORM_HELPER_VERSION_FILE}' as it is now deprecated."
+        )
+
+        if self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+
+        if not self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+            warnings.append(f"{missing_default_version_message}{self.deprecated_version_file}\n")
+
+        if not self.platform_config_default and not self.deprecated_version_file:
+            message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
+            message += f"{missing_default_version_message}{self.local}\n"
+            errors.append(message)
+
+        return {
+            "warnings": warnings,
+            "errors": errors,
+        }

dbt_platform_helper/providers/terraform_manifest.py

@@ -2,21 +2,20 @@ import json
 from datetime import datetime
 from importlib.metadata import version
 from pathlib import Path
-from typing import Callable
-
-import click
 
 from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
 from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
 
 
 class TerraformManifestProvider:
     def __init__(
-        self, file_provider: FileProvider = FileProvider(), echo: Callable[[str], None] = click.echo
+        self, file_provider: FileProvider = FileProvider(), io: ClickIOProvider = ClickIOProvider()
     ):
         self.file_provider = file_provider
-        self.echo = echo
+        self.io = io
 
     def generate_codebase_pipeline_config(
         self,
@@ -24,28 +23,50 @@ class TerraformManifestProvider:
         terraform_platform_modules_version: str,
         ecr_imports: dict[str, str],
     ):
-        default_account = (
-            platform_config.get("environments", {})
-            .get("*", {})
-            .get("accounts", {})
-            .get("deploy", {})
-            .get("name")
-        )
+        default_account = self._get_account_for_env("*", platform_config)
+        state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
+
         terraform = {}
         self._add_header(terraform)
-        self._add_locals(terraform)
+        self._add_codebase_pipeline_locals(terraform)
         self._add_provider(terraform, default_account)
-        self._add_backend(terraform, platform_config, default_account)
+        self._add_backend(terraform, platform_config, default_account, state_key_suffix)
         self._add_codebase_pipeline_module(terraform, terraform_platform_modules_version)
         self._add_imports(terraform, ecr_imports)
+        self._write_terraform_json(terraform, "terraform/codebase-pipelines")
 
-        message = self.file_provider.mkfile(
-            str(Path(".").absolute()),
-            "terraform/codebase-pipelines/main.tf.json",
-            json.dumps(terraform, indent=2),
-            True,
+    def generate_environment_config(
+        self,
+        platform_config: dict,
+        env: str,
+        terraform_platform_modules_version: str,
+    ):
+        platform_config = ConfigProvider.apply_environment_defaults(platform_config)
+        account = self._get_account_for_env(env, platform_config)
+
+        application_name = platform_config["application"]
+        state_key_suffix = f"{platform_config['application']}-{env}"
+        env_dir = f"terraform/environments/{env}"
+
+        terraform = {}
+        self._add_header(terraform)
+        self._add_environment_locals(terraform, application_name)
+        self._add_backend(terraform, platform_config, account, state_key_suffix)
+        self._add_extensions_module(terraform, terraform_platform_modules_version, env)
+        self._add_moved(terraform, platform_config)
+        self._ensure_no_hcl_manifest_file(env_dir)
+        self._write_terraform_json(terraform, env_dir)
+
+    @staticmethod
+    def _get_account_for_env(env, platform_config):
+        account = (
+            platform_config.get("environments", {})
+            .get(env, {})
+            .get("accounts", {})
+            .get("deploy", {})
+            .get("name")
         )
-        self.echo(message)
+        return account
 
     @staticmethod
     def _add_header(terraform: dict):
@@ -56,7 +77,7 @@ class TerraformManifestProvider:
         terraform["//"] = f"{version_header} {warning}"
 
     @staticmethod
-    def _add_locals(terraform: dict):
+    def _add_codebase_pipeline_locals(terraform: dict):
         terraform["locals"] = {
             "platform_config": '${yamldecode(file("../../platform-config.yml"))}',
             "application": '${local.platform_config["application"]}',
@@ -73,17 +94,17 @@ class TerraformManifestProvider:
         terraform["provider"]["aws"]["shared_credentials_files"] = ["~/.aws/config"]
 
     @staticmethod
-    def _add_backend(terraform: dict, platform_config: dict, default_account: str):
+    def _add_backend(terraform: dict, platform_config: dict, account: str, state_key_suffix: str):
         terraform["terraform"] = {
             "required_version": SUPPORTED_TERRAFORM_VERSION,
             "backend": {
                 "s3": {
-                    "bucket": f"terraform-platform-state-{default_account}",
-                    "key": f"tfstate/application/{platform_config['application']}-codebase-pipelines.tfstate",
+                    "bucket": f"terraform-platform-state-{account}",
+                    "key": f"tfstate/application/{state_key_suffix}.tfstate",
                     "region": "eu-west-2",
                     "encrypt": True,
-                    "kms_key_id": f"alias/terraform-platform-state-s3-key-{default_account}",
-                    "dynamodb_table": f"terraform-platform-lockdb-{default_account}",
+                    "kms_key_id": f"alias/terraform-platform-state-s3-key-{account}",
+                    "dynamodb_table": f"terraform-platform-lockdb-{account}",
                 }
             },
             "required_providers": {
@@ -110,6 +131,13 @@ class TerraformManifestProvider:
             }
         }
 
+    @staticmethod
+    def _add_extensions_module(terraform: dict, terraform_platform_modules_version: str, env: str):
+        source = f"git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref={terraform_platform_modules_version}"
+        terraform["module"] = {
+            "extensions": {"source": source, "args": "${local.args}", "environment": env}
+        }
+
     @staticmethod
     def _add_imports(terraform: dict, ecr_imports: dict[str, str]):
         if ecr_imports:
@@ -118,3 +146,66 @@ class TerraformManifestProvider:
                 "id": "${each.value}",
                 "to": "module.codebase-pipelines[each.key].aws_ecr_repository.this",
             }
+
+    @staticmethod
+    def _add_environment_locals(terraform: dict, app: str):
+        terraform["locals"] = {
+            "config": '${yamldecode(file("../../../platform-config.yml"))}',
+            "environments": '${local.config["environments"]}',
+            "env_config": '${{for name, config in local.environments: name => merge(lookup(local.environments, "*", {}), config)}}',
+            "args": {
+                "application": app,
+                "services": '${local.config["extensions"]}',
+                "env_config": "${local.env_config}",
+            },
+        }
+
+    @staticmethod
+    def _add_moved(terraform, platform_config):
+        extensions_comment = "Moved extensions-tf to just extensions - this block tells terraform this. Can be removed once all services have moved to the new naming."
+        terraform["moved"] = [
+            {
+                "//": extensions_comment,
+                "from": "module.extensions-tf",
+                "to": "module.extensions",
+            }
+        ]
+
+        extensions = platform_config.get("extensions", {})
+        s3_extension_names = [
+            extension_name
+            for extension_name, extension in extensions.items()
+            if extension["type"] == "s3"
+        ]
+        s3_comment = "S3 bucket resources are now indexed. Can be removed once all services have moved to terraform-platform-modules 5.x."
+
+        for name in s3_extension_names:
+            resources = [
+                "aws_s3_bucket_server_side_encryption_configuration.encryption-config",
+                "aws_s3_bucket_policy.bucket-policy",
+                "aws_kms_key.kms-key",
+                "aws_kms_alias.s3-bucket",
+            ]
+            moves = [f'module.extensions.module.s3["{name}"].{resource}' for resource in resources]
+            for move in moves:
+                terraform["moved"].append(
+                    {
+                        "//": s3_comment,
+                        "from": move,
+                        "to": f"{move}[0]",
+                    }
+                )
+
+    def _write_terraform_json(self, terraform: dict, env_dir: str):
+        message = self.file_provider.mkfile(
+            str(Path(env_dir).absolute()),
+            "main.tf.json",
+            json.dumps(terraform, indent=2),
+            True,
+        )
+        self.io.info(message)
+
+    def _ensure_no_hcl_manifest_file(self, env_dir):
+        message = self.file_provider.delete_file(env_dir, "main.tf")
+        if message:
+            self.io.info(f"Manifest has moved to main.tf.json. {message}")

dbt_platform_helper/providers/validation.py

@@ -3,17 +3,3 @@ from dbt_platform_helper.platform_exception import PlatformException
 
 class ValidationException(PlatformException):
     pass
-
-
-class IncompatibleMajorVersionException(ValidationException):
-    def __init__(self, app_version: str, check_version: str):
-        super().__init__()
-        self.app_version = app_version
-        self.check_version = check_version
-
-
-class IncompatibleMinorVersionException(ValidationException):
-    def __init__(self, app_version: str, check_version: str):
-        super().__init__()
-        self.app_version = app_version
-        self.check_version = check_version

dbt_platform_helper/providers/version.py

@@ -0,0 +1,36 @@
+from abc import ABC
+
+import requests
+
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+
+
+class VersionProvider(ABC):
+    pass
+
+
+# TODO add timeouts and exception handling for requests
+# TODO Alternatively use the gitpython package?
+class GithubVersionProvider(VersionProvider):
+    @staticmethod
+    def get_latest_version(repo_name: str, tags: bool = False) -> SemanticVersion:
+        if tags:
+            tags_list = requests.get(f"https://api.github.com/repos/{repo_name}/tags").json()
+            versions = [SemanticVersion.from_string(v["name"]) for v in tags_list]
+            versions.sort(reverse=True)
+            return versions[0]
+
+        package_info = requests.get(
+            f"https://api.github.com/repos/{repo_name}/releases/latest"
+        ).json()
+        return SemanticVersion.from_string(package_info["tag_name"])
+
+
+class PyPiVersionProvider(VersionProvider):
+    @staticmethod
+    def get_latest_version(project_name: str) -> SemanticVersion:
+        package_info = requests.get(f"https://pypi.org/pypi/{project_name}/json").json()
+        released_versions = package_info["releases"].keys()
+        parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
+        parsed_released_versions.sort(reverse=True)
+        return parsed_released_versions[0]

dbt_platform_helper/providers/vpc.py

@@ -98,9 +98,4 @@ class VpcProvider:
 
         sec_groups = self._get_security_groups(app, env, vpc_id)
 
-        if not sec_groups:
-            raise SecurityGroupNotFoundException(
-                f"No matching security groups found in vpc '{vpc_name}'"
-            )
-
         return Vpc(vpc_id, public_subnets, private_subnets, sec_groups)

dbt_platform_helper/providers/yaml_file.py

@@ -19,11 +19,13 @@ class FileNotFoundException(FileProviderException):
 
 
 class InvalidYamlException(YamlFileProviderException):
-    pass
+    def __init__(self, path: str):
+        super().__init__(f"""{path} is not valid YAML.""")
 
 
 class DuplicateKeysException(YamlFileProviderException):
-    pass
+    def __init__(self, duplicate_keys: str):
+        super().__init__(f"""Duplicate keys found in your config file: {duplicate_keys}.""")
 
 
 class YamlFileProvider:
@@ -39,7 +41,7 @@ class YamlFileProvider:
         try:
             yaml_content = yaml.safe_load(Path(path).read_text())
         except ParserError:
-            raise InvalidYamlException(f"{path} is not valid YAML.")
+            raise InvalidYamlException(path)
 
         if not yaml_content:
             return {}

dbt_platform_helper/utils/application.py

@@ -10,11 +10,11 @@ import boto3
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
 from dbt_platform_helper.utils.aws import get_ssm_secrets
 from dbt_platform_helper.utils.messages import abort_with_error
-from dbt_platform_helper.utils.platform_config import load_unvalidated_config_file
 
 
 @dataclass
@@ -125,8 +125,9 @@ def load_application(app=None, default_session=None) -> Application:
 
 def get_application_name(abort=abort_with_error):
     if Path(PLATFORM_CONFIG_FILE).exists():
+        config = ConfigProvider()
         try:
-            app_config = load_unvalidated_config_file()
+            app_config = config.load_unvalidated_config_file()
             return app_config["application"]
         except KeyError:
             abort(