dbt-platform-helper 13.0.2__tar.gz → 13.1.1__tar.gz

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 13.0.2
+Version: 13.1.1
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/commands/copilot.py

@@ -10,9 +10,9 @@ import yaml
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/commands/environment.py

@@ -1,13 +1,13 @@
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
-from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
 from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.cloudformation import CloudFormation
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.vpc import VpcProvider
 from dbt_platform_helper.utils.application import load_application

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/commands/pipeline.py

@@ -2,9 +2,9 @@
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
-from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.pipelines import Pipelines
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.ecr import ECRProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/domain/copilot_environment.py

@@ -132,7 +132,7 @@ class CopilotEnvironment:
 class CopilotTemplating:
     def __init__(
         self,
-        file_provider: FileProvider = None,
+        file_provider: FileProvider = FileProvider(),
         io: ClickIOProvider = ClickIOProvider(),
         # TODO file_provider can be moved up a layer.  File writing can be the responsibility of CopilotEnvironment generate
         # Or we align with PlatformTerraformManifestGenerator and rename from Templating to reflect the file writing responsibility

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/domain/database_copy.py

@@ -6,9 +6,9 @@ import boto3
 from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.maintenance_page import MaintenancePage
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.io import ClickIOProviderException
 from dbt_platform_helper.providers.vpc import Vpc

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/domain/pipelines.py

@@ -72,6 +72,16 @@ class Pipelines:
             platform_config_terraform_modules_default_version,
         )
 
+        # TODO - this whole code block/if-statement can fall away once the deploy_repository is a required key.
+        deploy_repository = ""
+        if "deploy_repository" in platform_config.keys():
+            deploy_repository = f"{platform_config['deploy_repository']}"
+        else:
+            self.io.warn(
+                "No `deploy_repository` key set in platform-config.yml, this will become a required key. See full platform config reference in the docs: https://platform.readme.trade.gov.uk/reference/platform-config-yml/#core-configuration"
+            )
+            deploy_repository = f"uktrade/{platform_config['application']}-deploy"
+
         if has_environment_pipelines:
             environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
             accounts = {
@@ -83,6 +93,7 @@ class Pipelines:
             for account in accounts:
                 self._generate_terraform_environment_pipeline_manifest(
                     platform_config["application"],
+                    deploy_repository,
                     account,
                     terraform_platform_modules_version,
                     deploy_branch,
@@ -102,7 +113,10 @@ class Pipelines:
             }
 
             self.terraform_manifest_provider.generate_codebase_pipeline_config(
-                platform_config, terraform_platform_modules_version, ecrs_that_need_importing
+                platform_config,
+                terraform_platform_modules_version,
+                ecrs_that_need_importing,
+                deploy_repository,
             )
 
     def _clean_pipeline_config(self, pipelines_dir: Path):
@@ -113,6 +127,7 @@ class Pipelines:
     def _generate_terraform_environment_pipeline_manifest(
         self,
         application: str,
+        deploy_repository: str,
         aws_account: str,
         terraform_platform_modules_version: str,
         deploy_branch: str,
@@ -122,6 +137,7 @@ class Pipelines:
         contents = env_pipeline_template.render(
             {
                 "application": application,
+                "deploy_repository": deploy_repository,
                 "aws_account": aws_account,
                 "terraform_platform_modules_version": terraform_platform_modules_version,
                 "deploy_branch": deploy_branch,

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/providers/config.py

@@ -4,8 +4,8 @@ from pathlib import Path
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.domain.config_validator import ConfigValidator
-from dbt_platform_helper.domain.config_validator import ConfigValidatorError
+from dbt_platform_helper.providers.config_validator import ConfigValidator
+from dbt_platform_helper.providers.config_validator import ConfigValidatorError
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
 from dbt_platform_helper.providers.yaml_file import FileNotFoundException

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/providers/platform_config_schema.py

@@ -14,8 +14,8 @@ class PlatformConfigSchema:
     def schema() -> Schema:
         return Schema(
             {
-                # The following line is for the AWS Copilot version, will be removed under DBTP-1002
                 "application": str,
+                Optional("deploy_repository"): str,
                 Optional("default_versions"): PlatformConfigSchema.__default_versions_schema(),
                 Optional("environments"): PlatformConfigSchema.__environments_schema(),
                 Optional("codebase_pipelines"): PlatformConfigSchema.__codebase_pipelines_schema(),
@@ -134,7 +134,7 @@ class PlatformConfigSchema:
                 Optional("additional_ecr_repository"): str,
                 Optional("deploy_repository_branch"): str,
                 "services": [{str: [str]}],
-                "pipelines": [
+                Optional("pipelines"): [
                     Or(
                         {
                             "name": str,
@@ -435,7 +435,7 @@ class PlatformConfigSchema:
         return True
 
     @staticmethod
-    def __valid_s3_base_definition() -> dict:
+    def __s3_bucket_schema() -> dict:
         def _valid_s3_bucket_arn(key):
             return Regex(
                 r"^arn:aws:s3::.*",
@@ -485,6 +485,10 @@ class PlatformConfigSchema:
 
         return dict(
             {
+                "type": "s3",
+                Optional("objects"): [
+                    {"key": str, Optional("body"): str, Optional("content_type"): str}
+                ],
                 Optional("readonly"): bool,
                 Optional("serve_static_content"): bool,
                 Optional("serve_static_param_name"): str,
@@ -502,7 +506,9 @@ class PlatformConfigSchema:
                         },
                         Optional("cross_environment_service_access"): {
                             PlatformConfigSchema.__valid_schema_key(): {
-                                "application": str,
+                                # Deprecated: We didn't implement cross application access, no service teams are asking for it.
+                                # application should be removed once we can confirm that no-one is using it.
+                                Optional("application"): str,
                                 "environment": PlatformConfigSchema.__valid_environment_name(),
                                 "account": str,
                                 "service": str,
@@ -518,18 +524,19 @@ class PlatformConfigSchema:
             }
         )
 
-    @staticmethod
-    def __s3_bucket_schema() -> dict:
-        return PlatformConfigSchema.__valid_s3_base_definition() | {
-            "type": "s3",
-            Optional("objects"): [
-                {"key": str, Optional("body"): str, Optional("content_type"): str}
-            ],
-        }
-
     @staticmethod
     def __s3_bucket_policy_schema() -> dict:
-        return PlatformConfigSchema.__valid_s3_base_definition() | {"type": "s3-policy"}
+        return dict(
+            {
+                "type": "s3-policy",
+                Optional("services"): Or("__all__", [str]),
+                Optional("environments"): {
+                    PlatformConfigSchema.__valid_environment_name(): {
+                        "bucket_name": PlatformConfigSchema.valid_s3_bucket_name,
+                    },
+                },
+            }
+        )
 
     @staticmethod
     def string_matching_regex(regex_pattern: str) -> Callable:

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/providers/terraform_manifest.py

@@ -22,6 +22,7 @@ class TerraformManifestProvider:
         platform_config: dict,
         terraform_platform_modules_version: str,
         ecr_imports: dict[str, str],
+        deploy_repository: str,
     ):
         default_account = self._get_account_for_env("*", platform_config)
         state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
@@ -31,7 +32,9 @@ class TerraformManifestProvider:
         self._add_codebase_pipeline_locals(terraform)
         self._add_provider(terraform, default_account)
         self._add_backend(terraform, platform_config, default_account, state_key_suffix)
-        self._add_codebase_pipeline_module(terraform, terraform_platform_modules_version)
+        self._add_codebase_pipeline_module(
+            terraform, terraform_platform_modules_version, deploy_repository
+        )
         self._add_imports(terraform, ecr_imports)
         self._write_terraform_json(terraform, "terraform/codebase-pipelines")
 
@@ -113,7 +116,9 @@ class TerraformManifestProvider:
         }
 
     @staticmethod
-    def _add_codebase_pipeline_module(terraform: dict, terraform_platform_modules_version: str):
+    def _add_codebase_pipeline_module(
+        terraform: dict, terraform_platform_modules_version: str, deploy_repository: str
+    ):
         source = f"git::https://github.com/uktrade/terraform-platform-modules.git//codebase-pipelines?depth=1&ref={terraform_platform_modules_version}"
         terraform["module"] = {
             "codebase-pipelines": {
@@ -122,8 +127,9 @@ class TerraformManifestProvider:
                 "application": "${local.application}",
                 "codebase": "${each.key}",
                 "repository": "${each.value.repository}",
+                "deploy_repository": f"{deploy_repository}",
                 "additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
-                "pipelines": "${each.value.pipelines}",
+                "pipelines": '${lookup(each.value, "pipelines", [])}',
                 "services": "${each.value.services}",
                 "requires_image_build": '${lookup(each.value, "requires_image_build", true)}',
                 "slack_channel": '${lookup(each.value, "slack_channel", "/codebuild/slack_oauth_channel")}',

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -40,7 +40,7 @@ module "environment-pipelines" {
 
   application         = "{{ application }}"
   pipeline_name       = each.key
-  repository          = "uktrade/{{ application }}-deploy"
+  repository          = "{{ deploy_repository }}"
 
   environments        = each.value.environments
   all_pipelines       = local.all_pipelines

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/utils/validation.py

@@ -1,6 +1,6 @@
 from schema import SchemaError
 
-from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
 
 

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/dbt_platform_helper/utils/versioning.py

@@ -5,8 +5,6 @@ from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version
 from pathlib import Path
 
-import click
-
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
 from dbt_platform_helper.platform_exception import PlatformException
@@ -137,23 +135,21 @@ def get_platform_helper_versions(
 
 
 # Validates the returned PlatformHelperVersionStatus and echos useful warnings
-# Should use IO provider
 # Could return ValidationMessages (warnings and errors) which are output elsewhere
-def _process_version_file_warnings(versions: PlatformHelperVersionStatus):
+def _process_version_file_warnings(versions: PlatformHelperVersionStatus, io=ClickIOProvider()):
     messages = versions.warn()
 
     if messages.get("errors"):
-        click.secho("\n".join(messages["errors"]), fg="red")
+        io.error("\n".join(messages["errors"]))
 
     if messages.get("warnings"):
-        click.secho("\n".join(messages["warnings"]), fg="yellow")
+        io.warn("\n".join(messages["warnings"]))
 
 
 # TODO called at the beginning of every command.  This is platform-version base functionality
-def check_platform_helper_version_needs_update():
+def check_platform_helper_version_needs_update(io=ClickIOProvider()):
     if not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ:
         return
-
     versions = get_platform_helper_versions(include_project_versions=False)
     local_version = versions.local
     latest_release = versions.latest
@@ -165,9 +161,9 @@ def check_platform_helper_version_needs_update():
     try:
         local_version.validate_compatibility_with(latest_release)
     except IncompatibleMajorVersionException:
-        click.secho(message, fg="red")
+        io.error(message)
     except IncompatibleMinorVersionException:
-        click.secho(message, fg="yellow")
+        io.warn(message)
 
 
 # TODO can stay as utility for now

{dbt_platform_helper-13.0.2 → dbt_platform_helper-13.1.1}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "13.0.2"
+version = "13.1.1"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"