dbt-platform-helper 15.2.1__tar.gz → 15.3.0__tar.gz

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 15.2.1
+Version: 15.3.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/constants.py

@@ -1,6 +1,13 @@
 # TODO: DBTP-1888: Move to Config provider
 PLATFORM_CONFIG_FILE = "platform-config.yml"
 PLATFORM_HELPER_VERSION_OVERRIDE_KEY = "PLATFORM_HELPER_VERSION_OVERRIDE"
+TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR = "TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE"
+TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
+    "TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE"
+)
+TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
+    "TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE"
+)
 PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
 SUPPORTED_TERRAFORM_VERSION = "~> 1.8"
 SUPPORTED_AWS_PROVIDER_VERSION = "~> 5"

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/domain/pipelines.py

@@ -1,4 +1,3 @@
-import os
 from collections.abc import Callable
 from os import makedirs
 from pathlib import Path
@@ -9,8 +8,17 @@ from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
 from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
 from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
+from dbt_platform_helper.constants import (
+    TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
+)
+from dbt_platform_helper.constants import (
+    TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
+)
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.ecr import ECRProvider
+from dbt_platform_helper.providers.environment_variable import (
+    EnvironmentVariableProvider,
+)
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
@@ -28,6 +36,7 @@ class Pipelines:
         get_codestar_arn: Callable[[str], str],
         io: ClickIOProvider = ClickIOProvider(),
         file_provider: FileProvider = FileProvider(),
+        environment_variable_provider: EnvironmentVariableProvider = None,
         platform_helper_version_override: str = None,
     ):
         self.config_provider = config_provider
@@ -37,8 +46,12 @@ class Pipelines:
         self.ecr_provider = ecr_provider
         self.io = io
         self.file_provider = file_provider
-        self.platform_helper_version_override = platform_helper_version_override or os.environ.get(
-            PLATFORM_HELPER_VERSION_OVERRIDE_KEY
+        self.environment_variable_provider = (
+            environment_variable_provider or EnvironmentVariableProvider()
+        )
+        self.platform_helper_version_override = (
+            platform_helper_version_override
+            or self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
         )
 
     def generate(
@@ -86,6 +99,13 @@ class Pipelines:
             )
             deploy_repository = f"uktrade/{platform_config['application']}-deploy"
 
+        env_pipeline_module_source = (
+            self.environment_variable_provider.get(
+                TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
+            )
+            or f"git::git@github.com:uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref={platform_helper_version_for_template}"
+        )
+
         if has_environment_pipelines:
             environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
             accounts = {
@@ -99,7 +119,7 @@ class Pipelines:
                     platform_config["application"],
                     deploy_repository,
                     account,
-                    platform_helper_version_for_template,
+                    env_pipeline_module_source,
                     deploy_branch,
                 )
 
@@ -116,11 +136,19 @@ class Pipelines:
                 if repo in ecrs_already_provisioned
             }
 
+            codebase_pipeline_module_source = (
+                self.environment_variable_provider.get(
+                    TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
+                )
+                or f"git::git@github.com:uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref={platform_helper_version_for_template}"
+            )
+
             self.terraform_manifest_provider.generate_codebase_pipeline_config(
                 platform_config,
                 platform_helper_version_for_template,
                 ecrs_that_need_importing,
                 deploy_repository,
+                codebase_pipeline_module_source,
             )
 
     def _clean_pipeline_config(self, pipelines_dir: Path):
@@ -133,7 +161,7 @@ class Pipelines:
         application: str,
         deploy_repository: str,
         aws_account: str,
-        platform_helper_version: str,
+        module_source: str,
         deploy_branch: str,
     ):
         env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
@@ -143,7 +171,7 @@ class Pipelines:
                 "application": application,
                 "deploy_repository": deploy_repository,
                 "aws_account": aws_account,
-                "platform_helper_version": platform_helper_version,
+                "module_source": module_source,
                 "deploy_branch": deploy_branch,
                 "terraform_version": SUPPORTED_TERRAFORM_VERSION,
                 "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/domain/terraform_environment.py

@@ -1,7 +1,11 @@
-import os
-
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
+from dbt_platform_helper.constants import (
+    TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR,
+)
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.environment_variable import (
+    EnvironmentVariableProvider,
+)
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
 
@@ -19,14 +23,19 @@ class TerraformEnvironment:
         self,
         config_provider,
         manifest_provider: TerraformManifestProvider = None,
+        environment_variable_provider: EnvironmentVariableProvider = None,
         io: ClickIOProvider = ClickIOProvider(),
         platform_helper_version_override: str = None,
     ):
         self.io = io
         self.config_provider = config_provider
         self.manifest_provider = manifest_provider or TerraformManifestProvider()
-        self.platform_helper_version_override = platform_helper_version_override or os.environ.get(
-            PLATFORM_HELPER_VERSION_OVERRIDE_KEY
+        self.environment_variable_provider = (
+            environment_variable_provider or EnvironmentVariableProvider()
+        )
+        self.platform_helper_version_override = (
+            platform_helper_version_override
+            or self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
         )
 
     def generate(
@@ -48,6 +57,13 @@ class TerraformEnvironment:
         if self.platform_helper_version_override:
             platform_helper_version_for_template = self.platform_helper_version_override
 
+        module_source_override = self.environment_variable_provider.get(
+            TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR
+        )
+
         self.manifest_provider.generate_environment_config(
-            config, environment_name, platform_helper_version_for_template
+            config,
+            environment_name,
+            platform_helper_version_for_template,
+            module_source_override,
         )

dbt_platform_helper-15.3.0/dbt_platform_helper/providers/environment_variable.py

@@ -0,0 +1,24 @@
+import os
+from typing import Optional
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class EnvironmentVariableProvider:
+
+    @staticmethod
+    def get(env_var: str) -> Optional[str]:
+        """Returns the stripped value or None if not set or empty."""
+        value = os.environ.get(env_var)
+        if value and value.strip():
+            return value.strip()
+        return None
+
+    @staticmethod
+    def get_required(env_var: str) -> str:
+        """Returns the stripped value or raises a PlatformException if not set
+        or empty."""
+        value = os.environ.get(env_var)
+        if not value or not value.strip():
+            raise PlatformException(f"Environment variable '{env_var}' is not set or is empty")
+        return value.strip()

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/providers/files.py

@@ -17,7 +17,7 @@ class FileProvider:
         file_path.write_text(contents)
 
         action = "overwritten" if file_exists and overwrite else "created"
-        return f"File {file_name} {action}"
+        return f"File {file_path} {action}"
 
     @staticmethod
     def delete_file(base_path: str, file_name: str):

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/providers/io.py

@@ -1,18 +1,21 @@
-import os
-
 import click
 
 from dbt_platform_helper.platform_exception import PlatformException
-
-DEBUG = os.environ.get("DEBUG", False)
+from dbt_platform_helper.providers.environment_variable import (
+    EnvironmentVariableProvider,
+)
 
 
 class ClickIOProvider:
+    def __init__(self, env_var_provider=EnvironmentVariableProvider()):
+        self.env_var_provider = env_var_provider
+        self.debug_flag = self.env_var_provider.get("DEBUG")
+
     def warn(self, message: str):
         click.secho(message, fg="magenta")
 
     def debug(self, message: str):
-        if DEBUG == "True":
+        if self.debug_flag and self.debug_flag.strip().upper() == "TRUE":
             click.secho(message, fg="green")
 
     def error(self, message: str):

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/providers/load_balancers.py

@@ -28,7 +28,8 @@ class LoadBalancerProvider:
     def find_target_group(self, app: str, env: str, svc: str) -> str:
         target_group_arn = None
 
-        response = self.rg_tagging_client.get_resources(
+        paginator = self.rg_tagging_client.get_paginator("get_resources")
+        page_iterator = paginator.paginate(
             TagFilters=[
                 {
                     "Key": "copilot-application",
@@ -48,19 +49,18 @@ class LoadBalancerProvider:
             ResourceTypeFilters=[
                 "elasticloadbalancing:targetgroup",
             ],
-        )  # TODO: DBTP-1942: should be paginated
-        for resource in response["ResourceTagMappingList"]:
-            tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
-
-            if (
-                "copilot-service" in tags
-                and tags["copilot-service"] == svc
-                and "copilot-environment" in tags
-                and tags["copilot-environment"] == env
-                and "copilot-application" in tags
-                and tags["copilot-application"] == app
-            ):
-                target_group_arn = resource["ResourceARN"]
+        )
+
+        for page in page_iterator:
+            for resource in page["ResourceTagMappingList"]:
+                tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
+
+                if (
+                    tags.get("copilot-service") == svc
+                    and tags.get("copilot-environment") == env
+                    and tags.get("copilot-application") == app
+                ):
+                    target_group_arn = resource["ResourceARN"]
 
         if not target_group_arn:
             self.io.error(
@@ -69,11 +69,12 @@ class LoadBalancerProvider:
 
         return target_group_arn
 
-    def get_https_certificate_for_application(self, app: str, env: str) -> str:
-        listener_arn = self.get_https_listener_for_application(app, env)
-        certificates = self.evlb_client.describe_listener_certificates(ListenerArn=listener_arn)[
-            "Certificates"
-        ]  # TODO: DBTP-1942: should be paginated
+    def get_https_certificate_for_listener(self, listener_arn: str, env: str):
+        certificates = []
+        paginator = self.evlb_client.get_paginator("describe_listener_certificates")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            certificates.extend(page["Certificates"])
 
         try:
             certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
@@ -82,12 +83,22 @@ class LoadBalancerProvider:
 
         return certificate_arn
 
+    def get_https_certificate_for_application(self, app: str, env: str) -> str:
+        listener_arn = self.get_https_listener_for_application(app, env)
+        return self.get_https_certificate_for_listener(listener_arn, env)
+
+    def get_listeners_for_load_balancer(self, load_balancer_arn):
+        listeners = []
+        paginator = self.evlb_client.get_paginator("describe_listeners")
+        page_iterator = paginator.paginate(LoadBalancerArn=load_balancer_arn)
+        for page in page_iterator:
+            listeners.extend(page["Listeners"])
+
+        return listeners
+
     def get_https_listener_for_application(self, app: str, env: str) -> str:
         load_balancer_arn = self.get_load_balancer_for_application(app, env)
-
-        listeners = self.evlb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)[
-            "Listeners"
-        ]  # TODO: DBTP-1942: should be paginated
+        listeners = self.get_listeners_for_load_balancer(load_balancer_arn)
 
         listener_arn = None
 
@@ -101,16 +112,23 @@ class LoadBalancerProvider:
 
         return listener_arn
 
-    def get_load_balancer_for_application(self, app: str, env: str) -> str:
-        describe_response = self.evlb_client.describe_load_balancers()
-        load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
+    def get_load_balancers(self):
+        load_balancers = []
+        paginator = self.evlb_client.get_paginator("describe_load_balancers")
+        page_iterator = paginator.paginate()
+        for page in page_iterator:
+            load_balancers.extend(lb["LoadBalancerArn"] for lb in page["LoadBalancers"])
 
+        return load_balancers
+
+    def get_load_balancer_for_application(self, app: str, env: str) -> str:
+        load_balancers = self.get_load_balancers()
         tag_descriptions = []
         for i in range(0, len(load_balancers), 20):
             chunk = load_balancers[i : i + 20]
             tag_descriptions.extend(
                 self.evlb_client.describe_tags(ResourceArns=chunk)["TagDescriptions"]
-            )
+            )  # describe_tags cannot be paginated - 04/04/2025
 
         for lb in tag_descriptions:
             tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
@@ -121,9 +139,11 @@ class LoadBalancerProvider:
         raise LoadBalancerNotFoundException(app, env)
 
     def get_host_header_conditions(self, listener_arn: str, target_group_arn: str) -> list:
-        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
-            "Rules"
-        ]  # TODO: DBTP-1942: should be paginated
+        rules = []
+        paginator = self.evlb_client.get_paginator("describe_rules")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            rules.extend(page["Rules"])
 
         conditions = []
 
@@ -150,11 +170,19 @@ class LoadBalancerProvider:
         return conditions
 
     def get_rules_tag_descriptions_by_listener_arn(self, listener_arn: str) -> list:
-        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
-            "Rules"
-        ]  # TODO: DBTP-1942: should be paginated
+        rules = self.get_listener_rules_by_listener_arn(listener_arn)
+
         return self.get_rules_tag_descriptions(rules)
 
+    def get_listener_rules_by_listener_arn(self, listener_arn: str) -> list:
+        rules = []
+        paginator = self.evlb_client.get_paginator("describe_rules")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            rules.extend(page["Rules"])
+
+        return rules
+
     def get_rules_tag_descriptions(self, rules: list) -> list:
         tag_descriptions = []
         chunk_size = 20
@@ -164,7 +192,7 @@ class LoadBalancerProvider:
             resource_arns = [r["RuleArn"] for r in chunk]
             response = self.evlb_client.describe_tags(
                 ResourceArns=resource_arns
-            )  # TODO should be paginated
+            )  # describe_tags cannot be paginated - 04/04/2025
             tag_descriptions.extend(response["TagDescriptions"])
 
         return tag_descriptions

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/providers/terraform_manifest.py

@@ -23,6 +23,7 @@ class TerraformManifestProvider:
         platform_helper_version: str,
         ecr_imports: dict[str, str],
         deploy_repository: str,
+        module_source: str,
     ):
         default_account = self._get_account_for_env("*", platform_config)
         state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
@@ -32,7 +33,9 @@ class TerraformManifestProvider:
         self._add_codebase_pipeline_locals(terraform)
         self._add_provider(terraform, default_account)
         self._add_backend(terraform, platform_config, default_account, state_key_suffix)
-        self._add_codebase_pipeline_module(terraform, platform_helper_version, deploy_repository)
+        self._add_codebase_pipeline_module(
+            terraform, platform_helper_version, deploy_repository, module_source
+        )
         self._add_imports(terraform, ecr_imports)
         self._write_terraform_json(terraform, "terraform/codebase-pipelines")
 
@@ -41,6 +44,7 @@ class TerraformManifestProvider:
         platform_config: dict,
         env: str,
         platform_helper_version: str,
+        module_source_override: str = None,
     ):
         platform_config = ConfigProvider.apply_environment_defaults(platform_config)
         account = self._get_account_for_env(env, platform_config)
@@ -53,7 +57,7 @@ class TerraformManifestProvider:
         self._add_header(terraform)
         self._add_environment_locals(terraform, application_name)
         self._add_backend(terraform, platform_config, account, state_key_suffix)
-        self._add_extensions_module(terraform, platform_helper_version, env)
+        self._add_extensions_module(terraform, platform_helper_version, env, module_source_override)
         self._add_moved(terraform, platform_config)
         self._ensure_no_hcl_manifest_file(env_dir)
         self._write_terraform_json(terraform, env_dir)
@@ -115,9 +119,12 @@ class TerraformManifestProvider:
 
     @staticmethod
     def _add_codebase_pipeline_module(
-        terraform: dict, platform_helper_version: str, deploy_repository: str
+        terraform: dict,
+        platform_helper_version: str,
+        deploy_repository: str,
+        module_source: str,
     ):
-        source = f"git::https://github.com/uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref={platform_helper_version}"
+        source = module_source
         terraform["module"] = {
             "codebase-pipelines": {
                 "source": source,
@@ -133,12 +140,18 @@ class TerraformManifestProvider:
                 "requires_image_build": '${lookup(each.value, "requires_image_build", true)}',
                 "slack_channel": '${lookup(each.value, "slack_channel", "/codebuild/slack_oauth_channel")}',
                 "env_config": "${local.environments}",
+                "platform_tools_version": f"{platform_helper_version}",
             }
         }
 
     @staticmethod
-    def _add_extensions_module(terraform: dict, platform_helper_version: str, env: str):
-        source = f"git::https://github.com/uktrade/platform-tools.git//terraform/extensions?depth=1&ref={platform_helper_version}"
+    def _add_extensions_module(
+        terraform: dict, platform_helper_version: str, env: str, module_source_override: str = None
+    ):
+        source = (
+            module_source_override
+            or f"git::git@github.com:uktrade/platform-tools.git//terraform/extensions?depth=1&ref={platform_helper_version}"
+        )
         terraform["module"] = {
             "extensions": {
                 "source": source,
@@ -209,7 +222,7 @@ class TerraformManifestProvider:
 
     def _write_terraform_json(self, terraform: dict, env_dir: str):
         message = self.file_provider.mkfile(
-            str(Path(env_dir).absolute()),
+            str(Path(env_dir)),
             "main.tf.json",
             json.dumps(terraform, indent=2),
             True,

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -34,7 +34,7 @@ terraform {
 
 
 module "environment-pipelines" {
-  source = "git::https://github.com/uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref={{ platform_helper_version }}"
+  source = "{{ module_source }}"
 
   for_each = local.pipelines
 

{dbt_platform_helper-15.2.1 → dbt_platform_helper-15.3.0}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "15.2.1"
+version = "15.3.0"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"