dbt-platform-helper 15.3.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/providers/ecs.py

@@ -1,7 +1,10 @@
 import random
 import string
 import subprocess
-from typing import List
+from typing import Any
+from typing import Optional
+
+from botocore.exceptions import ClientError
 
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.platform_exception import ValidationException
@@ -39,7 +42,7 @@ class ECS:
         container_name: str,
         task_def_arn: str,
         vpc_config: Vpc,
-        env_vars: List[dict] = None,
+        env_vars: list[dict] = None,
     ):
         container_override = {"name": container_name}
         if env_vars:
@@ -107,13 +110,31 @@ class ECS:
             random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
             return f"conduit-{self.application_name}-{self.env}-{addon_name}-{random_id}"
 
-    def get_ecs_task_arns(self, cluster_arn: str, task_def_family: str):
-        """Gets the ECS task ARNs for a given task name and cluster ARN."""
-        tasks = self.ecs_client.list_tasks(
-            cluster=cluster_arn,
-            desiredStatus="RUNNING",
-            family=task_def_family,
-        )
+    def get_ecs_task_arns(
+        self,
+        cluster: str,
+        max_results: int = 100,
+        desired_status: str = "RUNNING",
+        service_name: Optional[str] = None,
+        started_by: Optional[str] = None,
+        task_def_family: Optional[str] = None,
+    ) -> list[str]:
+        """Returns the ECS task ARNs based on the parameters provided."""
+
+        params = {
+            "cluster": cluster,
+            "maxResults": max_results,
+            "desiredStatus": desired_status,
+        }
+
+        if service_name:
+            params["serviceName"] = service_name
+        if started_by:
+            params["startedBy"] = started_by
+        if task_def_family:
+            params["family"] = task_def_family
+
+        tasks = self.ecs_client.list_tasks(**params)
 
         if not tasks["taskArns"]:
             return []
@@ -137,7 +158,7 @@ class ECS:
         exceptions_to_catch=(ECSException,),
         message_on_false="ECS Agent Not running",
     )
-    def ecs_exec_is_available(self, cluster_arn: str, task_arns: List[str]) -> bool:
+    def ecs_exec_is_available(self, cluster_arn: str, task_arns: list[str]) -> bool:
         """
         Checks if the ExecuteCommandAgent is running on the specified ECS task.
 
@@ -168,7 +189,106 @@ class ECS:
         message_on_false="ECS task did not register in time",
     )
     def wait_for_task_to_register(self, cluster_arn: str, task_family: str) -> list[str]:
-        task_arns = self.get_ecs_task_arns(cluster_arn, task_family)
+        task_arns = self.get_ecs_task_arns(cluster=cluster_arn, task_def_family=task_family)
         if task_arns:
             return task_arns
         return False
+
+    def get_service_deployment_state(
+        self, cluster_name: str, service_name: str, start_time: float
+    ) -> tuple[Optional[str], Optional[str]]:
+        """
+        Returns status & statusReason for the deployment of an ECS service.
+
+        statusReason can be:
+            PENDING | SUCCESSFUL | STOPPED | STOP_REQUESTED |
+            IN_PROGRESS | ROLLBACK_REQUESTED | ROLLBACK_IN_PROGRESS |
+            ROLLBACK_SUCCESSFUL | ROLLBACK_FAILED
+        """
+        resp = self.ecs_client.list_service_deployments(
+            cluster=cluster_name, service=service_name, createdAt={"after": start_time}
+        )
+        deployments = resp.get("serviceDeployments", [])
+
+        if not deployments:
+            return None, f"No deployments found for '{service_name}'"
+
+        return deployments[0].get("status"), deployments[0].get("statusReason")
+
+    def get_container_names_from_ecs_tasks(
+        self, cluster_name: str, task_ids: list[str]
+    ) -> list[str]:
+        """Retrieve container names from each ECS task provided."""
+
+        response = self.ecs_client.describe_tasks(cluster=cluster_name, tasks=task_ids)
+
+        names = []
+        for task in response.get("tasks", []):
+            for container in task.get("containers", []):
+                if container["name"] not in names:
+                    names.append(container["name"])
+        return names
+
+    def register_task_definition(
+        self,
+        service: str,
+        task_definition: dict,
+        image_tag: Optional[str] = None,
+    ) -> str:
+        """Register a new task definition revision using provided model and
+        containerDefinitions."""
+
+        for container in task_definition["containerDefinitions"]:
+            if container["name"] == service:
+                container["image"] = f"{container['image']}:{image_tag}"
+                break
+
+        try:
+            task_definition_response = self.ecs_client.register_task_definition(**task_definition)
+            return task_definition_response["taskDefinition"]["taskDefinitionArn"]
+        except ClientError as err:
+            raise PlatformException(f"Error registering task definition: {err}")
+
+    def update_service(
+        self,
+        service: str,
+        task_def_arn: str,
+        environment: str,
+        application: str,
+        desired_count: int,
+    ) -> dict[str, Any]:
+        """Update an ECS service and return the response."""
+
+        try:
+            service_response = self.ecs_client.update_service(
+                cluster=f"{application}-{environment}-cluster",
+                service=f"{application}-{environment}-{service}",
+                taskDefinition=task_def_arn,
+                desiredCount=desired_count,
+            )
+            return service_response["service"]
+        except ClientError as err:
+            raise PlatformException(f"Error updating ECS service: {err}")
+
+    def describe_service(self, service: str, environment: str, application: str) -> dict[str, Any]:
+        """Return information about an ECS service."""
+
+        try:
+            service_response = self.ecs_client.describe_services(
+                cluster=f"{application}-{environment}-cluster",
+                services=[
+                    f"{application}-{environment}-{service}",
+                ],
+            )
+            return service_response["services"][0]
+        except ClientError as err:
+            raise PlatformException(f"Error retrieving ECS service: {err}")
+
+    def describe_tasks(self, cluster_name: str, task_ids: list[str]) -> list[dict[str, Any]]:
+        """Return information about ECS tasks."""
+
+        try:
+            response = self.ecs_client.describe_tasks(cluster=cluster_name, tasks=task_ids)
+            return response["tasks"]
+        except ClientError as err:
+            raise PlatformException(f"Error retrieving ECS tasks: {err}")

dbt_platform_helper/providers/environment_variable.py

@@ -7,12 +7,12 @@ from dbt_platform_helper.platform_exception import PlatformException
 class EnvironmentVariableProvider:
 
     @staticmethod
-    def get(env_var: str) -> Optional[str]:
+    def get(env_var: str, default=None) -> Optional[str]:
         """Returns the stripped value or None if not set or empty."""
         value = os.environ.get(env_var)
         if value and value.strip():
             return value.strip()
-        return None
+        return default
 
     @staticmethod
     def get_required(env_var: str) -> str:

dbt_platform_helper/providers/io.py

@@ -24,8 +24,12 @@ class ClickIOProvider:
     def info(self, message: str, **kwargs):
         click.secho(message, **kwargs)
 
-    def input(self, message: str) -> str:
-        return click.prompt(message)
+    def input(
+        self, message: str, hide_input=False, confirmation_prompt=False, input_type=str
+    ) -> str:
+        return click.prompt(
+            message, hide_input=hide_input, confirmation_prompt=confirmation_prompt, type=input_type
+        )
 
     def confirm(self, message: str) -> bool:
         try:
@@ -37,6 +41,9 @@ class ClickIOProvider:
         click.secho(f"Error: {message}", err=True, fg="red")
         exit(1)
 
+    def deploy_error(self, message: str):
+        click.secho(message, fg="red")
+
     # TODO: DBTP-1979: messages will be a ValidationMessages class rather than a free-rein dictionary
     def process_messages(self, messages: dict):
         if not messages:

dbt_platform_helper/providers/load_balancers.py

@@ -1,7 +1,14 @@
+import json
+from typing import Dict
+from typing import List
+
 from boto3 import Session
 
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM_TERRAFORM
+from dbt_platform_helper.constants import ROUTED_TO_PLATFORM_MODES
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 
 
@@ -12,12 +19,24 @@ def normalise_to_cidr(ip: str):
     return f"{ip}/{SINGLE_IPV4_CIDR_PREFIX_LENGTH}"
 
 
+class ALBDataNormaliser:
+
+    @staticmethod
+    def tags_to_dict(tags: List[Dict[str, str]]) -> Dict[str, str]:
+        return {tag.get("Key", ""): tag.get("Value", "") for tag in tags}
+
+    @staticmethod
+    def conditions_to_dict(conditions: List[Dict[str, List[str]]]) -> Dict[str, List[str]]:
+        return {condition.get("Field", ""): condition.get("Values", "") for condition in conditions}
+
+
 class LoadBalancerProvider:
 
     def __init__(self, session: Session = None, io: ClickIOProvider = ClickIOProvider()):
         self.session = session
         self.evlb_client = self._get_client("elbv2")
         self.rg_tagging_client = self._get_client("resourcegroupstaggingapi")
+        self.parameter_store_provider = ParameterStore(self._get_client("ssm"))
         self.io = io
 
     def _get_client(self, client: str):
@@ -26,21 +45,44 @@ class LoadBalancerProvider:
         return self.session.client(client)
 
     def find_target_group(self, app: str, env: str, svc: str) -> str:
+
+        # TODO once copilot is gone this is no longer needed
+        try:
+            result = self.parameter_store_provider.get_ssm_parameter_by_name(
+                f"/platform/applications/{app}/environments/{env}"
+            )["Value"]
+            env_config = json.loads(result)
+            service_deployment_mode = env_config["service_deployment_mode"]
+        except Exception:
+            service_deployment_mode = "copilot"
+
+        if service_deployment_mode in ROUTED_TO_PLATFORM_MODES:
+            application_key = "application"
+            environment_key = "environment"
+            service_key = "service"
+        else:
+            application_key = "copilot-application"
+            environment_key = "copilot-environment"
+            service_key = "copilot-service"
         target_group_arn = None
 
         paginator = self.rg_tagging_client.get_paginator("get_resources")
         page_iterator = paginator.paginate(
             TagFilters=[
                 {
-                    "Key": "copilot-application",
+                    "Key": application_key,
                     "Values": [
                         app,
                     ],
-                    "Key": "copilot-environment",
+                },
+                {
+                    "Key": environment_key,
                     "Values": [
                         env,
                     ],
-                    "Key": "copilot-service",
+                },
+                {
+                    "Key": service_key,
                     "Values": [
                         svc,
                     ],
@@ -56,9 +98,9 @@ class LoadBalancerProvider:
                 tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
 
                 if (
-                    tags.get("copilot-service") == svc
-                    and tags.get("copilot-environment") == env
-                    and tags.get("copilot-application") == app
+                    tags.get(service_key) == svc
+                    and tags.get(environment_key) == env
+                    and tags.get(application_key) == app
                 ):
                     target_group_arn = resource["ResourceARN"]
 
@@ -69,6 +111,29 @@ class LoadBalancerProvider:
 
         return target_group_arn
 
+    def get_target_groups(self, target_group_arns: List[str]) -> List[dict]:
+        tgs = []
+        paginator = self.evlb_client.get_paginator("describe_target_groups")
+        page_iterator = paginator.paginate(TargetGroupArns=target_group_arns)
+        for page in page_iterator:
+            tgs.extend(page["TargetGroups"])
+
+        return tgs
+
+    def get_target_groups_with_tags(
+        self, target_group_arns: List[str], normalise: bool = True
+    ) -> List[dict]:
+        target_groups = self.get_target_groups(target_group_arns)
+
+        tags = self.get_resources_tag_descriptions(target_groups, "TargetGroupArn")
+
+        tgs_with_tags = self.merge_in_tags_by_resource_arn(target_groups, tags, "TargetGroupArn")
+
+        if normalise:
+            for tg in tgs_with_tags:
+                tg["Tags"] = ALBDataNormaliser.tags_to_dict(tg["Tags"])
+        return tgs_with_tags
+
     def get_https_certificate_for_listener(self, listener_arn: str, env: str):
         certificates = []
         paginator = self.evlb_client.get_paginator("describe_listener_certificates")
@@ -87,7 +152,7 @@ class LoadBalancerProvider:
         listener_arn = self.get_https_listener_for_application(app, env)
         return self.get_https_certificate_for_listener(listener_arn, env)
 
-    def get_listeners_for_load_balancer(self, load_balancer_arn):
+    def get_listeners_for_load_balancer(self, load_balancer_arn: str) -> List[dict]:
         listeners = []
         paginator = self.evlb_client.get_paginator("describe_listeners")
         page_iterator = paginator.paginate(LoadBalancerArn=load_balancer_arn)
@@ -98,6 +163,7 @@ class LoadBalancerProvider:
 
     def get_https_listener_for_application(self, app: str, env: str) -> str:
         load_balancer_arn = self.get_load_balancer_for_application(app, env)
+        self.io.debug(f"Load Balancer ARN: {load_balancer_arn}")
         listeners = self.get_listeners_for_load_balancer(load_balancer_arn)
 
         listener_arn = None
@@ -112,7 +178,7 @@ class LoadBalancerProvider:
 
         return listener_arn
 
-    def get_load_balancers(self):
+    def get_load_balancers(self) -> List[dict]:
         load_balancers = []
         paginator = self.evlb_client.get_paginator("describe_load_balancers")
         page_iterator = paginator.paginate()
@@ -133,7 +199,11 @@ class LoadBalancerProvider:
         for lb in tag_descriptions:
             tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
             # TODO: DBTP-1967: copilot hangover, creates coupling to specific tags could update to check application and environment
-            if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
+            if (
+                tags.get("copilot-application") == app
+                and tags.get("copilot-environment") == env
+                and tags.get("managed-by", "") == MANAGED_BY_PLATFORM_TERRAFORM
+            ):
                 return lb["ResourceArn"]
 
         raise LoadBalancerNotFoundException(app, env)
@@ -149,8 +219,9 @@ class LoadBalancerProvider:
 
         for rule in rules:
             for action in rule["Actions"]:
-                if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
-                    conditions = rule["Conditions"]
+                if "TargetGroupArn" in action:
+                    if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                        conditions = rule["Conditions"]
 
         if not conditions:
             raise ListenerRuleConditionsNotFoundException(listener_arn)
@@ -172,7 +243,37 @@ class LoadBalancerProvider:
     def get_rules_tag_descriptions_by_listener_arn(self, listener_arn: str) -> list:
         rules = self.get_listener_rules_by_listener_arn(listener_arn)
 
-        return self.get_rules_tag_descriptions(rules)
+        return self.get_resources_tag_descriptions(rules)
+
+    def merge_in_tags_by_resource_arn(
+        self,
+        resources: List[dict],
+        tag_descriptions: List[dict],
+        resources_identifier: str = "RuleArn",
+    ):
+        tags_by_resource_arn = {
+            rule_tags.get("ResourceArn"): rule_tags for rule_tags in tag_descriptions if rule_tags
+        }
+        for resource in resources:
+            tags = tags_by_resource_arn[resource[resources_identifier]]
+            resource.update(tags)
+        return resources
+
+    def get_rules_with_tags_by_listener_arn(
+        self, listener_arn: str, normalise: bool = True
+    ) -> list:
+        rules = self.get_listener_rules_by_listener_arn(listener_arn)
+
+        tags = self.get_resources_tag_descriptions(rules)
+
+        rules_with_tags = self.merge_in_tags_by_resource_arn(rules, tags)
+
+        if normalise:
+            for rule in rules_with_tags:
+                rule["Conditions"] = ALBDataNormaliser.conditions_to_dict(rule["Conditions"])
+                rule["Tags"] = ALBDataNormaliser.tags_to_dict(rule["Tags"])
+
+        return rules_with_tags
 
     def get_listener_rules_by_listener_arn(self, listener_arn: str) -> list:
         rules = []
@@ -183,13 +284,15 @@ class LoadBalancerProvider:
 
         return rules
 
-    def get_rules_tag_descriptions(self, rules: list) -> list:
+    def get_resources_tag_descriptions(
+        self, resources: list, resource_identifier: str = "RuleArn"
+    ) -> list:
         tag_descriptions = []
         chunk_size = 20
 
-        for i in range(0, len(rules), chunk_size):
-            chunk = rules[i : i + chunk_size]
-            resource_arns = [r["RuleArn"] for r in chunk]
+        for i in range(0, len(resources), chunk_size):
+            chunk = resources[i : i + chunk_size]
+            resource_arns = [r[resource_identifier] for r in chunk]
             response = self.evlb_client.describe_tags(
                 ResourceArns=resource_arns
             )  # describe_tags cannot be paginated - 04/04/2025
@@ -304,6 +407,9 @@ class LoadBalancerProvider:
 
         return deleted_rules
 
+    def delete_listener_rule_by_resource_arn(self, resource_arn: str) -> list:
+        return self.evlb_client.delete_rule(RuleArn=resource_arn)
+
 
 class LoadBalancerException(PlatformException):
     pass

dbt_platform_helper/providers/logs.py

@@ -0,0 +1,72 @@
+import time
+from typing import Any
+
+import boto3
+from botocore.exceptions import ClientError
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class LogsProvider:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+
+    def check_log_streams_present(self, log_group: str, expected_log_streams: list[str]) -> bool:
+        """
+        Check whether the logs streams provided exist or not.
+
+        Retry for up to 1 minute.
+        """
+
+        found_log_streams = set()
+        expected_log_streams = set(expected_log_streams)
+        timeout_seconds = 60
+        poll_interval_seconds = 5
+        deadline_seconds = time.monotonic() + timeout_seconds
+
+        while time.monotonic() < deadline_seconds:
+
+            remaining_log_streams = expected_log_streams - found_log_streams
+            if not remaining_log_streams:
+                return True
+
+            for log_stream in list(remaining_log_streams):
+                try:
+                    response = self.client.describe_log_streams(
+                        logGroupName=log_group, logStreamNamePrefix=log_stream, limit=1
+                    )
+                except ClientError as e:
+                    code = e.response.get("Error", {}).get("Code")
+                    if code == "ResourceNotFoundException":
+                        continue  # Log stream not there yet, keep going
+                    else:
+                        raise PlatformException(
+                            f"Failed to check if log stream '{log_stream}' exists due to an error {e}"
+                        )
+
+                for ls in response.get("logStreams", []):
+                    if ls.get("logStreamName") == log_stream:
+                        found_log_streams.add(log_stream)
+
+            if expected_log_streams - found_log_streams:
+                time.sleep(poll_interval_seconds)
+
+        missing_log_streams = expected_log_streams - found_log_streams
+        raise PlatformException(
+            f"Timed out waiting for the following log streams to create: {missing_log_streams}"
+        )
+
+    def get_log_stream_events(
+        self, log_group: str, log_stream: str, limit: int
+    ) -> list[dict[str, Any]]:
+        """Return events for a specific log stream."""
+
+        try:
+            self.check_log_streams_present(log_group=log_group, expected_log_streams=[log_stream])
+            response = self.client.get_log_events(
+                logGroupName=log_group, logStreamName=log_stream, limit=limit
+            )
+            return response["events"]
+        except ClientError as err:
+            raise PlatformException(f"Error retrieving log stream events: {err}")

dbt_platform_helper/providers/parameter_store.py

@@ -1,46 +1,133 @@
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Literal
+from typing import Optional
+from typing import Union
+
 import boto3
 
 from dbt_platform_helper.platform_exception import PlatformException
 
 
+@dataclass
+class Parameter:
+
+    name: str
+    value: str
+    arn: str = None
+    data_type: Literal["text", "aws:ec2:image"] = "text"
+    type: Literal["String", "StringList", "SecureString"] = (
+        "SecureString"  # Returned as 'Type' from AWS
+    )
+    version: int = None
+    tags: Optional[dict[str, str]] = field(default_factory=dict)
+
+    def tags_to_list(self):
+        return [{"Key": tag, "Value": value} for tag, value in self.tags.items()]
+
+    def __str__(self):
+        output = f"Application {self.name} with"
+
+        return f"{output} no environments"
+
+    def __eq__(self, other: "Parameter"):
+        return str(self.name) == str(other.name)
+
+
 class ParameterStore:
-    def __init__(self, ssm_client: boto3.client):
+    def __init__(self, ssm_client: boto3.client, with_model: bool = False):
         self.ssm_client = ssm_client
+        self.with_model = with_model
+
+    def __fetch_tags(self, parameter: Parameter, normalise=True):
+        response = self.ssm_client.list_tags_for_resource(
+            ResourceType="Parameter", ResourceId=parameter.name
+        )["TagList"]
+
+        if normalise:
+            return {tag["Key"]: tag["Value"] for tag in response}
+        else:
+            return response
 
-    def get_ssm_parameter_by_name(self, parameter_name: str) -> dict:
+    def get_ssm_parameter_by_name(
+        self, parameter_name: str, add_tags: bool = False
+    ) -> Union[dict, Parameter]:
         """
         Retrieves the latest version of a parameter from parameter store for a
         given name/arn.
 
         Args:
-            path (str): The parameter name to retrieve the parameter value for.
+            parameter_name (str): The parameter name to retrieve the parameter value for.
+            add_tags (bool): Whether to retrieve the tags for the SSM parameters requested
         Returns:
             dict: A dictionary representation of your ssm parameter
         """
+        parameter = self.ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)[
+            "Parameter"
+        ]
+
+        if not self.with_model:
+            return parameter
+
+        model = Parameter(
+            name=parameter["Name"],
+            value=parameter["Value"],
+            arn=parameter["ARN"],
+            data_type=parameter["DataType"],
+            type=parameter["Type"],
+            version=parameter["Version"],
+        )
 
-        return self.ssm_client.get_parameter(Name=parameter_name)["Parameter"]
+        if add_tags:
+            model.tags = self.__fetch_tags(model)
 
-    def get_ssm_parameters_by_path(self, path: str) -> list:
+        return model
+
+    def get_ssm_parameters_by_path(
+        self, path: str, add_tags: bool = False
+    ) -> Union[list[dict], list[Parameter]]:
         """
         Retrieves all SSM parameters for a given path from parameter store.
 
         Args:
             path (str): The parameter path to retrieve the parameters for. e.g. /copilot/applications/
+            add_tags (bool): Whether to retrieve the tags for the SSM parameters requested
         Returns:
             list: A list of dictionaries containing all SSM parameters under the provided path.
         """
 
         parameters = []
         paginator = self.ssm_client.get_paginator("get_parameters_by_path")
-        page_iterator = paginator.paginate(Path=path, Recursive=True)
+        page_iterator = paginator.paginate(Path=path, Recursive=True, WithDecryption=True)
 
         for page in page_iterator:
             parameters.extend(page.get("Parameters", []))
 
-        if parameters:
-            return parameters
-        else:
-            raise ParameterNotFoundForPathException()
+        if not self.with_model:
+            if parameters:
+                return parameters
+            else:
+                raise ParameterNotFoundForPathException()
+
+        to_model = lambda parameter: Parameter(
+            name=parameter["Name"],
+            value=parameter["Value"],
+            arn=parameter["ARN"],
+            data_type=parameter["DataType"],
+            type=parameter["Type"],
+            version=parameter["Version"],
+        )
+
+        models = [to_model(param) for param in parameters]
+
+        if add_tags:
+            for model in models:
+                model.tags = self.__fetch_tags(model)
+
+        return models
+
+    def put_parameter(self, data_dict: dict) -> dict:
+        return self.ssm_client.put_parameter(**data_dict)
 
 
 class ParameterNotFoundForPathException(PlatformException):