dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/providers/load_balancers.py

@@ -1,65 +1,414 @@
-import boto3
+import json
+from typing import Dict
+from typing import List
 
+from boto3 import Session
+
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM_TERRAFORM
+from dbt_platform_helper.constants import ROUTED_TO_PLATFORM_MODES
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+
+
+def normalise_to_cidr(ip: str):
+    if "/" in ip:
+        return ip
+    SINGLE_IPV4_CIDR_PREFIX_LENGTH = "32"
+    return f"{ip}/{SINGLE_IPV4_CIDR_PREFIX_LENGTH}"
+
+
+class ALBDataNormaliser:
 
-# TODO - a good candidate for a dataclass when this is refactored into a class.
-# Below methods should also really be refactored to not be so tightly coupled with eachother.
+    @staticmethod
+    def tags_to_dict(tags: List[Dict[str, str]]) -> Dict[str, str]:
+        return {tag.get("Key", ""): tag.get("Value", "") for tag in tags}
 
+    @staticmethod
+    def conditions_to_dict(conditions: List[Dict[str, List[str]]]) -> Dict[str, List[str]]:
+        return {condition.get("Field", ""): condition.get("Values", "") for condition in conditions}
 
-def get_load_balancer_for_application(session: boto3.Session, app: str, env: str) -> str:
-    lb_client = session.client("elbv2")
 
-    describe_response = lb_client.describe_load_balancers()
-    load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
+class LoadBalancerProvider:
 
-    load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
+    def __init__(self, session: Session = None, io: ClickIOProvider = ClickIOProvider()):
+        self.session = session
+        self.evlb_client = self._get_client("elbv2")
+        self.rg_tagging_client = self._get_client("resourcegroupstaggingapi")
+        self.parameter_store_provider = ParameterStore(self._get_client("ssm"))
+        self.io = io
 
-    load_balancer_arn = None
-    for lb in load_balancers:
-        tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
-        if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
-            load_balancer_arn = lb["ResourceArn"]
+    def _get_client(self, client: str):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client(client)
 
-    if not load_balancer_arn:
-        raise LoadBalancerNotFoundException(
-            f"No load balancer found for {app} in the {env} environment"
+    def find_target_group(self, app: str, env: str, svc: str) -> str:
+
+        # TODO once copilot is gone this is no longer needed
+        try:
+            result = self.parameter_store_provider.get_ssm_parameter_by_name(
+                f"/platform/applications/{app}/environments/{env}"
+            )["Value"]
+            env_config = json.loads(result)
+            service_deployment_mode = env_config["service_deployment_mode"]
+        except Exception:
+            service_deployment_mode = "copilot"
+
+        if service_deployment_mode in ROUTED_TO_PLATFORM_MODES:
+            application_key = "application"
+            environment_key = "environment"
+            service_key = "service"
+        else:
+            application_key = "copilot-application"
+            environment_key = "copilot-environment"
+            service_key = "copilot-service"
+        target_group_arn = None
+
+        paginator = self.rg_tagging_client.get_paginator("get_resources")
+        page_iterator = paginator.paginate(
+            TagFilters=[
+                {
+                    "Key": application_key,
+                    "Values": [
+                        app,
+                    ],
+                },
+                {
+                    "Key": environment_key,
+                    "Values": [
+                        env,
+                    ],
+                },
+                {
+                    "Key": service_key,
+                    "Values": [
+                        svc,
+                    ],
+                },
+            ],
+            ResourceTypeFilters=[
+                "elasticloadbalancing:targetgroup",
+            ],
         )
 
-    return load_balancer_arn
+        for page in page_iterator:
+            for resource in page["ResourceTagMappingList"]:
+                tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
+
+                if (
+                    tags.get(service_key) == svc
+                    and tags.get(environment_key) == env
+                    and tags.get(application_key) == app
+                ):
+                    target_group_arn = resource["ResourceARN"]
+
+        if not target_group_arn:
+            self.io.error(
+                f"No target group found for application: {app}, environment: {env}, service: {svc}",
+            )
+
+        return target_group_arn
+
+    def get_target_groups(self, target_group_arns: List[str]) -> List[dict]:
+        tgs = []
+        paginator = self.evlb_client.get_paginator("describe_target_groups")
+        page_iterator = paginator.paginate(TargetGroupArns=target_group_arns)
+        for page in page_iterator:
+            tgs.extend(page["TargetGroups"])
+
+        return tgs
+
+    def get_target_groups_with_tags(
+        self, target_group_arns: List[str], normalise: bool = True
+    ) -> List[dict]:
+        target_groups = self.get_target_groups(target_group_arns)
+
+        tags = self.get_resources_tag_descriptions(target_groups, "TargetGroupArn")
+
+        tgs_with_tags = self.merge_in_tags_by_resource_arn(target_groups, tags, "TargetGroupArn")
+
+        if normalise:
+            for tg in tgs_with_tags:
+                tg["Tags"] = ALBDataNormaliser.tags_to_dict(tg["Tags"])
+        return tgs_with_tags
+
+    def get_https_certificate_for_listener(self, listener_arn: str, env: str):
+        certificates = []
+        paginator = self.evlb_client.get_paginator("describe_listener_certificates")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            certificates.extend(page["Certificates"])
+
+        try:
+            certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+        except StopIteration:
+            raise CertificateNotFoundException(env)
+
+        return certificate_arn
+
+    def get_https_certificate_for_application(self, app: str, env: str) -> str:
+        listener_arn = self.get_https_listener_for_application(app, env)
+        return self.get_https_certificate_for_listener(listener_arn, env)
+
+    def get_listeners_for_load_balancer(self, load_balancer_arn: str) -> List[dict]:
+        listeners = []
+        paginator = self.evlb_client.get_paginator("describe_listeners")
+        page_iterator = paginator.paginate(LoadBalancerArn=load_balancer_arn)
+        for page in page_iterator:
+            listeners.extend(page["Listeners"])
+
+        return listeners
+
+    def get_https_listener_for_application(self, app: str, env: str) -> str:
+        load_balancer_arn = self.get_load_balancer_for_application(app, env)
+        self.io.debug(f"Load Balancer ARN: {load_balancer_arn}")
+        listeners = self.get_listeners_for_load_balancer(load_balancer_arn)
+
+        listener_arn = None
+
+        try:
+            listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
+        except StopIteration:
+            pass
+
+        if not listener_arn:
+            raise ListenerNotFoundException(app, env)
+
+        return listener_arn
+
+    def get_load_balancers(self) -> List[dict]:
+        load_balancers = []
+        paginator = self.evlb_client.get_paginator("describe_load_balancers")
+        page_iterator = paginator.paginate()
+        for page in page_iterator:
+            load_balancers.extend(lb["LoadBalancerArn"] for lb in page["LoadBalancers"])
+
+        return load_balancers
+
+    def get_load_balancer_for_application(self, app: str, env: str) -> str:
+        load_balancers = self.get_load_balancers()
+        tag_descriptions = []
+        for i in range(0, len(load_balancers), 20):
+            chunk = load_balancers[i : i + 20]
+            tag_descriptions.extend(
+                self.evlb_client.describe_tags(ResourceArns=chunk)["TagDescriptions"]
+            )  # describe_tags cannot be paginated - 04/04/2025
+
+        for lb in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
+            # TODO: DBTP-1967: copilot hangover, creates coupling to specific tags could update to check application and environment
+            if (
+                tags.get("copilot-application") == app
+                and tags.get("copilot-environment") == env
+                and tags.get("managed-by", "") == MANAGED_BY_PLATFORM_TERRAFORM
+            ):
+                return lb["ResourceArn"]
+
+        raise LoadBalancerNotFoundException(app, env)
+
+    def get_host_header_conditions(self, listener_arn: str, target_group_arn: str) -> list:
+        rules = []
+        paginator = self.evlb_client.get_paginator("describe_rules")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            rules.extend(page["Rules"])
+
+        conditions = []
 
+        for rule in rules:
+            for action in rule["Actions"]:
+                if "TargetGroupArn" in action:
+                    if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                        conditions = rule["Conditions"]
 
-def get_https_listener_for_application(session: boto3.Session, app: str, env: str) -> str:
-    load_balancer_arn = get_load_balancer_for_application(session, app, env)
-    lb_client = session.client("elbv2")
-    listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
+        if not conditions:
+            raise ListenerRuleConditionsNotFoundException(listener_arn)
 
-    listener_arn = None
+        # filter to host-header conditions
+        conditions = [
+            {i: condition[i] for i in condition if i != "Values"}
+            for condition in conditions
+            if condition["Field"] == "host-header"
+        ]
 
-    try:
-        listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
-    except StopIteration:
-        pass
+        # remove internal hosts
+        conditions[0]["HostHeaderConfig"]["Values"] = [
+            v for v in conditions[0]["HostHeaderConfig"]["Values"]
+        ]
 
-    if not listener_arn:
-        raise ListenerNotFoundException(f"No HTTPS listener for {app} in the {env} environment")
+        return conditions
 
-    return listener_arn
+    def get_rules_tag_descriptions_by_listener_arn(self, listener_arn: str) -> list:
+        rules = self.get_listener_rules_by_listener_arn(listener_arn)
 
+        return self.get_resources_tag_descriptions(rules)
 
-def get_https_certificate_for_application(session: boto3.Session, app: str, env: str) -> str:
+    def merge_in_tags_by_resource_arn(
+        self,
+        resources: List[dict],
+        tag_descriptions: List[dict],
+        resources_identifier: str = "RuleArn",
+    ):
+        tags_by_resource_arn = {
+            rule_tags.get("ResourceArn"): rule_tags for rule_tags in tag_descriptions if rule_tags
+        }
+        for resource in resources:
+            tags = tags_by_resource_arn[resource[resources_identifier]]
+            resource.update(tags)
+        return resources
 
-    listener_arn = get_https_listener_for_application(session, app, env)
-    cert_client = session.client("elbv2")
-    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
-        "Certificates"
-    ]
+    def get_rules_with_tags_by_listener_arn(
+        self, listener_arn: str, normalise: bool = True
+    ) -> list:
+        rules = self.get_listener_rules_by_listener_arn(listener_arn)
 
-    try:
-        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
-    except StopIteration:
-        raise CertificateNotFoundException(env)
+        tags = self.get_resources_tag_descriptions(rules)
 
-    return certificate_arn
+        rules_with_tags = self.merge_in_tags_by_resource_arn(rules, tags)
+
+        if normalise:
+            for rule in rules_with_tags:
+                rule["Conditions"] = ALBDataNormaliser.conditions_to_dict(rule["Conditions"])
+                rule["Tags"] = ALBDataNormaliser.tags_to_dict(rule["Tags"])
+
+        return rules_with_tags
+
+    def get_listener_rules_by_listener_arn(self, listener_arn: str) -> list:
+        rules = []
+        paginator = self.evlb_client.get_paginator("describe_rules")
+        page_iterator = paginator.paginate(ListenerArn=listener_arn)
+        for page in page_iterator:
+            rules.extend(page["Rules"])
+
+        return rules
+
+    def get_resources_tag_descriptions(
+        self, resources: list, resource_identifier: str = "RuleArn"
+    ) -> list:
+        tag_descriptions = []
+        chunk_size = 20
+
+        for i in range(0, len(resources), chunk_size):
+            chunk = resources[i : i + chunk_size]
+            resource_arns = [r[resource_identifier] for r in chunk]
+            response = self.evlb_client.describe_tags(
+                ResourceArns=resource_arns
+            )  # describe_tags cannot be paginated - 04/04/2025
+            tag_descriptions.extend(response["TagDescriptions"])
+
+        return tag_descriptions
+
+    def create_rule(
+        self,
+        listener_arn: str,
+        actions: list,
+        conditions: list,
+        priority: int,
+        tags: list,
+    ):
+        return self.evlb_client.create_rule(
+            ListenerArn=listener_arn,
+            Priority=priority,
+            Conditions=conditions,
+            Actions=actions,
+            Tags=tags,
+        )
+
+    def create_forward_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+        return self.create_rule(
+            listener_arn=listener_arn,
+            priority=priority,
+            conditions=conditions,
+            actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+            tags=[{"Key": "name", "Value": rule_name}, *additional_tags],
+        )
+
+    def create_header_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        header_name: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+
+        combined_conditions = [
+            {
+                "Field": "http-header",
+                "HttpHeaderConfig": {"HttpHeaderName": header_name, "Values": values},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
+        )
+
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
+        )
+
+    def create_source_ip_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+        combined_conditions = [
+            {
+                "Field": "source-ip",
+                "SourceIpConfig": {"Values": [normalise_to_cidr(value) for value in values]},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
+        )
+
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
+        )
+
+    def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> list:
+        deleted_rules = []
+
+        for description in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+            if tags.get("name") == tag_name:
+                if description["ResourceArn"]:
+                    self.evlb_client.delete_rule(RuleArn=description["ResourceArn"])
+                    deleted_rules.append(description)
+
+        return deleted_rules
+
+    def delete_listener_rule_by_resource_arn(self, resource_arn: str) -> list:
+        return self.evlb_client.delete_rule(RuleArn=resource_arn)
 
 
 class LoadBalancerException(PlatformException):
@@ -67,17 +416,28 @@ class LoadBalancerException(PlatformException):
 
 
 class LoadBalancerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No load balancer found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No HTTPS listener found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerRuleNotFoundException(LoadBalancerException):
     pass
 
 
+class ListenerRuleConditionsNotFoundException(LoadBalancerException):
+    def __init__(self, listener_arn):
+        super().__init__(f"No listener rule conditions found for listener ARN: {listener_arn}")
+
+
 class CertificateNotFoundException(PlatformException):
     def __init__(self, environment_name: str):
         super().__init__(

dbt_platform_helper/providers/logs.py

@@ -0,0 +1,72 @@
+import time
+from typing import Any
+
+import boto3
+from botocore.exceptions import ClientError
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class LogsProvider:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+
+    def check_log_streams_present(self, log_group: str, expected_log_streams: list[str]) -> bool:
+        """
+        Check whether the logs streams provided exist or not.
+
+        Retry for up to 1 minute.
+        """
+
+        found_log_streams = set()
+        expected_log_streams = set(expected_log_streams)
+        timeout_seconds = 60
+        poll_interval_seconds = 5
+        deadline_seconds = time.monotonic() + timeout_seconds
+
+        while time.monotonic() < deadline_seconds:
+
+            remaining_log_streams = expected_log_streams - found_log_streams
+            if not remaining_log_streams:
+                return True
+
+            for log_stream in list(remaining_log_streams):
+                try:
+                    response = self.client.describe_log_streams(
+                        logGroupName=log_group, logStreamNamePrefix=log_stream, limit=1
+                    )
+                except ClientError as e:
+                    code = e.response.get("Error", {}).get("Code")
+                    if code == "ResourceNotFoundException":
+                        continue  # Log stream not there yet, keep going
+                    else:
+                        raise PlatformException(
+                            f"Failed to check if log stream '{log_stream}' exists due to an error {e}"
+                        )
+
+                for ls in response.get("logStreams", []):
+                    if ls.get("logStreamName") == log_stream:
+                        found_log_streams.add(log_stream)
+
+            if expected_log_streams - found_log_streams:
+                time.sleep(poll_interval_seconds)
+
+        missing_log_streams = expected_log_streams - found_log_streams
+        raise PlatformException(
+            f"Timed out waiting for the following log streams to create: {missing_log_streams}"
+        )
+
+    def get_log_stream_events(
+        self, log_group: str, log_stream: str, limit: int
+    ) -> list[dict[str, Any]]:
+        """Return events for a specific log stream."""
+
+        try:
+            self.check_log_streams_present(log_group=log_group, expected_log_streams=[log_stream])
+            response = self.client.get_log_events(
+                logGroupName=log_group, logStreamName=log_stream, limit=limit
+            )
+            return response["events"]
+        except ClientError as err:
+            raise PlatformException(f"Error retrieving log stream events: {err}")

dbt_platform_helper/providers/parameter_store.py

@@ -0,0 +1,134 @@
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Literal
+from typing import Optional
+from typing import Union
+
+import boto3
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+@dataclass
+class Parameter:
+
+    name: str
+    value: str
+    arn: str = None
+    data_type: Literal["text", "aws:ec2:image"] = "text"
+    type: Literal["String", "StringList", "SecureString"] = (
+        "SecureString"  # Returned as 'Type' from AWS
+    )
+    version: int = None
+    tags: Optional[dict[str, str]] = field(default_factory=dict)
+
+    def tags_to_list(self):
+        return [{"Key": tag, "Value": value} for tag, value in self.tags.items()]
+
+    def __str__(self):
+        output = f"Application {self.name} with"
+
+        return f"{output} no environments"
+
+    def __eq__(self, other: "Parameter"):
+        return str(self.name) == str(other.name)
+
+
+class ParameterStore:
+    def __init__(self, ssm_client: boto3.client, with_model: bool = False):
+        self.ssm_client = ssm_client
+        self.with_model = with_model
+
+    def __fetch_tags(self, parameter: Parameter, normalise=True):
+        response = self.ssm_client.list_tags_for_resource(
+            ResourceType="Parameter", ResourceId=parameter.name
+        )["TagList"]
+
+        if normalise:
+            return {tag["Key"]: tag["Value"] for tag in response}
+        else:
+            return response
+
+    def get_ssm_parameter_by_name(
+        self, parameter_name: str, add_tags: bool = False
+    ) -> Union[dict, Parameter]:
+        """
+        Retrieves the latest version of a parameter from parameter store for a
+        given name/arn.
+
+        Args:
+            parameter_name (str): The parameter name to retrieve the parameter value for.
+            add_tags (bool): Whether to retrieve the tags for the SSM parameters requested
+        Returns:
+            dict: A dictionary representation of your ssm parameter
+        """
+        parameter = self.ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)[
+            "Parameter"
+        ]
+
+        if not self.with_model:
+            return parameter
+
+        model = Parameter(
+            name=parameter["Name"],
+            value=parameter["Value"],
+            arn=parameter["ARN"],
+            data_type=parameter["DataType"],
+            type=parameter["Type"],
+            version=parameter["Version"],
+        )
+
+        if add_tags:
+            model.tags = self.__fetch_tags(model)
+
+        return model
+
+    def get_ssm_parameters_by_path(
+        self, path: str, add_tags: bool = False
+    ) -> Union[list[dict], list[Parameter]]:
+        """
+        Retrieves all SSM parameters for a given path from parameter store.
+
+        Args:
+            path (str): The parameter path to retrieve the parameters for. e.g. /copilot/applications/
+            add_tags (bool): Whether to retrieve the tags for the SSM parameters requested
+        Returns:
+            list: A list of dictionaries containing all SSM parameters under the provided path.
+        """
+
+        parameters = []
+        paginator = self.ssm_client.get_paginator("get_parameters_by_path")
+        page_iterator = paginator.paginate(Path=path, Recursive=True, WithDecryption=True)
+
+        for page in page_iterator:
+            parameters.extend(page.get("Parameters", []))
+
+        if not self.with_model:
+            if parameters:
+                return parameters
+            else:
+                raise ParameterNotFoundForPathException()
+
+        to_model = lambda parameter: Parameter(
+            name=parameter["Name"],
+            value=parameter["Value"],
+            arn=parameter["ARN"],
+            data_type=parameter["DataType"],
+            type=parameter["Type"],
+            version=parameter["Version"],
+        )
+
+        models = [to_model(param) for param in parameters]
+
+        if add_tags:
+            for model in models:
+                model.tags = self.__fetch_tags(model)
+
+        return models
+
+    def put_parameter(self, data_dict: dict) -> dict:
+        return self.ssm_client.put_parameter(**data_dict)
+
+
+class ParameterNotFoundForPathException(PlatformException):
+    """Exception raised when no parameters are found for a given path."""