dbt-platform-helper 13.1.1__py3-none-any.whl → 13.2.0__py3-none-any.whl

dbt_platform_helper/providers/load_balancers.py

@@ -1,65 +1,280 @@
-import boto3
+from boto3 import Session
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 
-# TODO - a good candidate for a dataclass when this is refactored into a class.
-# Below methods should also really be refactored to not be so tightly coupled with eachother.
 
+def normalise_to_cidr(ip: str):
+    if "/" in ip:
+        return ip
+    SINGLE_IPV4_CIDR_PREFIX_LENGTH = "32"
+    return f"{ip}/{SINGLE_IPV4_CIDR_PREFIX_LENGTH}"
 
-def get_load_balancer_for_application(session: boto3.Session, app: str, env: str) -> str:
-    lb_client = session.client("elbv2")
 
-    describe_response = lb_client.describe_load_balancers()
-    load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
+class LoadBalancerProvider:
 
-    load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
+    def __init__(self, session: Session = None, io: ClickIOProvider = ClickIOProvider()):
+        self.session = session
+        self.evlb_client = self._get_client("elbv2")
+        self.rg_tagging_client = self._get_client("resourcegroupstaggingapi")
+        self.io = io
 
-    load_balancer_arn = None
-    for lb in load_balancers:
-        tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
-        if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
-            load_balancer_arn = lb["ResourceArn"]
+    def _get_client(self, client: str):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client(client)
 
-    if not load_balancer_arn:
-        raise LoadBalancerNotFoundException(
-            f"No load balancer found for {app} in the {env} environment"
-        )
+    def find_target_group(self, app: str, env: str, svc: str) -> str:
+        target_group_arn = None
+
+        response = self.rg_tagging_client.get_resources(
+            TagFilters=[
+                {
+                    "Key": "copilot-application",
+                    "Values": [
+                        app,
+                    ],
+                    "Key": "copilot-environment",
+                    "Values": [
+                        env,
+                    ],
+                    "Key": "copilot-service",
+                    "Values": [
+                        svc,
+                    ],
+                },
+            ],
+            ResourceTypeFilters=[
+                "elasticloadbalancing:targetgroup",
+            ],
+        )  # TODO should be paginated
+        for resource in response["ResourceTagMappingList"]:
+            tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
+
+            if (
+                "copilot-service" in tags
+                and tags["copilot-service"] == svc
+                and "copilot-environment" in tags
+                and tags["copilot-environment"] == env
+                and "copilot-application" in tags
+                and tags["copilot-application"] == app
+            ):
+                target_group_arn = resource["ResourceARN"]
+
+        if not target_group_arn:
+            self.io.error(
+                f"No target group found for application: {app}, environment: {env}, service: {svc}",
+            )
+
+        return target_group_arn
+
+    def get_https_certificate_for_application(self, app: str, env: str) -> str:
+        listener_arn = self.get_https_listener_for_application(app, env)
+        certificates = self.evlb_client.describe_listener_certificates(ListenerArn=listener_arn)[
+            "Certificates"
+        ]  # TODO should be paginated
+
+        try:
+            certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+        except StopIteration:
+            raise CertificateNotFoundException(env)
+
+        return certificate_arn
+
+    def get_https_listener_for_application(self, app: str, env: str) -> str:
+        load_balancer_arn = self.get_load_balancer_for_application(app, env)
+
+        listeners = self.evlb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)[
+            "Listeners"
+        ]  # TODO should be paginated
+
+        listener_arn = None
 
-    return load_balancer_arn
+        try:
+            listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
+        except StopIteration:
+            pass
 
+        if not listener_arn:
+            raise ListenerNotFoundException(app, env)
 
-def get_https_listener_for_application(session: boto3.Session, app: str, env: str) -> str:
-    load_balancer_arn = get_load_balancer_for_application(session, app, env)
-    lb_client = session.client("elbv2")
-    listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
+        return listener_arn
 
-    listener_arn = None
+    def get_load_balancer_for_application(self, app: str, env: str) -> str:
+        describe_response = self.evlb_client.describe_load_balancers()
+        load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
 
-    try:
-        listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
-    except StopIteration:
-        pass
+        tag_descriptions = []
+        for i in range(0, len(load_balancers), 20):
+            chunk = load_balancers[i : i + 20]
+            tag_descriptions.extend(
+                self.evlb_client.describe_tags(ResourceArns=chunk)["TagDescriptions"]
+            )
 
-    if not listener_arn:
-        raise ListenerNotFoundException(f"No HTTPS listener for {app} in the {env} environment")
+        for lb in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
+            # TODO copilot hangover, creates coupling to specific tags could update to check application and environment
+            if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
+                return lb["ResourceArn"]
 
-    return listener_arn
+        raise LoadBalancerNotFoundException(app, env)
 
+    def get_host_header_conditions(self, listener_arn: str, target_group_arn: str) -> list:
+        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
+            "Rules"
+        ]  # TODO should be paginated
 
-def get_https_certificate_for_application(session: boto3.Session, app: str, env: str) -> str:
+        conditions = []
 
-    listener_arn = get_https_listener_for_application(session, app, env)
-    cert_client = session.client("elbv2")
-    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
-        "Certificates"
-    ]
+        for rule in rules:
+            for action in rule["Actions"]:
+                if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                    conditions = rule["Conditions"]
 
-    try:
-        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
-    except StopIteration:
-        raise CertificateNotFoundException(env)
+        if not conditions:
+            raise ListenerRuleConditionsNotFoundException(listener_arn)
+
+        # filter to host-header conditions
+        conditions = [
+            {i: condition[i] for i in condition if i != "Values"}
+            for condition in conditions
+            if condition["Field"] == "host-header"
+        ]
+
+        # remove internal hosts
+        conditions[0]["HostHeaderConfig"]["Values"] = [
+            v for v in conditions[0]["HostHeaderConfig"]["Values"]
+        ]
+
+        return conditions
+
+    def get_rules_tag_descriptions_by_listener_arn(self, listener_arn: str) -> list:
+        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
+            "Rules"
+        ]  # TODO should be paginated
+        return self.get_rules_tag_descriptions(rules)
+
+    def get_rules_tag_descriptions(self, rules: list) -> list:
+        tag_descriptions = []
+        chunk_size = 20
+
+        for i in range(0, len(rules), chunk_size):
+            chunk = rules[i : i + chunk_size]
+            resource_arns = [r["RuleArn"] for r in chunk]
+            response = self.evlb_client.describe_tags(
+                ResourceArns=resource_arns
+            )  # TODO should be paginated
+            tag_descriptions.extend(response["TagDescriptions"])
+
+        return tag_descriptions
+
+    def create_rule(
+        self,
+        listener_arn: str,
+        actions: list,
+        conditions: list,
+        priority: int,
+        tags: list,
+    ):
+        return self.evlb_client.create_rule(
+            ListenerArn=listener_arn,
+            Priority=priority,
+            Conditions=conditions,
+            Actions=actions,
+            Tags=tags,
+        )
+
+    def create_forward_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+        return self.create_rule(
+            listener_arn=listener_arn,
+            priority=priority,
+            conditions=conditions,
+            actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+            tags=[{"Key": "name", "Value": rule_name}, *additional_tags],
+        )
 
-    return certificate_arn
+    def create_header_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        header_name: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+
+        combined_conditions = [
+            {
+                "Field": "http-header",
+                "HttpHeaderConfig": {"HttpHeaderName": header_name, "Values": values},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
+        )
+
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
+        )
+
+    def create_source_ip_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+        additional_tags: list = [],
+    ):
+        combined_conditions = [
+            {
+                "Field": "source-ip",
+                "SourceIpConfig": {"Values": [normalise_to_cidr(value) for value in values]},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
+        )
+
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
+        )
+
+    def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> list:
+        deleted_rules = []
+
+        for description in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+            if tags.get("name") == tag_name:
+                if description["ResourceArn"]:
+                    self.evlb_client.delete_rule(RuleArn=description["ResourceArn"])
+                    deleted_rules.append(description)
+
+        return deleted_rules
 
 
 class LoadBalancerException(PlatformException):
@@ -67,17 +282,28 @@ class LoadBalancerException(PlatformException):
 
 
 class LoadBalancerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No load balancer found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No HTTPS listener found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerRuleNotFoundException(LoadBalancerException):
     pass
 
 
+class ListenerRuleConditionsNotFoundException(LoadBalancerException):
+    def __init__(self, listener_arn):
+        super().__init__(f"No listener rule conditions found for listener ARN: {listener_arn}")
+
+
 class CertificateNotFoundException(PlatformException):
     def __init__(self, environment_name: str):
         super().__init__(

dbt_platform_helper/providers/semantic_version.py

@@ -35,6 +35,9 @@ class SemanticVersion:
             return "unknown"
         return ".".join([str(s) for s in [self.major, self.minor, self.patch]])
 
+    def __repr__(self) -> str:
+        return str(self)
+
     def __lt__(self, other) -> bool:
         return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
 
@@ -74,29 +77,49 @@ class SemanticVersion:
         return SemanticVersion(output_version[0], output_version[1], output_version[2])
 
 
+@dataclass
 class VersionStatus:
-    def __init__(
-        self, local_version: SemanticVersion = None, latest_release: SemanticVersion = None
-    ):
-        self.local = local_version
-        self.latest = latest_release
+    installed: SemanticVersion = None
+    latest: SemanticVersion = None
+
+    def __str__(self):
+        attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
+        return f"{self.__class__.__name__}: {attrs_str}"
 
     def is_outdated(self):
-        return self.local != self.latest
+        return self.installed != self.latest
 
-    def warn(self):
+    def validate(self):
         pass
 
 
 @dataclass
 class PlatformHelperVersionStatus(VersionStatus):
-    local: Optional[SemanticVersion] = None
+    installed: Optional[SemanticVersion] = None
     latest: Optional[SemanticVersion] = None
     deprecated_version_file: Optional[SemanticVersion] = None
     platform_config_default: Optional[SemanticVersion] = None
     pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
 
-    def warn(self) -> dict:
+    def __str__(self):
+        semantic_version_attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+
+        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
+
+        if self.pipeline_overrides.items():
+            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
+                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
+            )
+            class_str = ", ".join([class_str, pipeline_overrides_str])
+
+        return f"{self.__class__.__name__}: {class_str}"
+
+    def validate(self) -> dict:
         if self.platform_config_default and not self.deprecated_version_file:
             return {}
 
@@ -117,7 +140,7 @@ class PlatformHelperVersionStatus(VersionStatus):
 
         if not self.platform_config_default and not self.deprecated_version_file:
             message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
-            message += f"{missing_default_version_message}{self.local}\n"
+            message += f"{missing_default_version_message}{self.installed}\n"
             errors.append(message)
 
         return {

dbt_platform_helper/providers/version.py

@@ -1,14 +1,42 @@
 from abc import ABC
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+from pathlib import Path
 
 import requests
 
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.yaml_file import FileProviderException
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+class InstalledVersionProviderException(PlatformException):
+    pass
+
+
+class InstalledToolNotFoundException(InstalledVersionProviderException):
+    def __init__(
+        self,
+        tool_name: str,
+    ):
+        super().__init__(f"Package '{tool_name}' not found.")
 
 
 class VersionProvider(ABC):
     pass
 
 
+class InstalledVersionProvider:
+    @staticmethod
+    def get_installed_tool_version(tool_name: str) -> SemanticVersion:
+        try:
+            return SemanticVersion.from_string(version(tool_name))
+        except PackageNotFoundError:
+            raise InstalledToolNotFoundException(tool_name)
+
+
 # TODO add timeouts and exception handling for requests
 # TODO Alternatively use the gitpython package?
 class GithubVersionProvider(VersionProvider):
@@ -34,3 +62,17 @@ class PyPiVersionProvider(VersionProvider):
         parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
         parsed_released_versions.sort(reverse=True)
         return parsed_released_versions[0]
+
+
+class DeprecatedVersionFileVersionProvider(VersionProvider):
+    def __init__(self, file_provider: YamlFileProvider):
+        self.file_provider = file_provider or YamlFileProvider
+
+    def get_required_version(self) -> SemanticVersion:
+        deprecated_version_file = Path(PLATFORM_HELPER_VERSION_FILE)
+        try:
+            loaded_version = self.file_provider.load(deprecated_version_file)
+            version_from_file = SemanticVersion.from_string(loaded_version)
+        except FileProviderException:
+            version_from_file = None
+        return version_from_file

dbt_platform_helper/providers/yaml_file.py

@@ -45,7 +45,6 @@ class YamlFileProvider:
 
         if not yaml_content:
             return {}
-
         YamlFileProvider.lint_yaml_for_duplicate_keys(path)
 
         return yaml_content

dbt_platform_helper/utils/application.py

@@ -146,3 +146,17 @@ class ApplicationNotFoundException(ApplicationException):
         super().__init__(
             f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}"; ensure you have set the environment variable "AWS_PROFILE" correctly."""
         )
+
+
+class ApplicationServiceNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, svc_name: str):
+        super().__init__(
+            f"""The service {svc_name} was not found in the application {application_name}. It either does not exist, or has not been deployed."""
+        )
+
+
+class ApplicationEnvironmentNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, environment: str):
+        super().__init__(
+            f"""The environment "{environment}" either does not exist or has not been deployed for the application {application_name}."""
+        )

dbt_platform_helper/utils/aws.py

@@ -12,13 +12,16 @@ import botocore.exceptions
 import click
 import yaml
 from boto3 import Session
+from botocore.exceptions import ClientError
 
 from dbt_platform_helper.constants import REFRESH_TOKEN_MESSAGE
 from dbt_platform_helper.platform_exception import PlatformException
-from dbt_platform_helper.providers.aws import CopilotCodebaseNotFoundException
-from dbt_platform_helper.providers.aws import ImageNotFoundException
-from dbt_platform_helper.providers.aws import LogGroupNotFoundException
-from dbt_platform_helper.providers.aws import RepositoryNotFoundException
+from dbt_platform_helper.providers.aws.exceptions import (
+    CopilotCodebaseNotFoundException,
+)
+from dbt_platform_helper.providers.aws.exceptions import ImageNotFoundException
+from dbt_platform_helper.providers.aws.exceptions import LogGroupNotFoundException
+from dbt_platform_helper.providers.aws.exceptions import RepositoryNotFoundException
 from dbt_platform_helper.providers.validation import ValidationException
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
@@ -484,3 +487,23 @@ def wait_for_log_group_to_exist(log_client, log_group_name, attempts=30):
 
     if not log_group_exists:
         raise LogGroupNotFoundException(log_group_name)
+
+
+def get_image_build_project(codebuild_client, application, codebase):
+    project_name = f"{application}-{codebase}-codebase-image-build"
+    response = codebuild_client.batch_get_projects(names=[project_name])
+
+    if bool(response.get("projects")):
+        return project_name
+    else:
+        return f"{application}-{codebase}-codebase-pipeline-image-build"
+
+
+def get_manual_release_pipeline(codepipeline_client, application, codebase):
+    pipeline_name = f"{application}-{codebase}-manual-release"
+    try:
+        codepipeline_client.get_pipeline(name=pipeline_name)
+        return pipeline_name
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "PipelineNotFoundException":
+            return f"{pipeline_name}-pipeline"

dbt_platform_helper/utils/tool_versioning.py

@@ -0,0 +1,96 @@
+import re
+import subprocess
+from pathlib import Path
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.semantic_version import VersionStatus
+from dbt_platform_helper.providers.validation import ValidationException
+from dbt_platform_helper.providers.version import DeprecatedVersionFileVersionProvider
+from dbt_platform_helper.providers.version import GithubVersionProvider
+from dbt_platform_helper.providers.version import InstalledVersionProvider
+from dbt_platform_helper.providers.version import PyPiVersionProvider
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+# TODO to be removed after config tests are updated - temporary wrapper mid-refactor
+def get_platform_helper_version_status(
+    include_project_versions=True,
+    yaml_provider=YamlFileProvider,
+) -> PlatformHelperVersionStatus:
+    return PlatformHelperVersioning(
+        pypi_provider=PyPiVersionProvider,
+        installed_version_provider=InstalledVersionProvider(),
+        version_file_version_provider=DeprecatedVersionFileVersionProvider(yaml_provider),
+        config_provider=ConfigProvider(),
+    )._get_version_status(include_project_versions=include_project_versions)
+
+
+def get_required_terraform_platform_modules_version(
+    cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
+):
+    version_preference_order = [
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
+    ]
+    return [version for version in version_preference_order if version][0]
+
+
+##################################################################################
+# Only used in Config domain
+# TODO Relocate along with tests when we refactor config command in DBTP-1538
+##################################################################################
+
+
+# Getting version from the "Generated by" comment in a file that was generated from a template
+# TODO where does this belong?  It sort of belongs to our platform-helper templating
+def get_template_generated_with_version(template_file_path: str) -> SemanticVersion:
+    try:
+        template_contents = Path(template_file_path).read_text()
+        template_version = re.match(
+            r"# Generated by platform-helper ([v.\-0-9]+)", template_contents
+        ).group(1)
+        return SemanticVersion.from_string(template_version)
+    except (IndexError, AttributeError):
+        raise ValidationException(f"Template {template_file_path} has no version information")
+
+
+def validate_template_version(app_version: SemanticVersion, template_file_path: str):
+    app_version.validate_compatibility_with(get_template_generated_with_version(template_file_path))
+
+
+# Local version and latest release of tool.
+# Used only in config command.
+# TODO Move to config domain
+def get_copilot_versions() -> VersionStatus:
+    copilot_version = None
+
+    try:
+        response = subprocess.run("copilot --version", capture_output=True, shell=True)
+        [copilot_version] = re.findall(r"[0-9.]+", response.stdout.decode("utf8"))
+    except ValueError:
+        pass
+
+    return VersionStatus(
+        SemanticVersion.from_string(copilot_version),
+        GithubVersionProvider.get_latest_version("aws/copilot-cli"),
+    )
+
+
+# Local version and latest release of tool.
+# Used only in config command.
+# TODO Move to config domain
+def get_aws_versions() -> VersionStatus:
+    aws_version = None
+    try:
+        response = subprocess.run("aws --version", capture_output=True, shell=True)
+        matched = re.match(r"aws-cli/([0-9.]+)", response.stdout.decode("utf8"))
+        aws_version = SemanticVersion.from_string(matched.group(1))
+    except ValueError:
+        pass
+
+    return VersionStatus(aws_version, GithubVersionProvider.get_latest_version("aws/aws-cli", True))