dbt-platform-helper 13.1.0__py3-none-any.whl → 13.1.2__py3-none-any.whl

dbt_platform_helper/domain/versioning.py

@@ -0,0 +1,67 @@
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.platform_helper_versioning import (
+    PlatformHelperVersioning,
+)
+from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.utils.files import running_as_installed_package
+
+
+class PlatformHelperVersionNotFoundException(PlatformException):
+    def __init__(self):
+        super().__init__(f"""Platform helper version could not be resolved.""")
+
+
+class RequiredVersion:
+    def __init__(self, io=None, platform_helper_versioning=None):
+        self.io = io or ClickIOProvider()
+        self.platform_helper_versioning = platform_helper_versioning or PlatformHelperVersioning(
+            io=self.io
+        )
+
+    def get_required_platform_helper_version(
+        self, pipeline: str = None, version_status: PlatformHelperVersionStatus = None
+    ) -> str:
+        pipeline_version = version_status.pipeline_overrides.get(pipeline)
+        version_precedence = [
+            pipeline_version,
+            version_status.platform_config_default,
+            version_status.deprecated_version_file,
+        ]
+        non_null_version_precedence = [
+            f"{v}" if isinstance(v, SemanticVersion) else v for v in version_precedence if v
+        ]
+
+        out = non_null_version_precedence[0] if non_null_version_precedence else None
+
+        if not out:
+            raise PlatformHelperVersionNotFoundException
+
+        return out
+
+    def get_required_version(self, pipeline=None):
+        version_status = self.platform_helper_versioning.get_status()
+        self.io.process_messages(version_status.validate())
+        required_version = self.get_required_platform_helper_version(pipeline, version_status)
+        self.io.info(required_version)
+        return required_version
+
+    # Used in the generate command
+    def check_platform_helper_version_mismatch(self):
+        if not running_as_installed_package():
+            return
+
+        version_status = self.platform_helper_versioning.get_status()
+        self.io.process_messages(version_status.validate())
+
+        required_version = SemanticVersion.from_string(
+            self.get_required_platform_helper_version(version_status=version_status)
+        )
+
+        if not version_status.local == required_version:
+            message = (
+                f"WARNING: You are running platform-helper v{version_status.local} against "
+                f"v{required_version} specified for the project."
+            )
+            self.io.warn(message)

dbt_platform_helper/providers/io.py

@@ -26,6 +26,20 @@ class ClickIOProvider:
         click.secho(f"Error: {message}", err=True, fg="red")
         exit(1)
 
+    # TODO messages will be a ValidationMessages class rather than a free-rein dictionary
+    def process_messages(self, messages: dict):
+        if not messages:
+            return
+
+        if messages.get("errors"):
+            self.error("\n".join(messages["errors"]))
+
+        if messages.get("warnings"):
+            self.warn("\n".join(messages["warnings"]))
+
+        if messages.get("info"):
+            self.info("\n".join(messages["info"]))
+
 
 class ClickIOProviderException(PlatformException):
     pass

dbt_platform_helper/providers/load_balancers.py

@@ -1,65 +1,269 @@
-import boto3
+from boto3 import Session
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 
-# TODO - a good candidate for a dataclass when this is refactored into a class.
-# Below methods should also really be refactored to not be so tightly coupled with eachother.
 
+def normalise_to_cidr(ip: str):
+    if "/" in ip:
+        return ip
+    SINGLE_IPV4_CIDR_PREFIX_LENGTH = "32"
+    return f"{ip}/{SINGLE_IPV4_CIDR_PREFIX_LENGTH}"
 
-def get_load_balancer_for_application(session: boto3.Session, app: str, env: str) -> str:
-    lb_client = session.client("elbv2")
 
-    describe_response = lb_client.describe_load_balancers()
-    load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
+class LoadBalancerProvider:
 
-    load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
+    def __init__(self, session: Session = None, io: ClickIOProvider = ClickIOProvider()):
+        self.session = session
+        self.evlb_client = self._get_client("elbv2")
+        self.rg_tagging_client = self._get_client("resourcegroupstaggingapi")
+        self.io = io
 
-    load_balancer_arn = None
-    for lb in load_balancers:
-        tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
-        if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
-            load_balancer_arn = lb["ResourceArn"]
+    def _get_client(self, client: str):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client(client)
 
-    if not load_balancer_arn:
-        raise LoadBalancerNotFoundException(
-            f"No load balancer found for {app} in the {env} environment"
-        )
+    def find_target_group(self, app: str, env: str, svc: str) -> str:
+        target_group_arn = None
+
+        response = self.rg_tagging_client.get_resources(
+            TagFilters=[
+                {
+                    "Key": "copilot-application",
+                    "Values": [
+                        app,
+                    ],
+                    "Key": "copilot-environment",
+                    "Values": [
+                        env,
+                    ],
+                    "Key": "copilot-service",
+                    "Values": [
+                        svc,
+                    ],
+                },
+            ],
+            ResourceTypeFilters=[
+                "elasticloadbalancing:targetgroup",
+            ],
+        )  # TODO should be paginated
+        for resource in response["ResourceTagMappingList"]:
+            tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
+
+            if (
+                "copilot-service" in tags
+                and tags["copilot-service"] == svc
+                and "copilot-environment" in tags
+                and tags["copilot-environment"] == env
+                and "copilot-application" in tags
+                and tags["copilot-application"] == app
+            ):
+                target_group_arn = resource["ResourceARN"]
+
+        if not target_group_arn:
+            self.io.error(
+                f"No target group found for application: {app}, environment: {env}, service: {svc}",
+            )
+
+        return target_group_arn
+
+    def get_https_certificate_for_application(self, app: str, env: str) -> str:
+        listener_arn = self.get_https_listener_for_application(app, env)
+        certificates = self.evlb_client.describe_listener_certificates(ListenerArn=listener_arn)[
+            "Certificates"
+        ]  # TODO should be paginated
+
+        try:
+            certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+        except StopIteration:
+            raise CertificateNotFoundException(env)
+
+        return certificate_arn
+
+    def get_https_listener_for_application(self, app: str, env: str) -> str:
+        load_balancer_arn = self.get_load_balancer_for_application(app, env)
+
+        listeners = self.evlb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)[
+            "Listeners"
+        ]  # TODO should be paginated
+
+        listener_arn = None
 
-    return load_balancer_arn
+        try:
+            listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
+        except StopIteration:
+            pass
 
+        if not listener_arn:
+            raise ListenerNotFoundException(app, env)
 
-def get_https_listener_for_application(session: boto3.Session, app: str, env: str) -> str:
-    load_balancer_arn = get_load_balancer_for_application(session, app, env)
-    lb_client = session.client("elbv2")
-    listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
+        return listener_arn
 
-    listener_arn = None
+    def get_load_balancer_for_application(self, app: str, env: str) -> str:
+        describe_response = self.evlb_client.describe_load_balancers()
+        load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
 
-    try:
-        listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
-    except StopIteration:
-        pass
+        tag_descriptions = []
+        for i in range(0, len(load_balancers), 20):
+            chunk = load_balancers[i : i + 20]
+            tag_descriptions.extend(
+                self.evlb_client.describe_tags(ResourceArns=chunk)["TagDescriptions"]
+            )
 
-    if not listener_arn:
-        raise ListenerNotFoundException(f"No HTTPS listener for {app} in the {env} environment")
+        for lb in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
+            # TODO copilot hangover, creates coupling to specific tags could update to check application and environment
+            if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
+                return lb["ResourceArn"]
 
-    return listener_arn
+        raise LoadBalancerNotFoundException(app, env)
 
+    def get_host_header_conditions(self, listener_arn: str, target_group_arn: str) -> list:
+        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
+            "Rules"
+        ]  # TODO should be paginated
 
-def get_https_certificate_for_application(session: boto3.Session, app: str, env: str) -> str:
+        conditions = []
 
-    listener_arn = get_https_listener_for_application(session, app, env)
-    cert_client = session.client("elbv2")
-    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
-        "Certificates"
-    ]
+        for rule in rules:
+            for action in rule["Actions"]:
+                if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                    conditions = rule["Conditions"]
 
-    try:
-        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
-    except StopIteration:
-        raise CertificateNotFoundException(env)
+        if not conditions:
+            raise ListenerRuleConditionsNotFoundException(listener_arn)
+
+        # filter to host-header conditions
+        conditions = [
+            {i: condition[i] for i in condition if i != "Values"}
+            for condition in conditions
+            if condition["Field"] == "host-header"
+        ]
+
+        # remove internal hosts
+        conditions[0]["HostHeaderConfig"]["Values"] = [
+            v for v in conditions[0]["HostHeaderConfig"]["Values"]
+        ]
+
+        return conditions
+
+    def get_rules_tag_descriptions_by_listener_arn(self, listener_arn: str) -> list:
+        rules = self.evlb_client.describe_rules(ListenerArn=listener_arn)[
+            "Rules"
+        ]  # TODO should be paginated
+        return self.get_rules_tag_descriptions(rules)
+
+    def get_rules_tag_descriptions(self, rules: list) -> list:
+        tag_descriptions = []
+        chunk_size = 20
+
+        for i in range(0, len(rules), chunk_size):
+            chunk = rules[i : i + chunk_size]
+            resource_arns = [r["RuleArn"] for r in chunk]
+            response = self.evlb_client.describe_tags(
+                ResourceArns=resource_arns
+            )  # TODO should be paginated
+            tag_descriptions.extend(response["TagDescriptions"])
+
+        return tag_descriptions
+
+    def create_rule(
+        self,
+        listener_arn: str,
+        actions: list,
+        conditions: list,
+        priority: int,
+        tags: list,
+    ):
+        return self.evlb_client.create_rule(
+            ListenerArn=listener_arn,
+            Priority=priority,
+            Conditions=conditions,
+            Actions=actions,
+            Tags=tags,
+        )
+
+    def create_forward_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+    ):
+        return self.create_rule(
+            listener_arn=listener_arn,
+            priority=priority,
+            conditions=conditions,
+            actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+            tags=[
+                {"Key": "name", "Value": rule_name},
+            ],
+        )
 
-    return certificate_arn
+    def create_header_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        header_name: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+    ):
+
+        combined_conditions = [
+            {
+                "Field": "http-header",
+                "HttpHeaderConfig": {"HttpHeaderName": header_name, "Values": values},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn, target_group_arn, rule_name, priority, combined_conditions
+        )
+
+        self.io.info(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        )
+
+    def create_source_ip_rule(
+        self,
+        listener_arn: str,
+        target_group_arn: str,
+        values: list,
+        rule_name: str,
+        priority: int,
+        conditions: list,
+    ):
+        combined_conditions = [
+            {
+                "Field": "source-ip",
+                "SourceIpConfig": {"Values": [normalise_to_cidr(value) for value in values]},
+            }
+        ] + conditions
+
+        self.create_forward_rule(
+            listener_arn, target_group_arn, rule_name, priority, combined_conditions
+        )
+
+        self.io.info(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        )
+
+    def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> str:
+        current_rule_arn = None
+
+        for description in tag_descriptions:
+            tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+            if tags.get("name") == tag_name:
+                current_rule_arn = description["ResourceArn"]
+                if current_rule_arn:
+                    self.evlb_client.delete_rule(RuleArn=current_rule_arn)
+
+        return current_rule_arn
 
 
 class LoadBalancerException(PlatformException):
@@ -67,17 +271,28 @@ class LoadBalancerException(PlatformException):
 
 
 class LoadBalancerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No load balancer found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerNotFoundException(LoadBalancerException):
-    pass
+    def __init__(self, application_name, env):
+        super().__init__(
+            f"No HTTPS listener found for environment {env} in the application {application_name}."
+        )
 
 
 class ListenerRuleNotFoundException(LoadBalancerException):
     pass
 
 
+class ListenerRuleConditionsNotFoundException(LoadBalancerException):
+    def __init__(self, listener_arn):
+        super().__init__(f"No listener rule conditions found for listener ARN: {listener_arn}")
+
+
 class CertificateNotFoundException(PlatformException):
     def __init__(self, environment_name: str):
         super().__init__(

dbt_platform_helper/providers/platform_config_schema.py

@@ -506,7 +506,9 @@ class PlatformConfigSchema:
                         },
                         Optional("cross_environment_service_access"): {
                             PlatformConfigSchema.__valid_schema_key(): {
-                                "application": str,
+                                # Deprecated: We didn't implement cross application access, no service teams are asking for it.
+                                # application should be removed once we can confirm that no-one is using it.
+                                Optional("application"): str,
                                 "environment": PlatformConfigSchema.__valid_environment_name(),
                                 "account": str,
                                 "service": str,

dbt_platform_helper/providers/platform_helper_versioning.py

@@ -0,0 +1,107 @@
+import os
+from pathlib import Path
+
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMajorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMinorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.version import LocalVersionProvider
+from dbt_platform_helper.providers.version import LocalVersionProviderException
+from dbt_platform_helper.providers.version import PyPiVersionProvider
+from dbt_platform_helper.providers.yaml_file import FileProviderException
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+from dbt_platform_helper.utils.files import running_as_installed_package
+
+
+class PlatformHelperVersioning:
+    def __init__(
+        self,
+        io: ClickIOProvider = ClickIOProvider(),
+        file_provider: FileProvider = YamlFileProvider,
+        config_provider: ConfigProvider = ConfigProvider(),
+        pypi_provider: PyPiVersionProvider = PyPiVersionProvider,
+        local_version_provider: LocalVersionProvider = LocalVersionProvider(),
+    ):
+        self.io = io
+        self.file_provider = file_provider
+        self.config_provider = config_provider
+        self.pypi_provider = pypi_provider
+        self.local_version_provider = local_version_provider
+
+    def get_status(
+        self,
+        include_project_versions: bool = True,
+    ) -> PlatformHelperVersionStatus:
+        try:
+            locally_installed_version = self.local_version_provider.get_installed_tool_version(
+                "dbt-platform-helper"
+            )
+        except LocalVersionProviderException:
+            locally_installed_version = None
+
+        latest_release = self.pypi_provider.get_latest_version("dbt-platform-helper")
+
+        if not include_project_versions:
+            return PlatformHelperVersionStatus(
+                local=locally_installed_version,
+                latest=latest_release,
+            )
+
+        deprecated_version_file = Path(PLATFORM_HELPER_VERSION_FILE)
+        try:
+            loaded_version = self.file_provider.load(deprecated_version_file)
+            version_from_file = SemanticVersion.from_string(loaded_version)
+        except FileProviderException:
+            version_from_file = None
+
+        platform_config_default, pipeline_overrides = None, {}
+
+        platform_config = self.config_provider.load_unvalidated_config_file()
+
+        if platform_config:
+            platform_config_default = SemanticVersion.from_string(
+                platform_config.get("default_versions", {}).get("platform-helper")
+            )
+
+            pipeline_overrides = {
+                name: pipeline.get("versions", {}).get("platform-helper")
+                for name, pipeline in platform_config.get("environment_pipelines", {}).items()
+                if pipeline.get("versions", {}).get("platform-helper")
+            }
+
+        out = PlatformHelperVersionStatus(
+            local=locally_installed_version,
+            latest=latest_release,
+            deprecated_version_file=version_from_file,
+            platform_config_default=platform_config_default,
+            pipeline_overrides=pipeline_overrides,
+        )
+
+        return out
+
+    def check_if_needs_update(self):
+        if not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ:
+            return
+
+        version_status = self.get_status(include_project_versions=False)
+
+        message = (
+            f"You are running platform-helper v{version_status.local}, upgrade to "
+            f"v{version_status.latest} by running run `pip install "
+            "--upgrade dbt-platform-helper`."
+        )
+
+        try:
+            version_status.local.validate_compatibility_with(version_status.latest)
+        except IncompatibleMajorVersionException:
+            self.io.error(message)
+        except IncompatibleMinorVersionException:
+            self.io.warn(message)

dbt_platform_helper/providers/semantic_version.py

@@ -74,17 +74,22 @@ class SemanticVersion:
         return SemanticVersion(output_version[0], output_version[1], output_version[2])
 
 
+@dataclass
 class VersionStatus:
-    def __init__(
-        self, local_version: SemanticVersion = None, latest_release: SemanticVersion = None
-    ):
-        self.local = local_version
-        self.latest = latest_release
+    local: SemanticVersion = None
+    latest: SemanticVersion = None
+
+    def __str__(self):
+        attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
+        return f"{self.__class__.__name__}: {attrs_str}"
 
     def is_outdated(self):
         return self.local != self.latest
 
-    def warn(self):
+    def validate(self):
         pass
 
 
@@ -96,7 +101,22 @@ class PlatformHelperVersionStatus(VersionStatus):
     platform_config_default: Optional[SemanticVersion] = None
     pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
 
-    def warn(self) -> dict:
+    def __str__(self):
+        semantic_version_attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+
+        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
+
+        if self.pipeline_overrides.items():
+            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
+                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
+            )
+            class_str = ", ".join([class_str, pipeline_overrides_str])
+
+        return f"{self.__class__.__name__}: {class_str}"
+
+    def validate(self) -> dict:
         if self.platform_config_default and not self.deprecated_version_file:
             return {}
 

dbt_platform_helper/providers/version.py

@@ -1,14 +1,38 @@
 from abc import ABC
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
 
 import requests
 
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
 
 
+class LocalVersionProviderException(PlatformException):
+    pass
+
+
+class InstalledToolNotFoundException(LocalVersionProviderException):
+    def __init__(
+        self,
+        tool_name: str,
+    ):
+        super().__init__(f"Package '{tool_name}' not found.")
+
+
 class VersionProvider(ABC):
     pass
 
 
+class LocalVersionProvider:
+    @staticmethod
+    def get_installed_tool_version(tool_name: str) -> SemanticVersion:
+        try:
+            return SemanticVersion.from_string(version(tool_name))
+        except PackageNotFoundError:
+            raise InstalledToolNotFoundException(tool_name)
+
+
 # TODO add timeouts and exception handling for requests
 # TODO Alternatively use the gitpython package?
 class GithubVersionProvider(VersionProvider):

dbt_platform_helper/utils/application.py

@@ -146,3 +146,17 @@ class ApplicationNotFoundException(ApplicationException):
         super().__init__(
             f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}"; ensure you have set the environment variable "AWS_PROFILE" correctly."""
         )
+
+
+class ApplicationServiceNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, svc_name: str):
+        super().__init__(
+            f"""The service {svc_name} was not found in the application {application_name}. It either does not exist, or has not been deployed."""
+        )
+
+
+class ApplicationEnvironmentNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, environment: str):
+        super().__init__(
+            f"""The environment "{environment}" either does not exist or has not been deployed for the application {application_name}."""
+        )

dbt_platform_helper/utils/files.py

@@ -51,3 +51,9 @@ def generate_override_files_from_template(base_path, overrides_path, output_dir,
 
     generate_files_for_dir("*")
     generate_files_for_dir("bin/*")
+
+
+# TODO: we've moved this from versioning utils and removed the duplication in platform_helper_versioning
+# Need to review if this is the correct place for this function to reside longer-term
+def running_as_installed_package():
+    return "site-packages" in __file__