dbt-platform-helper 13.1.2__py3-none-any.whl → 13.3.0__py3-none-any.whl

dbt_platform_helper/providers/load_balancers.py

@@ -192,15 +192,14 @@ class LoadBalancerProvider:
         rule_name: str,
         priority: int,
         conditions: list,
+        additional_tags: list = [],
     ):
         return self.create_rule(
             listener_arn=listener_arn,
             priority=priority,
             conditions=conditions,
             actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
-            tags=[
-                {"Key": "name", "Value": rule_name},
-            ],
+            tags=[{"Key": "name", "Value": rule_name}, *additional_tags],
         )
 
     def create_header_rule(
@@ -212,6 +211,7 @@ class LoadBalancerProvider:
         rule_name: str,
         priority: int,
         conditions: list,
+        additional_tags: list = [],
     ):
 
         combined_conditions = [
@@ -222,11 +222,16 @@ class LoadBalancerProvider:
         ] + conditions
 
         self.create_forward_rule(
-            listener_arn, target_group_arn, rule_name, priority, combined_conditions
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
         )
 
-        self.io.info(
-            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
         )
 
     def create_source_ip_rule(
@@ -237,6 +242,7 @@ class LoadBalancerProvider:
         rule_name: str,
         priority: int,
         conditions: list,
+        additional_tags: list = [],
     ):
         combined_conditions = [
             {
@@ -246,24 +252,29 @@ class LoadBalancerProvider:
         ] + conditions
 
         self.create_forward_rule(
-            listener_arn, target_group_arn, rule_name, priority, combined_conditions
+            listener_arn,
+            target_group_arn,
+            rule_name,
+            priority,
+            combined_conditions,
+            additional_tags,
         )
 
-        self.io.info(
-            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        self.io.debug(
+            f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
         )
 
-    def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> str:
-        current_rule_arn = None
+    def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> list:
+        deleted_rules = []
 
         for description in tag_descriptions:
             tags = {t["Key"]: t["Value"] for t in description["Tags"]}
             if tags.get("name") == tag_name:
-                current_rule_arn = description["ResourceArn"]
-                if current_rule_arn:
-                    self.evlb_client.delete_rule(RuleArn=current_rule_arn)
+                if description["ResourceArn"]:
+                    self.evlb_client.delete_rule(RuleArn=description["ResourceArn"])
+                    deleted_rules.append(description)
 
-        return current_rule_arn
+        return deleted_rules
 
 
 class LoadBalancerException(PlatformException):

dbt_platform_helper/providers/parameter_store.py

@@ -0,0 +1,47 @@
+import boto3
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ParameterStore:
+    def __init__(self, ssm_client: boto3.client):
+        self.ssm_client = ssm_client
+
+    def get_ssm_parameter_by_name(self, parameter_name: str) -> dict:
+        """
+        Retrieves the latest version of a parameter from parameter store for a
+        given name/arn.
+
+        Args:
+            path (str): The parameter name to retrieve the parameter value for.
+        Returns:
+            dict: A dictionary representation of your ssm parameter
+        """
+
+        return self.ssm_client.get_parameter(Name=parameter_name)["Parameter"]
+
+    def get_ssm_parameters_by_path(self, path: str) -> list:
+        """
+        Retrieves all SSM parameters for a given path from parameter store.
+
+        Args:
+            path (str): The parameter path to retrieve the parameters for. e.g. /copilot/applications/
+        Returns:
+            list: A list of dictionaries containing all SSM parameters under the provided path.
+        """
+
+        parameters = []
+        paginator = self.ssm_client.get_paginator("get_parameters_by_path")
+        page_iterator = paginator.paginate(Path=path, Recursive=True)
+
+        for page in page_iterator:
+            parameters.extend(page.get("Parameters", []))
+
+        if parameters:
+            return parameters
+        else:
+            raise ParameterNotFoundForPathException()
+
+
+class ParameterNotFoundForPathException(PlatformException):
+    """Exception raised when no parameters are found for a given path."""

dbt_platform_helper/providers/platform_config_schema.py

@@ -28,6 +28,7 @@ class PlatformConfigSchema:
                         PlatformConfigSchema.__monitoring_schema(),
                         PlatformConfigSchema.__opensearch_schema(),
                         PlatformConfigSchema.__postgres_schema(),
+                        PlatformConfigSchema.__datadog_schema(),
                         PlatformConfigSchema.__prometheus_policy_schema(),
                         PlatformConfigSchema.__redis_schema(),
                         PlatformConfigSchema.__s3_bucket_schema(),
@@ -48,6 +49,7 @@ class PlatformConfigSchema:
             "postgres": Schema(PlatformConfigSchema.__postgres_schema()),
             "prometheus-policy": Schema(PlatformConfigSchema.__prometheus_policy_schema()),
             "redis": Schema(PlatformConfigSchema.__redis_schema()),
+            "datadog": Schema(PlatformConfigSchema.__datadog_schema()),
             "s3": Schema(PlatformConfigSchema.__s3_bucket_schema()),
             "s3-policy": Schema(PlatformConfigSchema.__s3_bucket_policy_schema()),
             "subscription-filter": PlatformConfigSchema.__no_configuration_required_schema(
@@ -434,6 +436,21 @@ class PlatformConfigSchema:
 
         return True
 
+    @staticmethod
+    def __datadog_schema() -> dict:
+        return {
+            "type": "datadog",
+            Optional("environments"): {
+                Optional(PlatformConfigSchema.__valid_environment_name()): {
+                    "team_name": str,
+                    "contact_name": str,
+                    "contact_email": str,
+                    "documentation_url": str,
+                    "services_to_monitor": list,
+                }
+            },
+        }
+
     @staticmethod
     def __s3_bucket_schema() -> dict:
         def _valid_s3_bucket_arn(key):

dbt_platform_helper/providers/semantic_version.py

@@ -1,12 +1,6 @@
 import re
-from dataclasses import dataclass
-from dataclasses import field
-from typing import Dict
-from typing import Optional
 from typing import Union
 
-from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
 from dbt_platform_helper.providers.validation import ValidationException
 
 
@@ -35,13 +29,20 @@ class SemanticVersion:
             return "unknown"
         return ".".join([str(s) for s in [self.major, self.minor, self.patch]])
 
+    def __repr__(self) -> str:
+        return str(self)
+
     def __lt__(self, other) -> bool:
         return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
 
     def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return (self.major, self.minor, self.patch) == (other.major, other.minor, other.patch)
 
     def validate_compatibility_with(self, other):
+        if other is None:
+            raise ValidationException("Cannot compare NoneType")
         if (self.major == 0 and other.major == 0) and (
             self.minor != other.minor or self.patch != other.patch
         ):
@@ -54,93 +55,22 @@ class SemanticVersion:
             raise IncompatibleMinorVersionException(str(self), str(other))
 
     @staticmethod
-    def from_string(version_string: Union[str, None]):
+    def _cast_to_int_with_fallback(input, fallback=-1):
+        try:
+            return int(input)
+        except ValueError:
+            return fallback
+
+    @classmethod
+    def from_string(self, version_string: Union[str, None]):
         if version_string is None:
             return None
 
-        version_plain = version_string.replace("v", "")
-        version_segments = re.split(r"[.\-]", version_plain)
+        version_segments = re.split(r"[.\-]", version_string.replace("v", ""))
 
         if len(version_segments) != 3:
             return None
 
-        output_version = [0, 0, 0]
-        for index, segment in enumerate(version_segments):
-            try:
-                output_version[index] = int(segment)
-            except ValueError:
-                output_version[index] = -1
-
-        return SemanticVersion(output_version[0], output_version[1], output_version[2])
-
-
-@dataclass
-class VersionStatus:
-    local: SemanticVersion = None
-    latest: SemanticVersion = None
-
-    def __str__(self):
-        attrs = {
-            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
-        }
-        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
-        return f"{self.__class__.__name__}: {attrs_str}"
-
-    def is_outdated(self):
-        return self.local != self.latest
-
-    def validate(self):
-        pass
-
-
-@dataclass
-class PlatformHelperVersionStatus(VersionStatus):
-    local: Optional[SemanticVersion] = None
-    latest: Optional[SemanticVersion] = None
-    deprecated_version_file: Optional[SemanticVersion] = None
-    platform_config_default: Optional[SemanticVersion] = None
-    pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
-
-    def __str__(self):
-        semantic_version_attrs = {
-            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
-        }
-
-        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
-
-        if self.pipeline_overrides.items():
-            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
-                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
-            )
-            class_str = ", ".join([class_str, pipeline_overrides_str])
-
-        return f"{self.__class__.__name__}: {class_str}"
-
-    def validate(self) -> dict:
-        if self.platform_config_default and not self.deprecated_version_file:
-            return {}
-
-        warnings = []
-        errors = []
-
-        missing_default_version_message = f"Create a section in the root of '{PLATFORM_CONFIG_FILE}':\n\ndefault_versions:\n  platform-helper: "
-        deprecation_message = (
-            f"Please delete '{PLATFORM_HELPER_VERSION_FILE}' as it is now deprecated."
-        )
-
-        if self.platform_config_default and self.deprecated_version_file:
-            warnings.append(deprecation_message)
-
-        if not self.platform_config_default and self.deprecated_version_file:
-            warnings.append(deprecation_message)
-            warnings.append(f"{missing_default_version_message}{self.deprecated_version_file}\n")
-
-        if not self.platform_config_default and not self.deprecated_version_file:
-            message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
-            message += f"{missing_default_version_message}{self.local}\n"
-            errors.append(message)
+        major, minor, patch = [self._cast_to_int_with_fallback(s) for s in version_segments]
 
-        return {
-            "warnings": warnings,
-            "errors": errors,
-        }
+        return SemanticVersion(major, minor, patch)

dbt_platform_helper/providers/terraform_manifest.py

@@ -128,6 +128,7 @@ class TerraformManifestProvider:
                 "codebase": "${each.key}",
                 "repository": "${each.value.repository}",
                 "deploy_repository": f"{deploy_repository}",
+                "deploy_repository_branch": '${lookup(each.value, "deploy_repository_branch", "main")}',
                 "additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
                 "pipelines": '${lookup(each.value, "pipelines", [])}',
                 "services": "${each.value.services}",

dbt_platform_helper/providers/version.py

@@ -1,18 +1,36 @@
+import re
+import subprocess
 from abc import ABC
+from abc import abstractmethod
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version
+from pathlib import Path
+from typing import Union
 
-import requests
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.yaml_file import FileProviderException
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
-class LocalVersionProviderException(PlatformException):
+def set_up_retry():
+    session = Session()
+    retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[403, 500, 502, 503, 504])
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+    return session
+
+
+class InstalledVersionProviderException(PlatformException):
     pass
 
 
-class InstalledToolNotFoundException(LocalVersionProviderException):
+class InstalledToolNotFoundException(InstalledVersionProviderException):
     def __init__(
         self,
         tool_name: str,
@@ -21,40 +39,98 @@ class InstalledToolNotFoundException(LocalVersionProviderException):
 
 
 class VersionProvider(ABC):
-    pass
+    @abstractmethod
+    def get_semantic_version() -> SemanticVersion:
+        raise NotImplementedError("Must be implemented in subclasses")
 
 
-class LocalVersionProvider:
+class InstalledVersionProvider:
     @staticmethod
-    def get_installed_tool_version(tool_name: str) -> SemanticVersion:
+    def get_semantic_version(tool_name: str) -> SemanticVersion:
         try:
             return SemanticVersion.from_string(version(tool_name))
         except PackageNotFoundError:
             raise InstalledToolNotFoundException(tool_name)
 
 
-# TODO add timeouts and exception handling for requests
-# TODO Alternatively use the gitpython package?
-class GithubVersionProvider(VersionProvider):
+class GithubLatestVersionProvider(VersionProvider):
     @staticmethod
-    def get_latest_version(repo_name: str, tags: bool = False) -> SemanticVersion:
-        if tags:
-            tags_list = requests.get(f"https://api.github.com/repos/{repo_name}/tags").json()
-            versions = [SemanticVersion.from_string(v["name"]) for v in tags_list]
-            versions.sort(reverse=True)
-            return versions[0]
+    def get_semantic_version(
+        repo_name: str, tags: bool = False, request_session=set_up_retry(), io=ClickIOProvider()
+    ) -> Union[SemanticVersion, None]:
+
+        semantic_version = None
+        try:
+            if tags:
+                response = request_session.get(f"https://api.github.com/repos/{repo_name}/tags")
+
+                versions = [SemanticVersion.from_string(v["name"]) for v in response.json()]
+                versions.sort(reverse=True)
+                semantic_version = versions[0]
+            else:
+                package_info = request_session.get(
+                    f"https://api.github.com/repos/{repo_name}/releases/latest"
+                ).json()
+                semantic_version = SemanticVersion.from_string(package_info["tag_name"])
+        except Exception as e:
+            io.error(f"Exception occured when calling Github with:\n{str(e)}")
 
-        package_info = requests.get(
-            f"https://api.github.com/repos/{repo_name}/releases/latest"
-        ).json()
-        return SemanticVersion.from_string(package_info["tag_name"])
+        return semantic_version
 
 
-class PyPiVersionProvider(VersionProvider):
+class PyPiLatestVersionProvider(VersionProvider):
     @staticmethod
-    def get_latest_version(project_name: str) -> SemanticVersion:
-        package_info = requests.get(f"https://pypi.org/pypi/{project_name}/json").json()
-        released_versions = package_info["releases"].keys()
-        parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
-        parsed_released_versions.sort(reverse=True)
-        return parsed_released_versions[0]
+    def get_semantic_version(
+        project_name: str, request_session=set_up_retry(), io=ClickIOProvider()
+    ) -> Union[SemanticVersion, None]:
+        semantic_version = None
+        try:
+            package_info = request_session.get(f"https://pypi.org/pypi/{project_name}/json").json()
+            released_versions = package_info["releases"].keys()
+            parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
+            parsed_released_versions.sort(reverse=True)
+            semantic_version = parsed_released_versions[0]
+        except Exception as e:
+            io.error(f"Exception occured when calling PyPi with:\n{str(e)}")
+        return semantic_version
+
+
+class DeprecatedVersionFileVersionProvider(VersionProvider):
+    def __init__(self, file_provider: YamlFileProvider):
+        self.file_provider = file_provider or YamlFileProvider
+
+    def get_semantic_version(self) -> Union[SemanticVersion, None]:
+        deprecated_version_file = Path(PLATFORM_HELPER_VERSION_FILE)
+        try:
+            loaded_version = self.file_provider.load(deprecated_version_file)
+            version_from_file = SemanticVersion.from_string(loaded_version)
+        except FileProviderException:
+            version_from_file = None
+        return version_from_file
+
+
+class AWSCLIInstalledVersionProvider(VersionProvider):
+    @staticmethod
+    def get_semantic_version() -> Union[SemanticVersion, None]:
+        installed_aws_version = None
+        try:
+            response = subprocess.run("aws --version", capture_output=True, shell=True)
+            matched = re.match(r"aws-cli/([0-9.]+)", response.stdout.decode("utf8"))
+            installed_aws_version = matched.group(1)
+        except (ValueError, AttributeError):
+            pass
+        return SemanticVersion.from_string(installed_aws_version)
+
+
+class CopilotInstalledVersionProvider(VersionProvider):
+    @staticmethod
+    def get_semantic_version() -> Union[SemanticVersion, None]:
+        copilot_version = None
+
+        try:
+            response = subprocess.run("copilot --version", capture_output=True, shell=True)
+            [copilot_version] = re.findall(r"[0-9.]+", response.stdout.decode("utf8"))
+        except ValueError:
+            pass
+
+        return SemanticVersion.from_string(copilot_version)

dbt_platform_helper/providers/version_status.py

@@ -0,0 +1,80 @@
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Dict
+from typing import Optional
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+
+
+@dataclass
+class VersionStatus:
+    installed: SemanticVersion = None
+    latest: SemanticVersion = None
+
+    def __str__(self):
+        attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
+        return f"{self.__class__.__name__}: {attrs_str}"
+
+    def is_outdated(self):
+        return self.installed != self.latest
+
+    def validate(self):
+        pass
+
+
+@dataclass
+class PlatformHelperVersionStatus(VersionStatus):
+    installed: Optional[SemanticVersion] = None
+    latest: Optional[SemanticVersion] = None
+    deprecated_version_file: Optional[SemanticVersion] = None
+    platform_config_default: Optional[SemanticVersion] = None
+    pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
+
+    def __str__(self):
+        semantic_version_attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+
+        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
+
+        if self.pipeline_overrides.items():
+            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
+                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
+            )
+            class_str = ", ".join([class_str, pipeline_overrides_str])
+
+        return f"{self.__class__.__name__}: {class_str}"
+
+    def validate(self) -> dict:
+        if self.platform_config_default and not self.deprecated_version_file:
+            return {}
+
+        warnings = []
+        errors = []
+
+        missing_default_version_message = f"Create a section in the root of '{PLATFORM_CONFIG_FILE}':\n\ndefault_versions:\n  platform-helper: "
+        deprecation_message = (
+            f"Please delete '{PLATFORM_HELPER_VERSION_FILE}' as it is now deprecated."
+        )
+
+        if self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+
+        if not self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+            warnings.append(f"{missing_default_version_message}{self.deprecated_version_file}\n")
+
+        if not self.platform_config_default and not self.deprecated_version_file:
+            message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
+            message += f"{missing_default_version_message}{self.installed}\n"
+            errors.append(message)
+
+        return {
+            "warnings": warnings,
+            "errors": errors,
+        }

dbt_platform_helper/providers/yaml_file.py

@@ -45,7 +45,6 @@ class YamlFileProvider:
 
         if not yaml_content:
             return {}
-
         YamlFileProvider.lint_yaml_for_duplicate_keys(path)
 
         return yaml_content