dbt-platform-helper 13.2.0__py3-none-any.whl → 13.4.0__py3-none-any.whl

dbt_platform_helper/providers/config_validator.py

@@ -17,7 +17,10 @@ class ConfigValidatorError(PlatformException):
 class ConfigValidator:
 
     def __init__(
-        self, validations: Callable[[dict], None] = None, io: ClickIOProvider = ClickIOProvider()
+        self,
+        validations: Callable[[dict], None] = None,
+        io: ClickIOProvider = ClickIOProvider(),
+        session: boto3.Session = None,
     ):
         self.validations = validations or [
             self.validate_supported_redis_versions,
@@ -28,6 +31,7 @@ class ConfigValidator:
             self.validate_database_migration_input_sources,
         ]
         self.io = io
+        self.session = session
 
     def run_validations(self, config: dict):
         for validation in self.validations:
@@ -76,10 +80,15 @@ class ConfigValidator:
                 f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
             )
 
+    def _get_client(self, service_name: str):
+        if self.session:
+            return self.session.client(service_name)
+        return boto3.client(service_name)
+
     def validate_supported_redis_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            aws_provider=Redis(boto3.client("elasticache")),
+            aws_provider=Redis(self._get_client("elasticache")),
             extension_type="redis",  # TODO this is information which can live in the RedisProvider
             version_key="engine",  # TODO this is information which can live in the RedisProvider
         )
@@ -87,7 +96,7 @@ class ConfigValidator:
     def validate_supported_opensearch_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            aws_provider=Opensearch(boto3.client("opensearch")),
+            aws_provider=Opensearch(self._get_client("opensearch")),
             extension_type="opensearch",  # TODO this is information which can live in the OpensearchProvider
             version_key="engine",  # TODO this is information which can live in the OpensearchProvider
         )
@@ -206,21 +215,14 @@ class ConfigValidator:
                         f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
                     )
 
+                # TODO - The from_account and to_account properties are deprecated and will be removed when terraform-platform-modules is merged with platform-tools
                 if from_account != to_account:
-                    if "from_account" not in section:
-                        errors.append(
-                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'from_account' parameter must be present."
-                        )
-                    elif section["from_account"] != from_account:
+                    if "from_account" in section and section["from_account"] != from_account:
                         errors.append(
                             f"Incorrect value for 'from_account' for environment '{from_env}'"
                         )
 
-                    if "to_account" not in section:
-                        errors.append(
-                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'to_account' parameter must be present."
-                        )
-                    elif section["to_account"] != to_account:
+                    if "to_account" in section and section["to_account"] != to_account:
                         errors.append(
                             f"Incorrect value for 'to_account' for environment '{to_env}'"
                         )

dbt_platform_helper/providers/ecr.py

@@ -1,20 +1,77 @@
+import botocore
 from boto3 import Session
 
+from dbt_platform_helper.providers.aws.exceptions import ImageNotFoundException
+from dbt_platform_helper.providers.aws.exceptions import RepositoryNotFoundException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 
 
 class ECRProvider:
-    def __init__(self, session: Session = None):
+    def __init__(self, session: Session = None, click_io: ClickIOProvider = ClickIOProvider()):
         self.session = session
-        self.client = None
-
-    def _get_client(self):
-        if not self.session:
-            self.session = get_aws_session_or_abort()
-        return self.session.client("ecr")
+        self.click_io = click_io
 
     def get_ecr_repo_names(self) -> list[str]:
         out = []
         for page in self._get_client().get_paginator("describe_repositories").paginate():
             out.extend([repo["repositoryName"] for repo in page.get("repositories", {})])
         return out
+
+    def get_image_details(
+        self, application: Application, codebase: str, image_ref: str
+    ) -> list[dict]:
+        """Check if image exists in AWS ECR, and return a list of dictionaries
+        containing image metadata."""
+
+        repository = f"{application.name}/{codebase}"
+
+        try:
+            image_info = self._get_client().describe_images(
+                repositoryName=repository,
+                imageIds=[{"imageTag": image_ref}],
+            )
+
+            self._check_image_details_exists(image_info, image_ref)
+
+            return image_info.get("imageDetails")
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "ImageNotFoundException":
+                raise ImageNotFoundException(image_ref)
+            if e.response["Error"]["Code"] == "RepositoryNotFoundException":
+                raise RepositoryNotFoundException(repository)
+
+    def find_commit_tag(self, image_details: list[dict], image_ref: str) -> str:
+        """Loop through imageTags list to query for an image tag starting with
+        'commit-', and return that value if found."""
+
+        if image_ref.startswith("commit-"):
+            return image_ref
+
+        if image_details:
+            for image in image_details:
+                image_tags = image.get("imageTags", {})
+                for tag in image_tags:
+                    if tag.startswith("commit-"):
+                        self.click_io.info(
+                            f'INFO: The tag "{image_ref}" is not a unique, commit-specific tag. Deploying the corresponding commit tag "{tag}" instead.'
+                        )
+                        return tag
+        self.click_io.warn(
+            f'WARNING: The AWS ECR image "{image_ref}" has no associated commit tag so deploying "{image_ref}". Note this could result in images with unintended or incompatible changes being deployed if new ECS Tasks for your service.'
+        )
+        return image_ref
+
+    @staticmethod
+    def _check_image_details_exists(image_info: dict, image_ref: str):
+        """Error handling for any unexpected scenario where AWS ECR returns a
+        malformed response."""
+
+        if "imageDetails" not in image_info:
+            raise ImageNotFoundException(image_ref)
+
+    def _get_client(self):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client("ecr")

dbt_platform_helper/providers/io.py

@@ -13,8 +13,8 @@ class ClickIOProvider:
     def error(self, message: str):
         click.secho(f"Error: {message}", fg="red")
 
-    def info(self, message: str):
-        click.secho(message)
+    def info(self, message: str, **kwargs):
+        click.secho(message, **kwargs)
 
     def input(self, message: str) -> str:
         return click.prompt(message)

dbt_platform_helper/providers/parameter_store.py

@@ -0,0 +1,47 @@
+import boto3
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ParameterStore:
+    def __init__(self, ssm_client: boto3.client):
+        self.ssm_client = ssm_client
+
+    def get_ssm_parameter_by_name(self, parameter_name: str) -> dict:
+        """
+        Retrieves the latest version of a parameter from parameter store for a
+        given name/arn.
+
+        Args:
+            path (str): The parameter name to retrieve the parameter value for.
+        Returns:
+            dict: A dictionary representation of your ssm parameter
+        """
+
+        return self.ssm_client.get_parameter(Name=parameter_name)["Parameter"]
+
+    def get_ssm_parameters_by_path(self, path: str) -> list:
+        """
+        Retrieves all SSM parameters for a given path from parameter store.
+
+        Args:
+            path (str): The parameter path to retrieve the parameters for. e.g. /copilot/applications/
+        Returns:
+            list: A list of dictionaries containing all SSM parameters under the provided path.
+        """
+
+        parameters = []
+        paginator = self.ssm_client.get_paginator("get_parameters_by_path")
+        page_iterator = paginator.paginate(Path=path, Recursive=True)
+
+        for page in page_iterator:
+            parameters.extend(page.get("Parameters", []))
+
+        if parameters:
+            return parameters
+        else:
+            raise ParameterNotFoundForPathException()
+
+
+class ParameterNotFoundForPathException(PlatformException):
+    """Exception raised when no parameters are found for a given path."""

dbt_platform_helper/providers/platform_config_schema.py

@@ -28,6 +28,7 @@ class PlatformConfigSchema:
                         PlatformConfigSchema.__monitoring_schema(),
                         PlatformConfigSchema.__opensearch_schema(),
                         PlatformConfigSchema.__postgres_schema(),
+                        PlatformConfigSchema.__datadog_schema(),
                         PlatformConfigSchema.__prometheus_policy_schema(),
                         PlatformConfigSchema.__redis_schema(),
                         PlatformConfigSchema.__s3_bucket_schema(),
@@ -48,6 +49,7 @@ class PlatformConfigSchema:
             "postgres": Schema(PlatformConfigSchema.__postgres_schema()),
             "prometheus-policy": Schema(PlatformConfigSchema.__prometheus_policy_schema()),
             "redis": Schema(PlatformConfigSchema.__redis_schema()),
+            "datadog": Schema(PlatformConfigSchema.__datadog_schema()),
             "s3": Schema(PlatformConfigSchema.__s3_bucket_schema()),
             "s3-policy": Schema(PlatformConfigSchema.__s3_bucket_policy_schema()),
             "subscription-filter": PlatformConfigSchema.__no_configuration_required_schema(
@@ -434,6 +436,21 @@ class PlatformConfigSchema:
 
         return True
 
+    @staticmethod
+    def __datadog_schema() -> dict:
+        return {
+            "type": "datadog",
+            Optional("environments"): {
+                Optional(PlatformConfigSchema.__valid_environment_name()): {
+                    "team_name": str,
+                    "contact_name": str,
+                    "contact_email": str,
+                    "documentation_url": str,
+                    "services_to_monitor": list,
+                }
+            },
+        }
+
     @staticmethod
     def __s3_bucket_schema() -> dict:
         def _valid_s3_bucket_arn(key):

dbt_platform_helper/providers/semantic_version.py

@@ -1,12 +1,6 @@
 import re
-from dataclasses import dataclass
-from dataclasses import field
-from typing import Dict
-from typing import Optional
 from typing import Union
 
-from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
 from dbt_platform_helper.providers.validation import ValidationException
 
 
@@ -42,9 +36,13 @@ class SemanticVersion:
         return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
 
     def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return (self.major, self.minor, self.patch) == (other.major, other.minor, other.patch)
 
     def validate_compatibility_with(self, other):
+        if other is None:
+            raise ValidationException("Cannot compare NoneType")
         if (self.major == 0 and other.major == 0) and (
             self.minor != other.minor or self.patch != other.patch
         ):
@@ -57,93 +55,22 @@ class SemanticVersion:
             raise IncompatibleMinorVersionException(str(self), str(other))
 
     @staticmethod
-    def from_string(version_string: Union[str, None]):
+    def _cast_to_int_with_fallback(input, fallback=-1):
+        try:
+            return int(input)
+        except ValueError:
+            return fallback
+
+    @classmethod
+    def from_string(self, version_string: Union[str, None]):
         if version_string is None:
             return None
 
-        version_plain = version_string.replace("v", "")
-        version_segments = re.split(r"[.\-]", version_plain)
+        version_segments = re.split(r"[.\-]", version_string.replace("v", ""))
 
         if len(version_segments) != 3:
             return None
 
-        output_version = [0, 0, 0]
-        for index, segment in enumerate(version_segments):
-            try:
-                output_version[index] = int(segment)
-            except ValueError:
-                output_version[index] = -1
+        major, minor, patch = [self._cast_to_int_with_fallback(s) for s in version_segments]
 
-        return SemanticVersion(output_version[0], output_version[1], output_version[2])
-
-
-@dataclass
-class VersionStatus:
-    installed: SemanticVersion = None
-    latest: SemanticVersion = None
-
-    def __str__(self):
-        attrs = {
-            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
-        }
-        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
-        return f"{self.__class__.__name__}: {attrs_str}"
-
-    def is_outdated(self):
-        return self.installed != self.latest
-
-    def validate(self):
-        pass
-
-
-@dataclass
-class PlatformHelperVersionStatus(VersionStatus):
-    installed: Optional[SemanticVersion] = None
-    latest: Optional[SemanticVersion] = None
-    deprecated_version_file: Optional[SemanticVersion] = None
-    platform_config_default: Optional[SemanticVersion] = None
-    pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
-
-    def __str__(self):
-        semantic_version_attrs = {
-            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
-        }
-
-        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
-
-        if self.pipeline_overrides.items():
-            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
-                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
-            )
-            class_str = ", ".join([class_str, pipeline_overrides_str])
-
-        return f"{self.__class__.__name__}: {class_str}"
-
-    def validate(self) -> dict:
-        if self.platform_config_default and not self.deprecated_version_file:
-            return {}
-
-        warnings = []
-        errors = []
-
-        missing_default_version_message = f"Create a section in the root of '{PLATFORM_CONFIG_FILE}':\n\ndefault_versions:\n  platform-helper: "
-        deprecation_message = (
-            f"Please delete '{PLATFORM_HELPER_VERSION_FILE}' as it is now deprecated."
-        )
-
-        if self.platform_config_default and self.deprecated_version_file:
-            warnings.append(deprecation_message)
-
-        if not self.platform_config_default and self.deprecated_version_file:
-            warnings.append(deprecation_message)
-            warnings.append(f"{missing_default_version_message}{self.deprecated_version_file}\n")
-
-        if not self.platform_config_default and not self.deprecated_version_file:
-            message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
-            message += f"{missing_default_version_message}{self.installed}\n"
-            errors.append(message)
-
-        return {
-            "warnings": warnings,
-            "errors": errors,
-        }
+        return SemanticVersion(major, minor, patch)

dbt_platform_helper/providers/terraform_manifest.py

@@ -128,6 +128,7 @@ class TerraformManifestProvider:
                 "codebase": "${each.key}",
                 "repository": "${each.value.repository}",
                 "deploy_repository": f"{deploy_repository}",
+                "deploy_repository_branch": '${lookup(each.value, "deploy_repository_branch", "main")}',
                 "additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
                 "pipelines": '${lookup(each.value, "pipelines", [])}',
                 "services": "${each.value.services}",

dbt_platform_helper/providers/version.py

@@ -1,17 +1,31 @@
+import re
+import subprocess
 from abc import ABC
+from abc import abstractmethod
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version
 from pathlib import Path
+from typing import Union
 
-import requests
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
 from dbt_platform_helper.providers.yaml_file import FileProviderException
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
+def set_up_retry():
+    session = Session()
+    retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[403, 500, 502, 503, 504])
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+    return session
+
+
 class InstalledVersionProviderException(PlatformException):
     pass
 
@@ -25,50 +39,67 @@ class InstalledToolNotFoundException(InstalledVersionProviderException):
 
 
 class VersionProvider(ABC):
-    pass
+    @abstractmethod
+    def get_semantic_version() -> SemanticVersion:
+        raise NotImplementedError("Must be implemented in subclasses")
 
 
 class InstalledVersionProvider:
     @staticmethod
-    def get_installed_tool_version(tool_name: str) -> SemanticVersion:
+    def get_semantic_version(tool_name: str) -> SemanticVersion:
         try:
             return SemanticVersion.from_string(version(tool_name))
         except PackageNotFoundError:
             raise InstalledToolNotFoundException(tool_name)
 
 
-# TODO add timeouts and exception handling for requests
-# TODO Alternatively use the gitpython package?
-class GithubVersionProvider(VersionProvider):
+class GithubLatestVersionProvider(VersionProvider):
     @staticmethod
-    def get_latest_version(repo_name: str, tags: bool = False) -> SemanticVersion:
-        if tags:
-            tags_list = requests.get(f"https://api.github.com/repos/{repo_name}/tags").json()
-            versions = [SemanticVersion.from_string(v["name"]) for v in tags_list]
-            versions.sort(reverse=True)
-            return versions[0]
+    def get_semantic_version(
+        repo_name: str, tags: bool = False, request_session=set_up_retry(), io=ClickIOProvider()
+    ) -> Union[SemanticVersion, None]:
+
+        semantic_version = None
+        try:
+            if tags:
+                response = request_session.get(f"https://api.github.com/repos/{repo_name}/tags")
 
-        package_info = requests.get(
-            f"https://api.github.com/repos/{repo_name}/releases/latest"
-        ).json()
-        return SemanticVersion.from_string(package_info["tag_name"])
+                versions = [SemanticVersion.from_string(v["name"]) for v in response.json()]
+                versions.sort(reverse=True)
+                semantic_version = versions[0]
+            else:
+                package_info = request_session.get(
+                    f"https://api.github.com/repos/{repo_name}/releases/latest"
+                ).json()
+                semantic_version = SemanticVersion.from_string(package_info["tag_name"])
+        except Exception as e:
+            io.error(f"Exception occured when calling Github with:\n{str(e)}")
 
+        return semantic_version
 
-class PyPiVersionProvider(VersionProvider):
+
+class PyPiLatestVersionProvider(VersionProvider):
     @staticmethod
-    def get_latest_version(project_name: str) -> SemanticVersion:
-        package_info = requests.get(f"https://pypi.org/pypi/{project_name}/json").json()
-        released_versions = package_info["releases"].keys()
-        parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
-        parsed_released_versions.sort(reverse=True)
-        return parsed_released_versions[0]
+    def get_semantic_version(
+        project_name: str, request_session=set_up_retry(), io=ClickIOProvider()
+    ) -> Union[SemanticVersion, None]:
+        semantic_version = None
+        try:
+            package_info = request_session.get(f"https://pypi.org/pypi/{project_name}/json").json()
+            released_versions = package_info["releases"].keys()
+            parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
+            parsed_released_versions.sort(reverse=True)
+            semantic_version = parsed_released_versions[0]
+        except Exception as e:
+            io.error(f"Exception occured when calling PyPi with:\n{str(e)}")
+        return semantic_version
 
 
 class DeprecatedVersionFileVersionProvider(VersionProvider):
     def __init__(self, file_provider: YamlFileProvider):
         self.file_provider = file_provider or YamlFileProvider
 
-    def get_required_version(self) -> SemanticVersion:
+    def get_semantic_version(self) -> Union[SemanticVersion, None]:
         deprecated_version_file = Path(PLATFORM_HELPER_VERSION_FILE)
         try:
             loaded_version = self.file_provider.load(deprecated_version_file)
@@ -76,3 +107,30 @@ class DeprecatedVersionFileVersionProvider(VersionProvider):
         except FileProviderException:
             version_from_file = None
         return version_from_file
+
+
+class AWSCLIInstalledVersionProvider(VersionProvider):
+    @staticmethod
+    def get_semantic_version() -> Union[SemanticVersion, None]:
+        installed_aws_version = None
+        try:
+            response = subprocess.run("aws --version", capture_output=True, shell=True)
+            matched = re.match(r"aws-cli/([0-9.]+)", response.stdout.decode("utf8"))
+            installed_aws_version = matched.group(1)
+        except (ValueError, AttributeError):
+            pass
+        return SemanticVersion.from_string(installed_aws_version)
+
+
+class CopilotInstalledVersionProvider(VersionProvider):
+    @staticmethod
+    def get_semantic_version() -> Union[SemanticVersion, None]:
+        copilot_version = None
+
+        try:
+            response = subprocess.run("copilot --version", capture_output=True, shell=True)
+            [copilot_version] = re.findall(r"[0-9.]+", response.stdout.decode("utf8"))
+        except ValueError:
+            pass
+
+        return SemanticVersion.from_string(copilot_version)

dbt_platform_helper/providers/version_status.py

@@ -0,0 +1,80 @@
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Dict
+from typing import Optional
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+
+
+@dataclass
+class VersionStatus:
+    installed: SemanticVersion = None
+    latest: SemanticVersion = None
+
+    def __str__(self):
+        attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+        attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
+        return f"{self.__class__.__name__}: {attrs_str}"
+
+    def is_outdated(self):
+        return self.installed != self.latest
+
+    def validate(self):
+        pass
+
+
+@dataclass
+class PlatformHelperVersionStatus(VersionStatus):
+    installed: Optional[SemanticVersion] = None
+    latest: Optional[SemanticVersion] = None
+    deprecated_version_file: Optional[SemanticVersion] = None
+    platform_config_default: Optional[SemanticVersion] = None
+    pipeline_overrides: Optional[Dict[str, str]] = field(default_factory=dict)
+
+    def __str__(self):
+        semantic_version_attrs = {
+            key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
+        }
+
+        class_str = ", ".join(f"{key}: {value}" for key, value in semantic_version_attrs.items())
+
+        if self.pipeline_overrides.items():
+            pipeline_overrides_str = "pipeline_overrides: " + ", ".join(
+                f"{key}: {value}" for key, value in self.pipeline_overrides.items()
+            )
+            class_str = ", ".join([class_str, pipeline_overrides_str])
+
+        return f"{self.__class__.__name__}: {class_str}"
+
+    def validate(self) -> dict:
+        if self.platform_config_default and not self.deprecated_version_file:
+            return {}
+
+        warnings = []
+        errors = []
+
+        missing_default_version_message = f"Create a section in the root of '{PLATFORM_CONFIG_FILE}':\n\ndefault_versions:\n  platform-helper: "
+        deprecation_message = (
+            f"Please delete '{PLATFORM_HELPER_VERSION_FILE}' as it is now deprecated."
+        )
+
+        if self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+
+        if not self.platform_config_default and self.deprecated_version_file:
+            warnings.append(deprecation_message)
+            warnings.append(f"{missing_default_version_message}{self.deprecated_version_file}\n")
+
+        if not self.platform_config_default and not self.deprecated_version_file:
+            message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
+            message += f"{missing_default_version_message}{self.installed}\n"
+            errors.append(message)
+
+        return {
+            "warnings": warnings,
+            "errors": errors,
+        }