dbt-platform-helper 13.1.1__py3-none-any.whl → 13.2.0__py3-none-any.whl

dbt_platform_helper/domain/pipelines.py

@@ -14,7 +14,7 @@ from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.versioning import (
+from dbt_platform_helper.utils.tool_versioning import (
     get_required_terraform_platform_modules_version,
 )
 

dbt_platform_helper/domain/terraform_environment.py

@@ -1,7 +1,7 @@
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
-from dbt_platform_helper.utils.versioning import (
+from dbt_platform_helper.utils.tool_versioning import (
     get_required_terraform_platform_modules_version,
 )
 

dbt_platform_helper/domain/versioning.py

@@ -0,0 +1,157 @@
+import os
+
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMajorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMinorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.version import DeprecatedVersionFileVersionProvider
+from dbt_platform_helper.providers.version import InstalledVersionProvider
+from dbt_platform_helper.providers.version import PyPiVersionProvider
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+def running_as_installed_package():
+    return "site-packages" in __file__
+
+
+def skip_version_checks():
+    return not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ
+
+
+class PlatformHelperVersionNotFoundException(PlatformException):
+    def __init__(self):
+        super().__init__(f"""Platform helper version could not be resolved.""")
+
+
+class PlatformHelperVersioning:
+    def __init__(
+        self,
+        io: ClickIOProvider = ClickIOProvider(),
+        version_file_version_provider: DeprecatedVersionFileVersionProvider = DeprecatedVersionFileVersionProvider(
+            YamlFileProvider
+        ),
+        config_provider: ConfigProvider = ConfigProvider(),
+        pypi_provider: PyPiVersionProvider = PyPiVersionProvider,
+        installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
+        skip_versioning_checks: bool = None,
+    ):
+        self.io = io
+        self.version_file_version_provider = version_file_version_provider
+        self.config_provider = config_provider
+        self.pypi_provider = pypi_provider
+        self.installed_version_provider = installed_version_provider
+        self.skip_versioning_checks = (
+            skip_versioning_checks if skip_versioning_checks is not None else skip_version_checks()
+        )
+
+    def get_required_version(self, pipeline=None):
+        version_status = self._get_version_status()
+        self.io.process_messages(version_status.validate())
+        required_version = self._resolve_required_version(pipeline, version_status)
+        self.io.info(required_version)
+        return required_version
+
+    # Used in the generate command
+    def check_platform_helper_version_mismatch(self):
+        if self.skip_versioning_checks:
+            return
+
+        version_status = self._get_version_status()
+        self.io.process_messages(version_status.validate())
+
+        required_version = SemanticVersion.from_string(
+            self._resolve_required_version(version_status=version_status)
+        )
+
+        if not version_status.installed == required_version:
+            message = (
+                f"WARNING: You are running platform-helper v{version_status.installed} against "
+                f"v{required_version} specified for the project."
+            )
+            self.io.warn(message)
+
+    def check_if_needs_update(self):
+        if self.skip_versioning_checks:
+            return
+
+        version_status = self._get_version_status(include_project_versions=False)
+
+        message = (
+            f"You are running platform-helper v{version_status.installed}, upgrade to "
+            f"v{version_status.latest} by running run `pip install "
+            "--upgrade dbt-platform-helper`."
+        )
+
+        try:
+            version_status.installed.validate_compatibility_with(version_status.latest)
+        except IncompatibleMajorVersionException:
+            self.io.error(message)
+        except IncompatibleMinorVersionException:
+            self.io.warn(message)
+
+    def _get_version_status(
+        self,
+        include_project_versions: bool = True,
+    ) -> PlatformHelperVersionStatus:
+        locally_installed_version = self.installed_version_provider.get_installed_tool_version(
+            "dbt-platform-helper"
+        )
+
+        latest_release = self.pypi_provider.get_latest_version("dbt-platform-helper")
+
+        if not include_project_versions:
+            return PlatformHelperVersionStatus(
+                installed=locally_installed_version,
+                latest=latest_release,
+            )
+
+        platform_config_default, pipeline_overrides = None, {}
+
+        platform_config = self.config_provider.load_unvalidated_config_file()
+
+        if platform_config:
+            platform_config_default = SemanticVersion.from_string(
+                platform_config.get("default_versions", {}).get("platform-helper")
+            )
+
+            pipeline_overrides = {
+                name: pipeline.get("versions", {}).get("platform-helper")
+                for name, pipeline in platform_config.get("environment_pipelines", {}).items()
+                if pipeline.get("versions", {}).get("platform-helper")
+            }
+        out = PlatformHelperVersionStatus(
+            installed=locally_installed_version,
+            latest=latest_release,
+            deprecated_version_file=self.version_file_version_provider.get_required_version(),
+            platform_config_default=platform_config_default,
+            pipeline_overrides=pipeline_overrides,
+        )
+
+        return out
+
+    def _resolve_required_version(
+        self, pipeline: str = None, version_status: PlatformHelperVersionStatus = None
+    ) -> str:
+        pipeline_version = version_status.pipeline_overrides.get(pipeline)
+        version_precedence = [
+            pipeline_version,
+            version_status.platform_config_default,
+            version_status.deprecated_version_file,
+        ]
+        non_null_version_precedence = [
+            f"{v}" if isinstance(v, SemanticVersion) else v for v in version_precedence if v
+        ]
+
+        out = non_null_version_precedence[0] if non_null_version_precedence else None
+
+        if not out:
+            raise PlatformHelperVersionNotFoundException
+
+        return out

dbt_platform_helper/providers/aws/interfaces.py

@@ -0,0 +1,13 @@
+from typing import Protocol
+
+
+class GetVersionsProtocol(Protocol):
+    def get_supported_versions(self) -> list[str]: ...
+
+
+class GetReferenceProtocol(Protocol):
+    def get_reference(self) -> str: ...
+
+
+class AwsGetVersionProtocol(GetReferenceProtocol, GetVersionsProtocol):
+    pass

dbt_platform_helper/providers/aws/opensearch.py

@@ -0,0 +1,23 @@
+import boto3
+
+
+class Opensearch:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+        self.engine = "OpenSearch"
+
+    def get_reference(self) -> str:
+        return self.engine.lower()
+
+    def get_supported_versions(self) -> list[str]:
+        response = self.client.list_versions()
+        all_versions = response["Versions"]
+
+        supported_versions = [
+            version.removeprefix(f"{self.engine}_")
+            for version in all_versions
+            if version.startswith(f"{self.engine}_")
+        ]
+
+        return supported_versions

dbt_platform_helper/providers/aws/redis.py

@@ -0,0 +1,21 @@
+import boto3
+
+
+class Redis:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+        self.engine = "redis"
+
+    def get_reference(self) -> str:
+        return self.engine.lower()
+
+    def get_supported_versions(self) -> list[str]:
+        supported_versions_response = self.client.describe_cache_engine_versions(Engine=self.engine)
+
+        supported_versions = [
+            version["EngineVersion"]
+            for version in supported_versions_response["CacheEngineVersions"]
+        ]
+
+        return supported_versions

dbt_platform_helper/providers/cache.py

@@ -1,10 +1,34 @@
 import os
+from abc import ABC
+from abc import abstractmethod
 from datetime import datetime
 
+from dbt_platform_helper.providers.aws.interfaces import AwsGetVersionProtocol
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
-class CacheProvider:
+class GetDataStrategy(ABC):
+    @abstractmethod
+    def retrieve_fresh_data(self):
+        pass
+
+    @abstractmethod
+    def get_data_identifier(self):
+        pass
+
+
+class GetAWSVersionStrategy(GetDataStrategy):
+    def __init__(self, client_provider: AwsGetVersionProtocol):
+        self.client_provider = client_provider
+
+    def retrieve_fresh_data(self):
+        return self.client_provider.get_supported_versions()
+
+    def get_data_identifier(self):
+        return self.client_provider.get_reference()
+
+
+class Cache:
     def __init__(
         self,
         file_provider: YamlFileProvider = None,
@@ -12,13 +36,25 @@ class CacheProvider:
         self._cache_file = ".platform-helper-config-cache.yml"
         self.file_provider = file_provider or YamlFileProvider
 
-    def read_supported_versions_from_cache(self, resource_name):
+    def get_data(self, strategy: GetDataStrategy):
+        """Main method to retrieve caching data using the client-specific
+        strategy."""
+        cache_key = strategy.get_data_identifier()
+        if self._cache_refresh_required(cache_key):
+            data = strategy.retrieve_fresh_data()
+            self._update_cache(cache_key, data)
+        else:
+            data = self._read_from_cache(cache_key)
+
+        return data
+
+    def _read_from_cache(self, resource_name):
 
         platform_helper_config = self.file_provider.load(self._cache_file)
 
         return platform_helper_config.get(resource_name).get("versions")
 
-    def update_cache(self, resource_name, supported_versions):
+    def _update_cache(self, resource_name, supported_versions):
 
         platform_helper_config = {}
 
@@ -40,7 +76,7 @@ class CacheProvider:
             "# [!] This file is autogenerated via the platform-helper. Do not edit.\n",
         )
 
-    def cache_refresh_required(self, resource_name) -> bool:
+    def _cache_refresh_required(self, resource_name) -> bool:
         """
         Checks if the platform-helper should reach out to AWS to 'refresh' its
         cached values.

dbt_platform_helper/providers/config_validator.py

@@ -3,9 +3,11 @@ from typing import Callable
 import boto3
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.aws.opensearch import Opensearch
+from dbt_platform_helper.providers.aws.redis import Redis
+from dbt_platform_helper.providers.cache import Cache
+from dbt_platform_helper.providers.cache import GetAWSVersionStrategy
 from dbt_platform_helper.providers.io import ClickIOProvider
-from dbt_platform_helper.providers.opensearch import OpensearchProvider
-from dbt_platform_helper.providers.redis import RedisProvider
 
 
 class ConfigValidatorError(PlatformException):
@@ -32,7 +34,7 @@ class ConfigValidator:
             validation(config)
 
     def _validate_extension_supported_versions(
-        self, config, extension_type, version_key, get_supported_versions
+        self, config, aws_provider, extension_type, version_key
     ):
         extensions = config.get("extensions", {})
         if not extensions:
@@ -44,7 +46,10 @@ class ConfigValidator:
             if extension.get("type") == extension_type
         ]
 
-        supported_extension_versions = get_supported_versions()
+        # In this format so it can be monkey patched initially via mock_get_data fixture
+        cache_provider = Cache()
+        get_data_strategy = GetAWSVersionStrategy(aws_provider)
+        supported_extension_versions = cache_provider.get_data(get_data_strategy)
         extensions_with_invalid_version = []
 
         for extension in extensions_for_type:
@@ -74,21 +79,17 @@ class ConfigValidator:
     def validate_supported_redis_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            extension_type="redis",
-            version_key="engine",
-            get_supported_versions=RedisProvider(
-                boto3.client("elasticache")
-            ).get_supported_redis_versions,
+            aws_provider=Redis(boto3.client("elasticache")),
+            extension_type="redis",  # TODO this is information which can live in the RedisProvider
+            version_key="engine",  # TODO this is information which can live in the RedisProvider
         )
 
     def validate_supported_opensearch_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            extension_type="opensearch",
-            version_key="engine",
-            get_supported_versions=OpensearchProvider(
-                boto3.client("opensearch")
-            ).get_supported_opensearch_versions,
+            aws_provider=Opensearch(boto3.client("opensearch")),
+            extension_type="opensearch",  # TODO this is information which can live in the OpensearchProvider
+            version_key="engine",  # TODO this is information which can live in the OpensearchProvider
         )
 
     def validate_environment_pipelines(self, config):

dbt_platform_helper/providers/copilot.py

@@ -4,7 +4,7 @@ import time
 from botocore.exceptions import ClientError
 
 from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
-from dbt_platform_helper.providers.aws import CreateTaskTimeoutException
+from dbt_platform_helper.providers.aws.exceptions import CreateTaskTimeoutException
 from dbt_platform_helper.providers.secrets import Secrets
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.messages import abort_with_error

dbt_platform_helper/providers/io.py

@@ -7,6 +7,9 @@ class ClickIOProvider:
     def warn(self, message: str):
         click.secho(message, fg="magenta")
 
+    def debug(self, message: str):
+        click.secho(message, fg="green")
+
     def error(self, message: str):
         click.secho(f"Error: {message}", fg="red")
 
@@ -26,6 +29,20 @@ class ClickIOProvider:
         click.secho(f"Error: {message}", err=True, fg="red")
         exit(1)
 
+    # TODO messages will be a ValidationMessages class rather than a free-rein dictionary
+    def process_messages(self, messages: dict):
+        if not messages:
+            return
+
+        if messages.get("errors"):
+            self.error("\n".join(messages["errors"]))
+
+        if messages.get("warnings"):
+            self.warn("\n".join(messages["warnings"]))
+
+        if messages.get("info"):
+            self.info("\n".join(messages["info"]))
+
 
 class ClickIOProviderException(PlatformException):
     pass

dbt_platform_helper/providers/kms.py

@@ -0,0 +1,22 @@
+import boto3
+
+
+class KMSProvider:
+    """A provider class for interacting with the AWS KMS (Key Management
+    Service)."""
+
+    def __init__(self, kms_client: boto3.client):
+        self.kms_client = kms_client
+
+    def describe_key(self, alias_name: str) -> dict:
+        """
+        Retrieves metadata about a KMS key using its alias.
+
+        Args:
+            alias_name (str): The alias name of the KMS key.
+
+        Returns:
+            dict: A dictionary containing metadata about the specified KMS key.
+        """
+        # The kms client can take an alias name as the KeyId
+        return self.kms_client.describe_key(KeyId=alias_name)