dbt-platform-helper 13.1.2__py3-none-any.whl → 13.3.0__py3-none-any.whl

dbt_platform_helper/domain/versioning.py

@@ -1,11 +1,33 @@
+import os
+
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
-from dbt_platform_helper.providers.platform_helper_versioning import (
-    PlatformHelperVersioning,
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMajorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMinorVersionException,
 )
-from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
 from dbt_platform_helper.providers.semantic_version import SemanticVersion
-from dbt_platform_helper.utils.files import running_as_installed_package
+from dbt_platform_helper.providers.version import AWSCLIInstalledVersionProvider
+from dbt_platform_helper.providers.version import CopilotInstalledVersionProvider
+from dbt_platform_helper.providers.version import DeprecatedVersionFileVersionProvider
+from dbt_platform_helper.providers.version import GithubLatestVersionProvider
+from dbt_platform_helper.providers.version import InstalledVersionProvider
+from dbt_platform_helper.providers.version import PyPiLatestVersionProvider
+from dbt_platform_helper.providers.version import VersionProvider
+from dbt_platform_helper.providers.version_status import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.version_status import VersionStatus
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+def running_as_installed_package():
+    return "site-packages" in __file__
+
+
+def skip_version_checks():
+    return not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ
 
 
 class PlatformHelperVersionNotFoundException(PlatformException):
@@ -13,14 +35,113 @@ class PlatformHelperVersionNotFoundException(PlatformException):
         super().__init__(f"""Platform helper version could not be resolved.""")
 
 
-class RequiredVersion:
-    def __init__(self, io=None, platform_helper_versioning=None):
-        self.io = io or ClickIOProvider()
-        self.platform_helper_versioning = platform_helper_versioning or PlatformHelperVersioning(
-            io=self.io
+class PlatformHelperVersioning:
+    def __init__(
+        self,
+        io: ClickIOProvider = ClickIOProvider(),
+        version_file_version_provider: DeprecatedVersionFileVersionProvider = DeprecatedVersionFileVersionProvider(
+            YamlFileProvider
+        ),
+        config_provider: ConfigProvider = ConfigProvider(),
+        latest_version_provider: VersionProvider = PyPiLatestVersionProvider,
+        installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
+        skip_versioning_checks: bool = None,
+    ):
+        self.io = io
+        self.version_file_version_provider = version_file_version_provider
+        self.config_provider = config_provider
+        self.latest_version_provider = latest_version_provider
+        self.installed_version_provider = installed_version_provider
+        self.skip_versioning_checks = (
+            skip_versioning_checks if skip_versioning_checks is not None else skip_version_checks()
+        )
+
+    def get_required_version(self, pipeline=None):
+        version_status = self._get_version_status()
+        self.io.process_messages(version_status.validate())
+        required_version = self._resolve_required_version(pipeline, version_status)
+        self.io.info(required_version)
+        return required_version
+
+    # Used in the generate command
+    def check_platform_helper_version_mismatch(self):
+        if self.skip_versioning_checks:
+            return
+
+        version_status = self._get_version_status()
+        self.io.process_messages(version_status.validate())
+
+        required_version = SemanticVersion.from_string(
+            self._resolve_required_version(version_status=version_status)
+        )
+
+        if not version_status.installed == required_version:
+            message = (
+                f"WARNING: You are running platform-helper v{version_status.installed} against "
+                f"v{required_version} specified for the project."
+            )
+            self.io.warn(message)
+
+    def check_if_needs_update(self):
+        if self.skip_versioning_checks:
+            return
+
+        version_status = self._get_version_status(include_project_versions=False)
+
+        message = (
+            f"You are running platform-helper v{version_status.installed}, upgrade to "
+            f"v{version_status.latest} by running run `pip install "
+            "--upgrade dbt-platform-helper`."
+        )
+
+        try:
+            version_status.installed.validate_compatibility_with(version_status.latest)
+        except IncompatibleMajorVersionException:
+            self.io.error(message)
+        except IncompatibleMinorVersionException:
+            self.io.warn(message)
+
+    def _get_version_status(
+        self,
+        include_project_versions: bool = True,
+    ) -> PlatformHelperVersionStatus:
+        locally_installed_version = self.installed_version_provider.get_semantic_version(
+            "dbt-platform-helper"
+        )
+
+        latest_release = self.latest_version_provider.get_semantic_version("dbt-platform-helper")
+
+        if not include_project_versions:
+            return PlatformHelperVersionStatus(
+                installed=locally_installed_version,
+                latest=latest_release,
+            )
+
+        platform_config_default, pipeline_overrides = None, {}
+
+        platform_config = self.config_provider.load_unvalidated_config_file()
+
+        if platform_config:
+            platform_config_default = SemanticVersion.from_string(
+                platform_config.get("default_versions", {}).get("platform-helper")
+            )
+
+            pipeline_overrides = {
+                name: pipeline.get("versions", {}).get("platform-helper")
+                for name, pipeline in platform_config.get("environment_pipelines", {}).items()
+                if pipeline.get("versions", {}).get("platform-helper")
+            }
+        out = PlatformHelperVersionStatus(
+            installed=locally_installed_version,
+            latest=latest_release,
+            deprecated_version_file=self.version_file_version_provider.get_semantic_version(),
+            platform_config_default=platform_config_default,
+            pipeline_overrides=pipeline_overrides,
         )
 
-    def get_required_platform_helper_version(
+        return out
+
+    def _resolve_required_version(
         self, pipeline: str = None, version_status: PlatformHelperVersionStatus = None
     ) -> str:
         pipeline_version = version_status.pipeline_overrides.get(pipeline)
@@ -40,28 +161,38 @@ class RequiredVersion:
 
         return out
 
-    def get_required_version(self, pipeline=None):
-        version_status = self.platform_helper_versioning.get_status()
-        self.io.process_messages(version_status.validate())
-        required_version = self.get_required_platform_helper_version(pipeline, version_status)
-        self.io.info(required_version)
-        return required_version
 
-    # Used in the generate command
-    def check_platform_helper_version_mismatch(self):
-        if not running_as_installed_package():
-            return
+class AWSVersioning:
+    def __init__(
+        self,
+        latest_version_provider: VersionProvider = None,
+        installed_version_provider: VersionProvider = None,
+    ):
+        self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
+        self.installed_version_provider = (
+            installed_version_provider or AWSCLIInstalledVersionProvider
+        )
 
-        version_status = self.platform_helper_versioning.get_status()
-        self.io.process_messages(version_status.validate())
+    def get_version_status(self) -> VersionStatus:
+        return VersionStatus(
+            self.installed_version_provider.get_semantic_version(),
+            self.latest_version_provider.get_semantic_version("aws/aws-cli", True),
+        )
 
-        required_version = SemanticVersion.from_string(
-            self.get_required_platform_helper_version(version_status=version_status)
+
+class CopilotVersioning:
+    def __init__(
+        self,
+        latest_version_provider: VersionProvider = None,
+        installed_version_provider: VersionProvider = None,
+    ):
+        self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
+        self.installed_version_provider = (
+            installed_version_provider or CopilotInstalledVersionProvider
         )
 
-        if not version_status.local == required_version:
-            message = (
-                f"WARNING: You are running platform-helper v{version_status.local} against "
-                f"v{required_version} specified for the project."
-            )
-            self.io.warn(message)
+    def get_version_status(self) -> VersionStatus:
+        return VersionStatus(
+            self.installed_version_provider.get_semantic_version(),
+            self.latest_version_provider.get_semantic_version("aws/copilot-cli"),
+        )

dbt_platform_helper/providers/{aws.py → aws/exceptions.py}

@@ -35,3 +35,13 @@ class CopilotCodebaseNotFoundException(PlatformException):
         super().__init__(
             f"""The codebase "{codebase}" either does not exist or has not been deployed."""
         )
+
+
+class CreateAccessTokenException(AWSException):
+    def __init__(self, client_id: str):
+        super().__init__(f"""Failed to create access token for Client "{client_id}".""")
+
+
+class UnableToRetrieveSSOAccountList(AWSException):
+    def __init__(self):
+        super().__init__("Unable to retrieve AWS SSO account list")

dbt_platform_helper/providers/aws/interfaces.py

@@ -0,0 +1,13 @@
+from typing import Protocol
+
+
+class GetVersionsProtocol(Protocol):
+    def get_supported_versions(self) -> list[str]: ...
+
+
+class GetReferenceProtocol(Protocol):
+    def get_reference(self) -> str: ...
+
+
+class AwsGetVersionProtocol(GetReferenceProtocol, GetVersionsProtocol):
+    pass

dbt_platform_helper/providers/aws/opensearch.py

@@ -0,0 +1,23 @@
+import boto3
+
+
+class Opensearch:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+        self.engine = "OpenSearch"
+
+    def get_reference(self) -> str:
+        return self.engine.lower()
+
+    def get_supported_versions(self) -> list[str]:
+        response = self.client.list_versions()
+        all_versions = response["Versions"]
+
+        supported_versions = [
+            version.removeprefix(f"{self.engine}_")
+            for version in all_versions
+            if version.startswith(f"{self.engine}_")
+        ]
+
+        return supported_versions

dbt_platform_helper/providers/aws/redis.py

@@ -0,0 +1,21 @@
+import boto3
+
+
+class Redis:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+        self.engine = "redis"
+
+    def get_reference(self) -> str:
+        return self.engine.lower()
+
+    def get_supported_versions(self) -> list[str]:
+        supported_versions_response = self.client.describe_cache_engine_versions(Engine=self.engine)
+
+        supported_versions = [
+            version["EngineVersion"]
+            for version in supported_versions_response["CacheEngineVersions"]
+        ]
+
+        return supported_versions

dbt_platform_helper/providers/aws/sso_auth.py

@@ -0,0 +1,61 @@
+import botocore
+from boto3 import Session
+
+from dbt_platform_helper.providers.aws.exceptions import CreateAccessTokenException
+from dbt_platform_helper.providers.aws.exceptions import UnableToRetrieveSSOAccountList
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+
+
+class SSOAuthProvider:
+    def __init__(self, session: Session = None):
+        self.session = session
+        self.sso_oidc = self._get_client("sso-oidc")
+        self.sso = self._get_client("sso")
+
+    def register(self, client_name, client_type):
+        client = self.sso_oidc.register_client(clientName=client_name, clientType=client_type)
+        client_id = client.get("clientId")
+        client_secret = client.get("clientSecret")
+
+        return client_id, client_secret
+
+    def start_device_authorization(self, client_id, client_secret, start_url):
+        authz = self.sso_oidc.start_device_authorization(
+            clientId=client_id,
+            clientSecret=client_secret,
+            startUrl=start_url,
+        )
+        url = authz.get("verificationUriComplete")
+        deviceCode = authz.get("deviceCode")
+
+        return url, deviceCode
+
+    def create_access_token(self, client_id, client_secret, device_code):
+        try:
+            response = self.sso_oidc.create_token(
+                clientId=client_id,
+                clientSecret=client_secret,
+                grantType="urn:ietf:params:oauth:grant-type:device_code",
+                deviceCode=device_code,
+            )
+
+            return response.get("accessToken")
+
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] != "AuthorizationPendingException":
+                raise CreateAccessTokenException(client_id)
+
+    def list_accounts(self, access_token, max_results=100):
+        aws_accounts_response = self.sso.list_accounts(
+            accessToken=access_token,
+            maxResults=max_results,
+        )
+
+        if len(aws_accounts_response.get("accountList", [])) == 0:
+            raise UnableToRetrieveSSOAccountList()
+        return aws_accounts_response.get("accountList")
+
+    def _get_client(self, client: str):
+        if not self.session:
+            self.session = get_aws_session_or_abort()
+        return self.session.client(client)

dbt_platform_helper/providers/cache.py

@@ -1,10 +1,34 @@
 import os
+from abc import ABC
+from abc import abstractmethod
 from datetime import datetime
 
+from dbt_platform_helper.providers.aws.interfaces import AwsGetVersionProtocol
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
-class CacheProvider:
+class GetDataStrategy(ABC):
+    @abstractmethod
+    def retrieve_fresh_data(self):
+        pass
+
+    @abstractmethod
+    def get_data_identifier(self):
+        pass
+
+
+class GetAWSVersionStrategy(GetDataStrategy):
+    def __init__(self, client_provider: AwsGetVersionProtocol):
+        self.client_provider = client_provider
+
+    def retrieve_fresh_data(self):
+        return self.client_provider.get_supported_versions()
+
+    def get_data_identifier(self):
+        return self.client_provider.get_reference()
+
+
+class Cache:
     def __init__(
         self,
         file_provider: YamlFileProvider = None,
@@ -12,13 +36,25 @@ class CacheProvider:
         self._cache_file = ".platform-helper-config-cache.yml"
         self.file_provider = file_provider or YamlFileProvider
 
-    def read_supported_versions_from_cache(self, resource_name):
+    def get_data(self, strategy: GetDataStrategy):
+        """Main method to retrieve caching data using the client-specific
+        strategy."""
+        cache_key = strategy.get_data_identifier()
+        if self._cache_refresh_required(cache_key):
+            data = strategy.retrieve_fresh_data()
+            self._update_cache(cache_key, data)
+        else:
+            data = self._read_from_cache(cache_key)
+
+        return data
+
+    def _read_from_cache(self, resource_name):
 
         platform_helper_config = self.file_provider.load(self._cache_file)
 
         return platform_helper_config.get(resource_name).get("versions")
 
-    def update_cache(self, resource_name, supported_versions):
+    def _update_cache(self, resource_name, supported_versions):
 
         platform_helper_config = {}
 
@@ -40,7 +76,7 @@ class CacheProvider:
             "# [!] This file is autogenerated via the platform-helper. Do not edit.\n",
         )
 
-    def cache_refresh_required(self, resource_name) -> bool:
+    def _cache_refresh_required(self, resource_name) -> bool:
         """
         Checks if the platform-helper should reach out to AWS to 'refresh' its
         cached values.

dbt_platform_helper/providers/config.py

@@ -64,7 +64,8 @@ class ConfigProvider:
         except FileProviderException:
             return {}
 
-    # TODO this general function should be moved out of ConfigProvider
+    # TODO remove function and push logic to where this is called.
+    # removed usage from config domain, code is very generic and doesn't require the overhead of a function
     def config_file_check(self, path=PLATFORM_CONFIG_FILE):
         if not Path(path).exists():
             self.io.abort_with_error(

dbt_platform_helper/providers/config_validator.py

@@ -3,9 +3,11 @@ from typing import Callable
 import boto3
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.aws.opensearch import Opensearch
+from dbt_platform_helper.providers.aws.redis import Redis
+from dbt_platform_helper.providers.cache import Cache
+from dbt_platform_helper.providers.cache import GetAWSVersionStrategy
 from dbt_platform_helper.providers.io import ClickIOProvider
-from dbt_platform_helper.providers.opensearch import OpensearchProvider
-from dbt_platform_helper.providers.redis import RedisProvider
 
 
 class ConfigValidatorError(PlatformException):
@@ -15,7 +17,10 @@ class ConfigValidatorError(PlatformException):
 class ConfigValidator:
 
     def __init__(
-        self, validations: Callable[[dict], None] = None, io: ClickIOProvider = ClickIOProvider()
+        self,
+        validations: Callable[[dict], None] = None,
+        io: ClickIOProvider = ClickIOProvider(),
+        session: boto3.Session = None,
     ):
         self.validations = validations or [
             self.validate_supported_redis_versions,
@@ -26,13 +31,14 @@ class ConfigValidator:
             self.validate_database_migration_input_sources,
         ]
         self.io = io
+        self.session = session
 
     def run_validations(self, config: dict):
         for validation in self.validations:
             validation(config)
 
     def _validate_extension_supported_versions(
-        self, config, extension_type, version_key, get_supported_versions
+        self, config, aws_provider, extension_type, version_key
     ):
         extensions = config.get("extensions", {})
         if not extensions:
@@ -44,7 +50,10 @@ class ConfigValidator:
             if extension.get("type") == extension_type
         ]
 
-        supported_extension_versions = get_supported_versions()
+        # In this format so it can be monkey patched initially via mock_get_data fixture
+        cache_provider = Cache()
+        get_data_strategy = GetAWSVersionStrategy(aws_provider)
+        supported_extension_versions = cache_provider.get_data(get_data_strategy)
         extensions_with_invalid_version = []
 
         for extension in extensions_for_type:
@@ -71,24 +80,25 @@ class ConfigValidator:
                 f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
             )
 
+    def _get_client(self, service_name: str):
+        if self.session:
+            return self.session.client(service_name)
+        return boto3.client(service_name)
+
     def validate_supported_redis_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            extension_type="redis",
-            version_key="engine",
-            get_supported_versions=RedisProvider(
-                boto3.client("elasticache")
-            ).get_supported_redis_versions,
+            aws_provider=Redis(self._get_client("elasticache")),
+            extension_type="redis",  # TODO this is information which can live in the RedisProvider
+            version_key="engine",  # TODO this is information which can live in the RedisProvider
         )
 
     def validate_supported_opensearch_versions(self, config):
         return self._validate_extension_supported_versions(
             config=config,
-            extension_type="opensearch",
-            version_key="engine",
-            get_supported_versions=OpensearchProvider(
-                boto3.client("opensearch")
-            ).get_supported_opensearch_versions,
+            aws_provider=Opensearch(self._get_client("opensearch")),
+            extension_type="opensearch",  # TODO this is information which can live in the OpensearchProvider
+            version_key="engine",  # TODO this is information which can live in the OpensearchProvider
         )
 
     def validate_environment_pipelines(self, config):
@@ -205,21 +215,14 @@ class ConfigValidator:
                         f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
                     )
 
+                # TODO - The from_account and to_account properties are deprecated and will be removed when terraform-platform-modules is merged with platform-tools
                 if from_account != to_account:
-                    if "from_account" not in section:
-                        errors.append(
-                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'from_account' parameter must be present."
-                        )
-                    elif section["from_account"] != from_account:
+                    if "from_account" in section and section["from_account"] != from_account:
                         errors.append(
                             f"Incorrect value for 'from_account' for environment '{from_env}'"
                         )
 
-                    if "to_account" not in section:
-                        errors.append(
-                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'to_account' parameter must be present."
-                        )
-                    elif section["to_account"] != to_account:
+                    if "to_account" in section and section["to_account"] != to_account:
                         errors.append(
                             f"Incorrect value for 'to_account' for environment '{to_env}'"
                         )

dbt_platform_helper/providers/copilot.py

@@ -4,7 +4,7 @@ import time
 from botocore.exceptions import ClientError
 
 from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
-from dbt_platform_helper.providers.aws import CreateTaskTimeoutException
+from dbt_platform_helper.providers.aws.exceptions import CreateTaskTimeoutException
 from dbt_platform_helper.providers.secrets import Secrets
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.messages import abort_with_error

dbt_platform_helper/providers/io.py

@@ -7,11 +7,14 @@ class ClickIOProvider:
     def warn(self, message: str):
         click.secho(message, fg="magenta")
 
+    def debug(self, message: str):
+        click.secho(message, fg="green")
+
     def error(self, message: str):
         click.secho(f"Error: {message}", fg="red")
 
-    def info(self, message: str):
-        click.secho(message)
+    def info(self, message: str, **kwargs):
+        click.secho(message, **kwargs)
 
     def input(self, message: str) -> str:
         return click.prompt(message)

dbt_platform_helper/providers/kms.py

@@ -0,0 +1,22 @@
+import boto3
+
+
+class KMSProvider:
+    """A provider class for interacting with the AWS KMS (Key Management
+    Service)."""
+
+    def __init__(self, kms_client: boto3.client):
+        self.kms_client = kms_client
+
+    def describe_key(self, alias_name: str) -> dict:
+        """
+        Retrieves metadata about a KMS key using its alias.
+
+        Args:
+            alias_name (str): The alias name of the KMS key.
+
+        Returns:
+            dict: A dictionary containing metadata about the specified KMS key.
+        """
+        # The kms client can take an alias name as the KeyId
+        return self.kms_client.describe_key(KeyId=alias_name)