dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.1__py3-none-any.whl

dbt_platform_helper/providers/config.py

@@ -0,0 +1,100 @@
+from copy import deepcopy
+from pathlib import Path
+
+import click
+from schema import SchemaError
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
+from dbt_platform_helper.providers.yaml_file import FileNotFoundException
+from dbt_platform_helper.providers.yaml_file import FileProviderException
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+from dbt_platform_helper.utils.messages import abort_with_error
+
+
+class ConfigProvider:
+    def __init__(
+        self,
+        config_validator: ConfigValidator,
+        file_provider: YamlFileProvider = None,
+        echo=click.secho,
+    ):
+        self.config = {}
+        self.validator = config_validator
+        self.echo = echo
+        self.file_provider = file_provider or YamlFileProvider
+
+    # TODO refactor so that apply_environment_defaults isn't set, discarded and set again
+    def get_enriched_config(self):
+        return self.apply_environment_defaults(self.load_and_validate_platform_config())
+
+    def validate_platform_config(self):
+        PlatformConfigSchema.schema().validate(self.config)
+
+        # TODO= logically this isn't validation but loading + parsing, to move.
+        # also, we apply defaults but discard that data.  Should we just apply
+        # defaults to config returned by load_and_validate
+        enriched_config = ConfigProvider.apply_environment_defaults(self.config)
+        self.validator.run_validations(enriched_config)
+
+    def load_and_validate_platform_config(self, path=PLATFORM_CONFIG_FILE):
+        try:
+            self.config = self.file_provider.load(path)
+        except FileNotFoundException as e:
+            abort_with_error(
+                f"{e} Please check it exists and you are in the root directory of your deployment project."
+            )
+        except FileProviderException as e:
+            abort_with_error(f"Error loading configuration from {path}: {e}")
+
+        try:
+            self.validate_platform_config()
+        except SchemaError as e:
+            abort_with_error(f"Schema error in {path}. {e}")
+
+        return self.config
+
+    @staticmethod
+    # TODO this general function should be moved out of ConfigProvider
+    def config_file_check(path=PLATFORM_CONFIG_FILE):
+        if not Path(path).exists():
+            abort_with_error(
+                f"`{path}` is missing. "
+                "Please check it exists and you are in the root directory of your deployment project."
+            )
+
+    @staticmethod
+    def apply_environment_defaults(config):
+        if "environments" not in config:
+            return config
+
+        enriched_config = deepcopy(config)
+
+        environments = enriched_config["environments"]
+        env_defaults = environments.get("*", {})
+        without_defaults_entry = {
+            name: data if data else {} for name, data in environments.items() if name != "*"
+        }
+
+        default_versions = config.get("default_versions", {})
+
+        def combine_env_data(data):
+            return {
+                **env_defaults,
+                **data,
+                "versions": {
+                    **default_versions,
+                    **env_defaults.get("versions", {}),
+                    **data.get("versions", {}),
+                },
+            }
+
+        defaulted_envs = {
+            env_name: combine_env_data(env_data)
+            for env_name, env_data in without_defaults_entry.items()
+        }
+
+        enriched_config["environments"] = defaulted_envs
+
+        return enriched_config

dbt_platform_helper/providers/copilot.py

@@ -64,6 +64,7 @@ def create_addon_client_task(
         f"{execution_role}"
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--secrets CONNECTION_SECRET={_get_secrets_provider(application, env).get_connection_secret_arn(secret_name)} "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,
@@ -98,6 +99,7 @@ def create_postgres_admin_task(
         f"--task-group-name {task_name} "
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--env-vars CONNECTION_SECRET='{connection_string}' "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,

dbt_platform_helper/providers/files.py

@@ -0,0 +1,26 @@
+from os import makedirs
+from pathlib import Path
+
+
+class FileProvider:
+
+    def load(path: str) -> str:
+        pass
+
+    @staticmethod
+    def mkfile(base_path: str, file_path: str, contents, overwrite=False) -> str:
+        file_path = Path(file_path)
+        file = Path(base_path).joinpath(file_path)
+        file_exists = file.exists()
+
+        if not file_path.parent.exists():
+            makedirs(file_path.parent)
+
+        if file_exists and not overwrite:
+            return f"File {file_path} exists; doing nothing"
+
+        action = "overwritten" if file_exists and overwrite else "created"
+
+        file.write_text(contents)
+
+        return f"File {file_path} {action}"

dbt_platform_helper/providers/opensearch.py

@@ -0,0 +1,36 @@
+from dbt_platform_helper.providers.cache import CacheProvider
+
+
+class OpensearchProvider:
+
+    def __init__(self, opensearch_client):
+        self.opensearch_client = opensearch_client
+
+    def get_supported_opensearch_versions(self) -> list[str]:
+
+        cache_provider = self.__get_cache_provider()
+
+        if cache_provider.cache_refresh_required("opensearch"):
+
+            response = self.opensearch_client.list_versions()
+            all_versions = response["Versions"]
+
+            opensearch_versions = [
+                version for version in all_versions if not version.startswith("Elasticsearch_")
+            ]
+            supported_versions = [
+                version.removeprefix("OpenSearch_") for version in opensearch_versions
+            ]
+
+            cache_provider.update_cache("opensearch", supported_versions)
+
+            return supported_versions
+
+        else:
+            return cache_provider.read_supported_versions_from_cache("opensearch")
+
+    # TODO - cache provider instantiated here rather than via dependancy injection since it will likely only be used in the get_supported_opensearch_versions method.
+    # If another method is added which needs a CacheProvider, it should be injected into the constructor instead.
+    @staticmethod
+    def __get_cache_provider():
+        return CacheProvider()