dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.0__py3-none-any.whl

dbt_platform_helper/providers/redis.py

@@ -0,0 +1,34 @@
+from dbt_platform_helper.providers.cache import CacheProvider
+
+
+class RedisProvider:
+    def __init__(self, elasticache_client):
+        self.elasticache_client = elasticache_client
+
+    def get_supported_redis_versions(self):
+
+        cache_provider = self.__get_cache_provider()
+
+        if cache_provider.cache_refresh_required("redis"):
+
+            supported_versions_response = self.elasticache_client.describe_cache_engine_versions(
+                Engine="redis"
+            )
+
+            supported_versions = [
+                version["EngineVersion"]
+                for version in supported_versions_response["CacheEngineVersions"]
+            ]
+
+            cache_provider.update_cache("redis", supported_versions)
+
+            return supported_versions
+
+        else:
+            return cache_provider.read_supported_versions_from_cache("redis")
+
+    # TODO - cache provider instantiated here rather than via dependancy injection since it will likely only be used in the get_supported_redis_versions method.
+    # If another method is added which needs a CacheProvider, it should be injected into the constructor instead.
+    @staticmethod
+    def __get_cache_provider():
+        return CacheProvider()

dbt_platform_helper/providers/yaml_file.py

@@ -0,0 +1,83 @@
+from abc import ABC
+from abc import abstractmethod
+from pathlib import Path
+
+import yaml
+from yaml.parser import ParserError
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+
+
+class FileProviderException(Exception):
+    pass
+
+
+class YamlFileProviderException(FileProviderException):
+    pass
+
+
+class FileNotFoundException(YamlFileProviderException):
+    pass
+
+
+class InvalidYamlException(YamlFileProviderException):
+    pass
+
+
+class DuplicateKeysException(YamlFileProviderException):
+    pass
+
+
+class FileProvider(ABC):
+    @abstractmethod
+    def load(path: str) -> dict:
+        raise NotImplementedError("Implement this in the subclass")
+
+
+class YamlFileProvider(FileProvider):
+    def load(path: str) -> dict:
+        """
+        Raises:
+            FileNotFoundException: file is not there
+            InvalidYamlException: file contains invalid yaml
+            DuplicateKeysException: yaml contains duplicate keys
+        """
+        if not Path(path).exists():
+            # TODO this error message is domain specific and should not mention deployment directory project here
+            raise FileNotFoundException(
+                f"`{path}` is missing. Please check it exists and you are in the root directory of your deployment project."
+            )
+        try:
+            yaml_content = yaml.safe_load(Path(path).read_text())
+        except ParserError:
+            raise InvalidYamlException(f"{path} is not valid YAML.")
+
+        if not yaml_content:
+            return {}
+
+        YamlFileProvider.lint_yaml_for_duplicate_keys(path)
+
+        return yaml_content
+
+    def write(path: str, contents: dict, comment: str = ""):
+        with open(path, "w") as file:
+            file.write(comment)
+            yaml.dump(contents, file)
+
+    @staticmethod
+    def lint_yaml_for_duplicate_keys(path):
+        duplicate_keys = []
+        with open(path, "r") as yaml_file:
+            file_contents = yaml_file.read()
+            results = linter.run(
+                file_contents, YamlLintConfig(yaml.dump({"rules": {"key-duplicates": "enable"}}))
+            )
+            duplicate_keys = [
+                "\t"
+                + f"Line {result.line}: {result.message}".replace(
+                    " in mapping (key-duplicates)", ""
+                )
+                for result in results
+            ]
+        if duplicate_keys:
+            raise DuplicateKeysException(",".join(duplicate_keys))

dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml

@@ -0,0 +1,67 @@
+# {% extra_header %}
+# {% version_info %}
+
+Metadata:
+  cfn-lint:
+    config:
+      ignore_checks:
+        - W2001 # Parameter not used
+
+Parameters:
+  # Copilot required Parameters...
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: The name of the service, job, or workflow being deployed.
+
+Resources: {% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Metadata:
+      'aws:copilot:description': 'An IAM ManagedPolicy for your service to access the bucket'
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: Grants Read access to the S3 bucket.
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: 'KMSDecryptAndGenerate'
+            Effect: Allow
+            Action:
+              - kms:Decrypt
+              - kms:GenerateDataKey
+            Resource: 'arn:aws:kms:eu-west-2:{{ resource.bucket_account }}:key/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ObjectActions'
+            Effect: Allow
+            Action:
+              {% if resource.read %}- s3:Get*{% endif %}
+              {% if resource.write %}- s3:Put*{% endif %}
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ListAction'
+            Effect: Allow
+            Action:
+              - s3:ListBucket
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+    {% endfor %}
+Outputs:{% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Description: "The IAM::ManagedPolicy to attach to the task role"
+    Value:
+      Ref: {{ resource.app_prefix }}XEnvAccessPolicy
+ {% endfor %}

dbt_platform_helper/utils/aws.py

@@ -13,12 +13,12 @@ import click
 import yaml
 from boto3 import Session
 
+from dbt_platform_helper.constants import REFRESH_TOKEN_MESSAGE
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.aws import AWSException
 from dbt_platform_helper.providers.aws import CopilotCodebaseNotFoundException
 from dbt_platform_helper.providers.aws import ImageNotFoundException
 from dbt_platform_helper.providers.aws import LogGroupNotFoundException
-from dbt_platform_helper.providers.cache import CacheProvider
 from dbt_platform_helper.providers.validation import ValidationException
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
@@ -27,9 +27,6 @@ AWS_SESSION_CACHE = {}
 
 
 def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
-    REFRESH_TOKEN_MESSAGE = (
-        "To refresh this SSO session run `aws sso login` with the corresponding profile"
-    )
     aws_profile = aws_profile or os.getenv("AWS_PROFILE")
     if aws_profile in AWS_SESSION_CACHE:
         return AWS_SESSION_CACHE[aws_profile]
@@ -358,59 +355,6 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
     return parameter_data
 
 
-def get_supported_redis_versions():
-
-    cache_provider = CacheProvider()
-
-    if cache_provider.cache_refresh_required("redis"):
-
-        session = get_aws_session_or_abort()
-        elasticache_client = session.client("elasticache")
-
-        supported_versions_response = elasticache_client.describe_cache_engine_versions(
-            Engine="redis"
-        )
-
-        supported_versions = [
-            version["EngineVersion"]
-            for version in supported_versions_response["CacheEngineVersions"]
-        ]
-
-        cache_provider.update_cache("redis", supported_versions)
-
-        return supported_versions
-
-    else:
-        return cache_provider.read_supported_versions_from_cache("redis")
-
-
-def get_supported_opensearch_versions():
-
-    cache_provider = CacheProvider()
-
-    if cache_provider.cache_refresh_required("opensearch"):
-
-        session = get_aws_session_or_abort()
-        opensearch_client = session.client("opensearch")
-
-        response = opensearch_client.list_versions()
-        all_versions = response["Versions"]
-
-        opensearch_versions = [
-            version for version in all_versions if not version.startswith("Elasticsearch_")
-        ]
-        supported_versions = [
-            version.removeprefix("OpenSearch_") for version in opensearch_versions
-        ]
-
-        cache_provider.update_cache("opensearch", supported_versions)
-
-        return supported_versions
-
-    else:
-        return cache_provider.read_supported_versions_from_cache("opensearch")
-
-
 def get_connection_string(
     session: Session,
     app: str,

dbt_platform_helper/utils/files.py

@@ -1,4 +1,3 @@
-from copy import deepcopy
 from os import makedirs
 from pathlib import Path
 
@@ -67,38 +66,3 @@ def generate_override_files_from_template(base_path, overrides_path, output_dir,
 
     generate_files_for_dir("*")
     generate_files_for_dir("bin/*")
-
-
-def apply_environment_defaults(config):
-    if "environments" not in config:
-        return config
-
-    enriched_config = deepcopy(config)
-
-    environments = enriched_config["environments"]
-    env_defaults = environments.get("*", {})
-    without_defaults_entry = {
-        name: data if data else {} for name, data in environments.items() if name != "*"
-    }
-
-    default_versions = config.get("default_versions", {})
-
-    def combine_env_data(data):
-        return {
-            **env_defaults,
-            **data,
-            "versions": {
-                **default_versions,
-                **env_defaults.get("versions", {}),
-                **data.get("versions", {}),
-            },
-        }
-
-    defaulted_envs = {
-        env_name: combine_env_data(env_data)
-        for env_name, env_data in without_defaults_entry.items()
-    }
-
-    enriched_config["environments"] = defaulted_envs
-
-    return enriched_config

dbt_platform_helper/utils/template.py

@@ -5,6 +5,16 @@ import jinja2
 from dbt_platform_helper.jinja2_tags import ExtraHeaderTag
 from dbt_platform_helper.jinja2_tags import VersionTag
 
+S3_CROSS_ACCOUNT_POLICY = "addons/svc/s3-cross-account-policy.yml"
+
+ADDON_TEMPLATE_MAP = {
+    "s3": ["addons/svc/s3-policy.yml"],
+    "s3-policy": ["addons/svc/s3-policy.yml"],
+    "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
+    "subscription-filter": ["addons/svc/subscription-filter.yml"],
+    "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
+}
+
 
 def camel_case(s):
     s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "")

dbt_platform_helper/utils/validation.py

@@ -1,24 +1,7 @@
-import os
-import re
-from pathlib import Path
-
-import click
-import yaml
 from schema import SchemaError
-from yaml.parser import ParserError
-from yamllint import config
-from yamllint import linter
 
-from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
-from dbt_platform_helper.constants import ENVIRONMENTS_KEY
-from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
-from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
-from dbt_platform_helper.providers.platform_config_schema import EXTENSION_SCHEMAS
-from dbt_platform_helper.providers.platform_config_schema import PLATFORM_CONFIG_SCHEMA
-from dbt_platform_helper.utils.aws import get_supported_opensearch_versions
-from dbt_platform_helper.utils.aws import get_supported_redis_versions
-from dbt_platform_helper.utils.files import apply_environment_defaults
-from dbt_platform_helper.utils.messages import abort_with_error
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
 
 
 def validate_addons(addons: dict):
@@ -33,7 +16,7 @@ def validate_addons(addons: dict):
             if not addon_type:
                 errors[addon_name] = f"Missing addon type in addon '{addon_name}'"
                 continue
-            schema = EXTENSION_SCHEMAS.get(addon_type, None)
+            schema = PlatformConfigSchema.extension_schemas().get(addon_type, None)
             if not schema:
                 errors[addon_name] = (
                     f"Unsupported addon type '{addon_type}' in addon '{addon_name}'"
@@ -43,312 +26,7 @@ def validate_addons(addons: dict):
         except SchemaError as ex:
             errors[addon_name] = f"Error in {addon_name}: {ex.code}"
 
-    _validate_extension_supported_versions(
-        config={"extensions": addons},
-        extension_type="redis",
-        version_key="engine",
-        get_supported_versions=get_supported_redis_versions,
-    )
-    _validate_extension_supported_versions(
-        config={"extensions": addons},
-        extension_type="opensearch",
-        version_key="engine",
-        get_supported_versions=get_supported_opensearch_versions,
-    )
+    ConfigValidator().validate_supported_redis_versions({"extensions": addons})
+    ConfigValidator().validate_supported_opensearch_versions({"extensions": addons})
 
     return errors
-
-
-def float_between_with_halfstep(lower, upper):
-    def is_between(value):
-        is_number = isinstance(value, int) or isinstance(value, float)
-        is_half_step = re.match(r"^\d+(\.[05])?$", str(value))
-
-        if is_number and is_half_step and lower <= value <= upper:
-            return True
-        raise SchemaError(f"should be a number between {lower} and {upper} in increments of 0.5")
-
-    return is_between
-
-
-def validate_platform_config(config):
-    PLATFORM_CONFIG_SCHEMA.validate(config)
-    enriched_config = apply_environment_defaults(config)
-    _validate_environment_pipelines(enriched_config)
-    _validate_environment_pipelines_triggers(enriched_config)
-    _validate_codebase_pipelines(enriched_config)
-    validate_database_copy_section(enriched_config)
-
-    _validate_extension_supported_versions(
-        config=config,
-        extension_type="redis",
-        version_key="engine",
-        get_supported_versions=get_supported_redis_versions,
-    )
-    _validate_extension_supported_versions(
-        config=config,
-        extension_type="opensearch",
-        version_key="engine",
-        get_supported_versions=get_supported_opensearch_versions,
-    )
-
-
-def _validate_extension_supported_versions(
-    config, extension_type, version_key, get_supported_versions
-):
-    extensions = config.get("extensions", {})
-    if not extensions:
-        return
-
-    extensions_for_type = [
-        extension
-        for extension in config.get("extensions", {}).values()
-        if extension.get("type") == extension_type
-    ]
-
-    supported_extension_versions = get_supported_versions()
-    extensions_with_invalid_version = []
-
-    for extension in extensions_for_type:
-
-        environments = extension.get("environments", {})
-
-        if not isinstance(environments, dict):
-            click.secho(
-                f"Error: {extension_type} extension definition is invalid type, expected dictionary",
-                fg="red",
-            )
-            continue
-        for environment, env_config in environments.items():
-
-            # An extension version doesn't need to be specified for all environments, provided one is specified under "*".
-            # So check if the version is set before checking if it's supported
-            extension_version = env_config.get(version_key)
-            if extension_version and extension_version not in supported_extension_versions:
-                extensions_with_invalid_version.append(
-                    {"environment": environment, "version": extension_version}
-                )
-
-    for version_failure in extensions_with_invalid_version:
-        click.secho(
-            f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
-            fg="red",
-        )
-
-
-def validate_database_copy_section(config):
-    extensions = config.get("extensions", {})
-    if not extensions:
-        return
-
-    postgres_extensions = {
-        key: ext for key, ext in extensions.items() if ext.get("type", None) == "postgres"
-    }
-
-    if not postgres_extensions:
-        return
-
-    errors = []
-
-    for extension_name, extension in postgres_extensions.items():
-        database_copy_sections = extension.get("database_copy", [])
-
-        if not database_copy_sections:
-            return
-
-        all_environments = [env for env in config.get("environments", {}).keys() if not env == "*"]
-        all_envs_string = ", ".join(all_environments)
-
-        for section in database_copy_sections:
-            from_env = section["from"]
-            to_env = section["to"]
-
-            from_account = _get_env_deploy_account_info(config, from_env, "id")
-            to_account = _get_env_deploy_account_info(config, to_env, "id")
-
-            if from_env == to_env:
-                errors.append(
-                    f"database_copy 'to' and 'from' cannot be the same environment in extension '{extension_name}'."
-                )
-
-            if "prod" in to_env:
-                errors.append(
-                    f"Copying to a prod environment is not supported: database_copy 'to' cannot be '{to_env}' in extension '{extension_name}'."
-                )
-
-            if from_env not in all_environments:
-                errors.append(
-                    f"database_copy 'from' parameter must be a valid environment ({all_envs_string}) but was '{from_env}' in extension '{extension_name}'."
-                )
-
-            if to_env not in all_environments:
-                errors.append(
-                    f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
-                )
-
-            if from_account != to_account:
-                if "from_account" not in section:
-                    errors.append(
-                        f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'from_account' parameter must be present."
-                    )
-                elif section["from_account"] != from_account:
-                    errors.append(
-                        f"Incorrect value for 'from_account' for environment '{from_env}'"
-                    )
-
-                if "to_account" not in section:
-                    errors.append(
-                        f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'to_account' parameter must be present."
-                    )
-                elif section["to_account"] != to_account:
-                    errors.append(f"Incorrect value for 'to_account' for environment '{to_env}'")
-
-    if errors:
-        abort_with_error("\n".join(errors))
-
-
-def _get_env_deploy_account_info(config, env, key):
-    return (
-        config.get("environments", {}).get(env, {}).get("accounts", {}).get("deploy", {}).get(key)
-    )
-
-
-def _validate_environment_pipelines(config):
-    bad_pipelines = {}
-    for pipeline_name, pipeline in config.get("environment_pipelines", {}).items():
-        bad_envs = []
-        pipeline_account = pipeline.get("account", None)
-        if pipeline_account:
-            for env in pipeline.get("environments", {}).keys():
-                env_account = _get_env_deploy_account_info(config, env, "name")
-                if not env_account == pipeline_account:
-                    bad_envs.append(env)
-        if bad_envs:
-            bad_pipelines[pipeline_name] = {"account": pipeline_account, "bad_envs": bad_envs}
-    if bad_pipelines:
-        message = "The following pipelines are misconfigured:"
-        for pipeline, detail in bad_pipelines.items():
-            envs = detail["bad_envs"]
-            acc = detail["account"]
-            message += f"  '{pipeline}' - these environments are not in the '{acc}' account: {', '.join(envs)}\n"
-        abort_with_error(message)
-
-
-def _validate_codebase_pipelines(config):
-    if CODEBASE_PIPELINES_KEY in config:
-        for codebase in config[CODEBASE_PIPELINES_KEY]:
-            codebase_environments = []
-
-            for pipeline in codebase["pipelines"]:
-                codebase_environments += [e["name"] for e in pipeline[ENVIRONMENTS_KEY]]
-
-            unique_codebase_environments = sorted(list(set(codebase_environments)))
-
-            if sorted(codebase_environments) != sorted(unique_codebase_environments):
-                abort_with_error(
-                    f"The {PLATFORM_CONFIG_FILE} file is invalid, each environment can only be "
-                    "listed in a single pipeline per codebase"
-                )
-
-
-def _validate_environment_pipelines_triggers(config):
-    errors = []
-    pipelines_with_triggers = {
-        pipeline_name: pipeline
-        for pipeline_name, pipeline in config.get("environment_pipelines", {}).items()
-        if "pipeline_to_trigger" in pipeline
-    }
-
-    for pipeline_name, pipeline in pipelines_with_triggers.items():
-        pipeline_to_trigger = pipeline["pipeline_to_trigger"]
-        if pipeline_to_trigger not in config.get("environment_pipelines", {}):
-            message = f"  '{pipeline_name}' - '{pipeline_to_trigger}' is not a valid target pipeline to trigger"
-
-            errors.append(message)
-            continue
-
-        if pipeline_to_trigger == pipeline_name:
-            message = f"  '{pipeline_name}' - pipelines cannot trigger themselves"
-            errors.append(message)
-
-    if errors:
-        error_message = "The following pipelines are misconfigured: \n"
-        abort_with_error(error_message + "\n  ".join(errors))
-
-
-def lint_yaml_for_duplicate_keys(file_path):
-    lint_yaml_config = """
-rules:
-  key-duplicates: enable
-"""
-    yaml_config = config.YamlLintConfig(lint_yaml_config)
-
-    with open(file_path, "r") as yaml_file:
-        file_contents = yaml_file.read()
-        results = linter.run(file_contents, yaml_config)
-
-    parsed_results = [
-        "\t" + f"Line {result.line}: {result.message}".replace(" in mapping (key-duplicates)", "")
-        for result in results
-    ]
-
-    return parsed_results
-
-
-def load_and_validate_platform_config(path=PLATFORM_CONFIG_FILE, disable_file_check=False):
-    if not disable_file_check:
-        config_file_check(path)
-    try:
-        conf = yaml.safe_load(Path(path).read_text())
-        duplicate_keys = lint_yaml_for_duplicate_keys(path)
-        if duplicate_keys:
-            abort_with_error(
-                "Duplicate keys found in platform-config:"
-                + os.linesep
-                + os.linesep.join(duplicate_keys)
-            )
-        validate_platform_config(conf)
-        return conf
-    except ParserError:
-        abort_with_error(f"{PLATFORM_CONFIG_FILE} is not valid YAML")
-    except SchemaError as e:
-        abort_with_error(f"Schema error in {PLATFORM_CONFIG_FILE}. {e}")
-
-
-def config_file_check(path=PLATFORM_CONFIG_FILE):
-    platform_config_exists = Path(path).exists()
-    errors = []
-    warnings = []
-
-    messages = {
-        "storage.yml": {"instruction": " under the key 'extensions'", "type": errors},
-        "extensions.yml": {"instruction": " under the key 'extensions'", "type": errors},
-        "pipelines.yml": {
-            "instruction": ", change the key 'codebases' to 'codebase_pipelines'",
-            "type": errors,
-        },
-        PLATFORM_HELPER_VERSION_FILE: {
-            "instruction": ", under the key `default_versions: platform-helper:`",
-            "type": warnings,
-        },
-    }
-
-    for file in messages.keys():
-        if Path(file).exists():
-            message = (
-                f"`{file}` is no longer supported. Please move its contents into the "
-                f"`{PLATFORM_CONFIG_FILE}` file{messages[file]['instruction']} and delete `{file}`."
-            )
-            messages[file]["type"].append(message)
-
-    if not errors and not warnings and not platform_config_exists:
-        errors.append(
-            f"`{PLATFORM_CONFIG_FILE}` is missing. "
-            "Please check it exists and you are in the root directory of your deployment project."
-        )
-
-    if warnings:
-        click.secho("\n".join(warnings), bg="yellow", fg="black")
-    if errors:
-        click.secho("\n".join(errors), bg="red", fg="white")
-        exit(1)

{dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 12.4.1
+Version: 12.5.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team