dbt-platform-helper 12.4.0__py3-none-any.whl → 12.5.0__py3-none-any.whl

dbt_platform_helper/providers/redis.py

@@ -0,0 +1,34 @@
+from dbt_platform_helper.providers.cache import CacheProvider
+
+
+class RedisProvider:
+    def __init__(self, elasticache_client):
+        self.elasticache_client = elasticache_client
+
+    def get_supported_redis_versions(self):
+
+        cache_provider = self.__get_cache_provider()
+
+        if cache_provider.cache_refresh_required("redis"):
+
+            supported_versions_response = self.elasticache_client.describe_cache_engine_versions(
+                Engine="redis"
+            )
+
+            supported_versions = [
+                version["EngineVersion"]
+                for version in supported_versions_response["CacheEngineVersions"]
+            ]
+
+            cache_provider.update_cache("redis", supported_versions)
+
+            return supported_versions
+
+        else:
+            return cache_provider.read_supported_versions_from_cache("redis")
+
+    # TODO - cache provider instantiated here rather than via dependancy injection since it will likely only be used in the get_supported_redis_versions method.
+    # If another method is added which needs a CacheProvider, it should be injected into the constructor instead.
+    @staticmethod
+    def __get_cache_provider():
+        return CacheProvider()

dbt_platform_helper/providers/yaml_file.py

@@ -0,0 +1,83 @@
+from abc import ABC
+from abc import abstractmethod
+from pathlib import Path
+
+import yaml
+from yaml.parser import ParserError
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+
+
+class FileProviderException(Exception):
+    pass
+
+
+class YamlFileProviderException(FileProviderException):
+    pass
+
+
+class FileNotFoundException(YamlFileProviderException):
+    pass
+
+
+class InvalidYamlException(YamlFileProviderException):
+    pass
+
+
+class DuplicateKeysException(YamlFileProviderException):
+    pass
+
+
+class FileProvider(ABC):
+    @abstractmethod
+    def load(path: str) -> dict:
+        raise NotImplementedError("Implement this in the subclass")
+
+
+class YamlFileProvider(FileProvider):
+    def load(path: str) -> dict:
+        """
+        Raises:
+            FileNotFoundException: file is not there
+            InvalidYamlException: file contains invalid yaml
+            DuplicateKeysException: yaml contains duplicate keys
+        """
+        if not Path(path).exists():
+            # TODO this error message is domain specific and should not mention deployment directory project here
+            raise FileNotFoundException(
+                f"`{path}` is missing. Please check it exists and you are in the root directory of your deployment project."
+            )
+        try:
+            yaml_content = yaml.safe_load(Path(path).read_text())
+        except ParserError:
+            raise InvalidYamlException(f"{path} is not valid YAML.")
+
+        if not yaml_content:
+            return {}
+
+        YamlFileProvider.lint_yaml_for_duplicate_keys(path)
+
+        return yaml_content
+
+    def write(path: str, contents: dict, comment: str = ""):
+        with open(path, "w") as file:
+            file.write(comment)
+            yaml.dump(contents, file)
+
+    @staticmethod
+    def lint_yaml_for_duplicate_keys(path):
+        duplicate_keys = []
+        with open(path, "r") as yaml_file:
+            file_contents = yaml_file.read()
+            results = linter.run(
+                file_contents, YamlLintConfig(yaml.dump({"rules": {"key-duplicates": "enable"}}))
+            )
+            duplicate_keys = [
+                "\t"
+                + f"Line {result.line}: {result.message}".replace(
+                    " in mapping (key-duplicates)", ""
+                )
+                for result in results
+            ]
+        if duplicate_keys:
+            raise DuplicateKeysException(",".join(duplicate_keys))

dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml

@@ -0,0 +1,67 @@
+# {% extra_header %}
+# {% version_info %}
+
+Metadata:
+  cfn-lint:
+    config:
+      ignore_checks:
+        - W2001 # Parameter not used
+
+Parameters:
+  # Copilot required Parameters...
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: The name of the service, job, or workflow being deployed.
+
+Resources: {% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Metadata:
+      'aws:copilot:description': 'An IAM ManagedPolicy for your service to access the bucket'
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: Grants Read access to the S3 bucket.
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: 'KMSDecryptAndGenerate'
+            Effect: Allow
+            Action:
+              - kms:Decrypt
+              - kms:GenerateDataKey
+            Resource: 'arn:aws:kms:eu-west-2:{{ resource.bucket_account }}:key/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ObjectActions'
+            Effect: Allow
+            Action:
+              {% if resource.read %}- s3:Get*{% endif %}
+              {% if resource.write %}- s3:Put*{% endif %}
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ListAction'
+            Effect: Allow
+            Action:
+              - s3:ListBucket
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+    {% endfor %}
+Outputs:{% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Description: "The IAM::ManagedPolicy to attach to the task role"
+    Value:
+      Ref: {{ resource.app_prefix }}XEnvAccessPolicy
+ {% endfor %}

dbt_platform_helper/utils/aws.py

@@ -13,15 +13,13 @@ import click
 import yaml
 from boto3 import Session
 
+from dbt_platform_helper.constants import REFRESH_TOKEN_MESSAGE
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.aws import AWSException
 from dbt_platform_helper.providers.aws import CopilotCodebaseNotFoundException
 from dbt_platform_helper.providers.aws import ImageNotFoundException
 from dbt_platform_helper.providers.aws import LogGroupNotFoundException
 from dbt_platform_helper.providers.validation import ValidationException
-from dbt_platform_helper.utils.files import cache_refresh_required
-from dbt_platform_helper.utils.files import read_supported_versions_from_cache
-from dbt_platform_helper.utils.files import write_to_cache
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
 SSM_PATH = "/copilot/{app}/{env}/secrets/{name}"
@@ -29,9 +27,6 @@ AWS_SESSION_CACHE = {}
 
 
 def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
-    REFRESH_TOKEN_MESSAGE = (
-        "To refresh this SSO session run `aws sso login` with the corresponding profile"
-    )
     aws_profile = aws_profile or os.getenv("AWS_PROFILE")
     if aws_profile in AWS_SESSION_CACHE:
         return AWS_SESSION_CACHE[aws_profile]
@@ -360,59 +355,6 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
     return parameter_data
 
 
-def get_supported_redis_versions():
-
-    if cache_refresh_required("redis"):
-
-        supported_versions = []
-
-        session = get_aws_session_or_abort()
-        elasticache_client = session.client("elasticache")
-
-        supported_versions_response = elasticache_client.describe_cache_engine_versions(
-            Engine="redis"
-        )
-
-        supported_versions = [
-            version["EngineVersion"]
-            for version in supported_versions_response["CacheEngineVersions"]
-        ]
-
-        write_to_cache("redis", supported_versions)
-
-        return supported_versions
-
-    else:
-        return read_supported_versions_from_cache("redis")
-
-
-def get_supported_opensearch_versions():
-
-    if cache_refresh_required("opensearch"):
-
-        supported_versions = []
-
-        session = get_aws_session_or_abort()
-        opensearch_client = session.client("opensearch")
-
-        response = opensearch_client.list_versions()
-        all_versions = response["Versions"]
-
-        opensearch_versions = [
-            version for version in all_versions if not version.startswith("Elasticsearch_")
-        ]
-        supported_versions = [
-            version.removeprefix("OpenSearch_") for version in opensearch_versions
-        ]
-
-        write_to_cache("opensearch", supported_versions)
-
-        return supported_versions
-
-    else:
-        return read_supported_versions_from_cache("opensearch")
-
-
 def get_connection_string(
     session: Session,
     app: str,

dbt_platform_helper/utils/files.py

@@ -1,6 +1,3 @@
-import os
-from copy import deepcopy
-from datetime import datetime
 from os import makedirs
 from pathlib import Path
 
@@ -9,8 +6,6 @@ import yaml
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 
-from dbt_platform_helper.constants import PLATFORM_HELPER_CACHE_FILE
-
 
 def to_yaml(value):
     return yaml.dump(value, sort_keys=False)
@@ -71,104 +66,3 @@ def generate_override_files_from_template(base_path, overrides_path, output_dir,
 
     generate_files_for_dir("*")
     generate_files_for_dir("bin/*")
-
-
-def apply_environment_defaults(config):
-    if "environments" not in config:
-        return config
-
-    enriched_config = deepcopy(config)
-
-    environments = enriched_config["environments"]
-    env_defaults = environments.get("*", {})
-    without_defaults_entry = {
-        name: data if data else {} for name, data in environments.items() if name != "*"
-    }
-
-    default_versions = config.get("default_versions", {})
-
-    def combine_env_data(data):
-        return {
-            **env_defaults,
-            **data,
-            "versions": {
-                **default_versions,
-                **env_defaults.get("versions", {}),
-                **data.get("versions", {}),
-            },
-        }
-
-    defaulted_envs = {
-        env_name: combine_env_data(env_data)
-        for env_name, env_data in without_defaults_entry.items()
-    }
-
-    enriched_config["environments"] = defaulted_envs
-
-    return enriched_config
-
-
-def read_supported_versions_from_cache(resource_name):
-
-    platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
-
-    return platform_helper_config.get(resource_name).get("versions")
-
-
-def write_to_cache(resource_name, supported_versions):
-
-    platform_helper_config = {}
-
-    if os.path.exists(PLATFORM_HELPER_CACHE_FILE):
-        platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
-
-    cache_dict = {
-        resource_name: {
-            "versions": supported_versions,
-            "date-retrieved": datetime.now().strftime("%d-%m-%y %H:%M:%S"),
-        }
-    }
-
-    platform_helper_config.update(cache_dict)
-
-    with open(PLATFORM_HELPER_CACHE_FILE, "w") as file:
-        file.write("# [!] This file is autogenerated via the platform-helper. Do not edit.\n")
-        yaml.dump(platform_helper_config, file)
-
-
-def cache_refresh_required(resource_name) -> bool:
-    """
-    Checks if the platform-helper should reach out to AWS to 'refresh' its
-    cached values.
-
-    An API call is needed if any of the following conditions are met:
-        1. No cache file (.platform-helper-config.yml) exists.
-        2. The resource name (e.g. redis, opensearch) does not exist within the cache file.
-        3. The date-retrieved value of the cached data is > than a time interval. In this case 1 day.
-    """
-
-    if not os.path.exists(PLATFORM_HELPER_CACHE_FILE):
-        return True
-
-    platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
-
-    if platform_helper_config.get(resource_name):
-        return check_if_cached_datetime_is_greater_than_interval(
-            platform_helper_config[resource_name].get("date-retrieved"), 1
-        )
-
-    return True
-
-
-def check_if_cached_datetime_is_greater_than_interval(date_retrieved, interval_in_days):
-
-    current_datetime = datetime.now()
-    cached_datetime = datetime.strptime(date_retrieved, "%d-%m-%y %H:%M:%S")
-    delta = current_datetime - cached_datetime
-
-    return delta.days > interval_in_days
-
-
-def read_file_as_yaml(file_name):
-
-    return yaml.safe_load(Path(file_name).read_text())

dbt_platform_helper/utils/template.py

@@ -5,6 +5,16 @@ import jinja2
 from dbt_platform_helper.jinja2_tags import ExtraHeaderTag
 from dbt_platform_helper.jinja2_tags import VersionTag
 
+S3_CROSS_ACCOUNT_POLICY = "addons/svc/s3-cross-account-policy.yml"
+
+ADDON_TEMPLATE_MAP = {
+    "s3": ["addons/svc/s3-policy.yml"],
+    "s3-policy": ["addons/svc/s3-policy.yml"],
+    "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
+    "subscription-filter": ["addons/svc/subscription-filter.yml"],
+    "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
+}
+
 
 def camel_case(s):
     s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "")