dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.1__py3-none-any.whl

dbt_platform_helper/providers/redis.py

@@ -0,0 +1,34 @@
+from dbt_platform_helper.providers.cache import CacheProvider
+
+
+class RedisProvider:
+    def __init__(self, elasticache_client):
+        self.elasticache_client = elasticache_client
+
+    def get_supported_redis_versions(self):
+
+        cache_provider = self.__get_cache_provider()
+
+        if cache_provider.cache_refresh_required("redis"):
+
+            supported_versions_response = self.elasticache_client.describe_cache_engine_versions(
+                Engine="redis"
+            )
+
+            supported_versions = [
+                version["EngineVersion"]
+                for version in supported_versions_response["CacheEngineVersions"]
+            ]
+
+            cache_provider.update_cache("redis", supported_versions)
+
+            return supported_versions
+
+        else:
+            return cache_provider.read_supported_versions_from_cache("redis")
+
+    # TODO - cache provider instantiated here rather than via dependancy injection since it will likely only be used in the get_supported_redis_versions method.
+    # If another method is added which needs a CacheProvider, it should be injected into the constructor instead.
+    @staticmethod
+    def __get_cache_provider():
+        return CacheProvider()

dbt_platform_helper/providers/vpc.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+from dbt_platform_helper.providers.aws import AWSException
+
+
+@dataclass
+class Vpc:
+    subnets: list[str]
+    security_groups: list[str]
+
+
+class VpcProvider:
+    def __init__(self, session):
+        self.ec2_client = session.client("ec2")
+        self.ec2_resource = session.resource("ec2")
+
+    def get_vpc_info_by_name(self, app: str, env: str, vpc_name: str) -> Vpc:
+        vpc_response = self.ec2_client.describe_vpcs(
+            Filters=[{"Name": "tag:Name", "Values": [vpc_name]}]
+        )
+
+        matching_vpcs = vpc_response.get("Vpcs", [])
+
+        if not matching_vpcs:
+            raise AWSException(f"VPC not found for name '{vpc_name}'")
+
+        vpc_id = vpc_response["Vpcs"][0].get("VpcId")
+
+        if not vpc_id:
+            raise AWSException(f"VPC id not present in vpc '{vpc_name}'")
+
+        vpc = self.ec2_resource.Vpc(vpc_id)
+
+        route_tables = self.ec2_client.describe_route_tables(
+            Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+        )["RouteTables"]
+
+        subnets = []
+        for route_table in route_tables:
+            private_routes = [route for route in route_table["Routes"] if "NatGatewayId" in route]
+            if not private_routes:
+                continue
+            for association in route_table["Associations"]:
+                if "SubnetId" in association:
+                    subnet_id = association["SubnetId"]
+                    subnets.append(subnet_id)
+
+        if not subnets:
+            raise AWSException(f"No private subnets found in vpc '{vpc_name}'")
+
+        tag_value = {"Key": "Name", "Value": f"copilot-{app}-{env}-env"}
+        sec_groups = [sg.id for sg in vpc.security_groups.all() if sg.tags and tag_value in sg.tags]
+
+        if not sec_groups:
+            raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
+
+        return Vpc(subnets, sec_groups)

dbt_platform_helper/providers/yaml_file.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+
+import yaml
+from yaml.parser import ParserError
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+
+
+class FileProviderException(Exception):
+    pass
+
+
+class YamlFileProviderException(FileProviderException):
+    pass
+
+
+class FileNotFoundException(FileProviderException):
+    pass
+
+
+class InvalidYamlException(YamlFileProviderException):
+    pass
+
+
+class DuplicateKeysException(YamlFileProviderException):
+    pass
+
+
+class YamlFileProvider:
+    def load(path: str) -> dict:
+        """
+        Raises:
+            FileNotFoundException: file is not there
+            InvalidYamlException: file contains invalid yaml
+            DuplicateKeysException: yaml contains duplicate keys
+        """
+        if not Path(path).exists():
+            raise FileNotFoundException(f"`{path}` is missing.")
+        try:
+            yaml_content = yaml.safe_load(Path(path).read_text())
+        except ParserError:
+            raise InvalidYamlException(f"{path} is not valid YAML.")
+
+        if not yaml_content:
+            return {}
+
+        YamlFileProvider.lint_yaml_for_duplicate_keys(path)
+
+        return yaml_content
+
+    def write(path: str, contents: dict, comment: str = ""):
+        with open(path, "w") as file:
+            file.write(comment)
+            yaml.dump(contents, file)
+
+    @staticmethod
+    def lint_yaml_for_duplicate_keys(path):
+        duplicate_keys = []
+        with open(path, "r") as yaml_file:
+            file_contents = yaml_file.read()
+            results = linter.run(
+                file_contents, YamlLintConfig(yaml.dump({"rules": {"key-duplicates": "enable"}}))
+            )
+            duplicate_keys = [
+                "\t"
+                + f"Line {result.line}: {result.message}".replace(
+                    " in mapping (key-duplicates)", ""
+                )
+                for result in results
+            ]
+        if duplicate_keys:
+            raise DuplicateKeysException(",".join(duplicate_keys))

dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml

@@ -0,0 +1,67 @@
+# {% extra_header %}
+# {% version_info %}
+
+Metadata:
+  cfn-lint:
+    config:
+      ignore_checks:
+        - W2001 # Parameter not used
+
+Parameters:
+  # Copilot required Parameters...
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: The name of the service, job, or workflow being deployed.
+
+Resources: {% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Metadata:
+      'aws:copilot:description': 'An IAM ManagedPolicy for your service to access the bucket'
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: Grants Read access to the S3 bucket.
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: 'KMSDecryptAndGenerate'
+            Effect: Allow
+            Action:
+              - kms:Decrypt
+              - kms:GenerateDataKey
+            Resource: 'arn:aws:kms:eu-west-2:{{ resource.bucket_account }}:key/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ObjectActions'
+            Effect: Allow
+            Action:
+              {% if resource.read %}- s3:Get*{% endif %}
+              {% if resource.write %}- s3:Put*{% endif %}
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}/*'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+          - Sid: 'S3ListAction'
+            Effect: Allow
+            Action:
+              - s3:ListBucket
+            Resource: 'arn:aws:s3:::{{ resource.bucket_name }}'
+            Condition:
+              StringEquals:
+                aws:PrincipalTag/copilot-environment:
+                  - "{{ resource.access_env }}"
+    {% endfor %}
+Outputs:{% for resource in resources %}
+  {{ resource.app_prefix }}XEnvAccessPolicy:
+    Description: "The IAM::ManagedPolicy to attach to the task role"
+    Value:
+      Ref: {{ resource.app_prefix }}XEnvAccessPolicy
+ {% endfor %}

dbt_platform_helper/utils/application.py

@@ -1,31 +1,28 @@
 import json
 import os
 import re
+from dataclasses import dataclass
+from dataclasses import field
 from pathlib import Path
 from typing import Dict
 
 import boto3
-import yaml
-from boto3 import Session
-from yaml.parser import ParserError
 
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
 from dbt_platform_helper.utils.aws import get_ssm_secrets
 from dbt_platform_helper.utils.messages import abort_with_error
+from dbt_platform_helper.utils.platform_config import load_unvalidated_config_file
 
 
+@dataclass
 class Environment:
     name: str
     account_id: str
     sessions: Dict[str, boto3.Session]
 
-    def __init__(self, name: str, account_id: str, sessions: Dict[str, boto3.Session]):
-        self.name = name
-        self.account_id = account_id
-        self.sessions = sessions
-
     @property
     def session(self):
         if self.account_id not in self.sessions:
@@ -36,24 +33,17 @@ class Environment:
         return self.sessions[self.account_id]
 
 
+@dataclass
 class Service:
     name: str
     kind: str
 
-    def __init__(self, name: str, kind: str):
-        self.name = name
-        self.kind = kind
-
 
+@dataclass
 class Application:
     name: str
-    environments: Dict[str, Environment]
-    services: Dict[str, Service]
-
-    def __init__(self, name: str):
-        self.name = name
-        self.environments = {}
-        self.services = {}
+    environments: Dict[str, Environment] = field(default_factory=dict)
+    services: Dict[str, Service] = field(default_factory=dict)
 
     def __str__(self):
         output = f"Application {self.name} with"
@@ -69,7 +59,7 @@ class Application:
         return str(self) == str(other)
 
 
-def load_application(app: str = None, default_session: Session = None) -> Application:
+def load_application(app=None, default_session=None) -> Application:
     application = Application(app if app else get_application_name())
     current_session = default_session if default_session else get_aws_session_or_abort()
 
@@ -81,7 +71,7 @@ def load_application(app: str = None, default_session: Session = None) -> Applic
             WithDecryption=False,
         )
     except ssm_client.exceptions.ParameterNotFound:
-        raise ApplicationNotFoundException(app)
+        raise ApplicationNotFoundException(application.name)
 
     path = f"/copilot/applications/{application.name}/environments"
     secrets = get_ssm_secrets(app, None, current_session, path)
@@ -115,27 +105,35 @@ def load_application(app: str = None, default_session: Session = None) -> Applic
         Recursive=False,
         WithDecryption=False,
     )
+    results = response["Parameters"]
+    while "NextToken" in response:
+        response = ssm_client.get_parameters_by_path(
+            Path=f"/copilot/applications/{application.name}/components",
+            Recursive=False,
+            WithDecryption=False,
+            NextToken=response["NextToken"],
+        )
+        results.extend(response["Parameters"])
 
     application.services = {
         svc["name"]: Service(svc["name"], svc["type"])
-        for svc in [json.loads(parameter["Value"]) for parameter in response["Parameters"]]
+        for svc in [json.loads(parameter["Value"]) for parameter in results]
     }
 
     return application
 
 
-def get_application_name():
-    app_name = None
-    try:
-        app_config = yaml.safe_load(Path("copilot/.workspace").read_text())
-        app_name = app_config["application"]
-    except (FileNotFoundError, ParserError):
-        pass
-
-    if app_name is None:
-        abort_with_error("Cannot get application name. No copilot/.workspace file found")
-
-    return app_name
+def get_application_name(abort=abort_with_error):
+    if Path(PLATFORM_CONFIG_FILE).exists():
+        try:
+            app_config = load_unvalidated_config_file()
+            return app_config["application"]
+        except KeyError:
+            abort(
+                f"Cannot get application name. No 'application' key can be found in {PLATFORM_CONFIG_FILE}"
+            )
+    else:
+        abort(f"Cannot get application name. {PLATFORM_CONFIG_FILE} is missing.")
 
 
 class ApplicationException(PlatformException):

dbt_platform_helper/utils/aws.py

@@ -13,12 +13,11 @@ import click
 import yaml
 from boto3 import Session
 
+from dbt_platform_helper.constants import REFRESH_TOKEN_MESSAGE
 from dbt_platform_helper.platform_exception import PlatformException
-from dbt_platform_helper.providers.aws import AWSException
 from dbt_platform_helper.providers.aws import CopilotCodebaseNotFoundException
 from dbt_platform_helper.providers.aws import ImageNotFoundException
 from dbt_platform_helper.providers.aws import LogGroupNotFoundException
-from dbt_platform_helper.providers.cache import CacheProvider
 from dbt_platform_helper.providers.validation import ValidationException
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
@@ -27,9 +26,6 @@ AWS_SESSION_CACHE = {}
 
 
 def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
-    REFRESH_TOKEN_MESSAGE = (
-        "To refresh this SSO session run `aws sso login` with the corresponding profile"
-    )
     aws_profile = aws_profile or os.getenv("AWS_PROFILE")
     if aws_profile in AWS_SESSION_CACHE:
         return AWS_SESSION_CACHE[aws_profile]
@@ -358,59 +354,6 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
     return parameter_data
 
 
-def get_supported_redis_versions():
-
-    cache_provider = CacheProvider()
-
-    if cache_provider.cache_refresh_required("redis"):
-
-        session = get_aws_session_or_abort()
-        elasticache_client = session.client("elasticache")
-
-        supported_versions_response = elasticache_client.describe_cache_engine_versions(
-            Engine="redis"
-        )
-
-        supported_versions = [
-            version["EngineVersion"]
-            for version in supported_versions_response["CacheEngineVersions"]
-        ]
-
-        cache_provider.update_cache("redis", supported_versions)
-
-        return supported_versions
-
-    else:
-        return cache_provider.read_supported_versions_from_cache("redis")
-
-
-def get_supported_opensearch_versions():
-
-    cache_provider = CacheProvider()
-
-    if cache_provider.cache_refresh_required("opensearch"):
-
-        session = get_aws_session_or_abort()
-        opensearch_client = session.client("opensearch")
-
-        response = opensearch_client.list_versions()
-        all_versions = response["Versions"]
-
-        opensearch_versions = [
-            version for version in all_versions if not version.startswith("Elasticsearch_")
-        ]
-        supported_versions = [
-            version.removeprefix("OpenSearch_") for version in opensearch_versions
-        ]
-
-        cache_provider.update_cache("opensearch", supported_versions)
-
-        return supported_versions
-
-    else:
-        return cache_provider.read_supported_versions_from_cache("opensearch")
-
-
 def get_connection_string(
     session: Session,
     app: str,
@@ -433,55 +376,6 @@ def get_connection_string(
     return f"postgres://{conn['username']}:{conn['password']}@{conn['host']}:{conn['port']}/{conn['dbname']}"
 
 
-class Vpc:
-    def __init__(self, subnets: list[str], security_groups: list[str]):
-        self.subnets = subnets
-        self.security_groups = security_groups
-
-
-def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) -> Vpc:
-    ec2_client = session.client("ec2")
-    vpc_response = ec2_client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [vpc_name]}])
-
-    matching_vpcs = vpc_response.get("Vpcs", [])
-
-    if not matching_vpcs:
-        raise AWSException(f"VPC not found for name '{vpc_name}'")
-
-    vpc_id = vpc_response["Vpcs"][0].get("VpcId")
-
-    if not vpc_id:
-        raise AWSException(f"VPC id not present in vpc '{vpc_name}'")
-
-    ec2_resource = session.resource("ec2")
-    vpc = ec2_resource.Vpc(vpc_id)
-
-    route_tables = ec2_client.describe_route_tables(
-        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
-    )["RouteTables"]
-
-    subnets = []
-    for route_table in route_tables:
-        private_routes = [route for route in route_table["Routes"] if "NatGatewayId" in route]
-        if not private_routes:
-            continue
-        for association in route_table["Associations"]:
-            if "SubnetId" in association:
-                subnet_id = association["SubnetId"]
-                subnets.append(subnet_id)
-
-    if not subnets:
-        raise AWSException(f"No private subnets found in vpc '{vpc_name}'")
-
-    tag_value = {"Key": "Name", "Value": f"copilot-{app}-{env}-env"}
-    sec_groups = [sg.id for sg in vpc.security_groups.all() if sg.tags and tag_value in sg.tags]
-
-    if not sec_groups:
-        raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
-
-    return Vpc(subnets, sec_groups)
-
-
 def start_build_extraction(codebuild_client, build_options):
     response = codebuild_client.start_build(**build_options)
     return response["build"]["arn"]

dbt_platform_helper/utils/files.py

@@ -1,35 +1,17 @@
-from copy import deepcopy
-from os import makedirs
-from pathlib import Path
-
 import click
 import yaml
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 
+from dbt_platform_helper.providers.files import FileProvider
+
 
 def to_yaml(value):
     return yaml.dump(value, sort_keys=False)
 
 
-def mkfile(base_path, file_path, contents, overwrite=False):
-    file_path = Path(file_path)
-    file = Path(base_path).joinpath(file_path)
-    file_exists = file.exists()
-
-    if not file_path.parent.exists():
-        makedirs(file_path.parent)
-
-    if file_exists and not overwrite:
-        return f"File {file_path} exists; doing nothing"
-
-    action = "overwritten" if file_exists and overwrite else "created"
-
-    file.write_text(contents)
-
-    return f"File {file_path} {action}"
-
-
+# TODO - extract file provider functionality from this - and figure out what it actually does!
+# Move to the new file provider - or potentially copilot?
 def generate_override_files(base_path, file_path, output_dir):
     def generate_files_for_dir(pattern):
         for file in file_path.glob(pattern):
@@ -37,7 +19,7 @@ def generate_override_files(base_path, file_path, output_dir):
                 contents = file.read_text()
                 file_name = str(file).removeprefix(f"{file_path}/")
                 click.echo(
-                    mkfile(
+                    FileProvider.mkfile(
                         base_path,
                         output_dir / file_name,
                         contents,
@@ -62,43 +44,10 @@ def generate_override_files_from_template(base_path, overrides_path, output_dir,
             if file.is_file():
                 file_name = str(file).removeprefix(f"{overrides_path}/")
                 contents = templates.get_template(str(file_name)).render(data)
-                message = mkfile(base_path, output_dir / file_name, contents, overwrite=True)
+                message = FileProvider.mkfile(
+                    base_path, output_dir / file_name, contents, overwrite=True
+                )
                 click.echo(message)
 
     generate_files_for_dir("*")
     generate_files_for_dir("bin/*")
-
-
-def apply_environment_defaults(config):
-    if "environments" not in config:
-        return config
-
-    enriched_config = deepcopy(config)
-
-    environments = enriched_config["environments"]
-    env_defaults = environments.get("*", {})
-    without_defaults_entry = {
-        name: data if data else {} for name, data in environments.items() if name != "*"
-    }
-
-    default_versions = config.get("default_versions", {})
-
-    def combine_env_data(data):
-        return {
-            **env_defaults,
-            **data,
-            "versions": {
-                **default_versions,
-                **env_defaults.get("versions", {}),
-                **data.get("versions", {}),
-            },
-        }
-
-    defaulted_envs = {
-        env_name: combine_env_data(env_data)
-        for env_name, env_data in without_defaults_entry.items()
-    }
-
-    enriched_config["environments"] = defaulted_envs
-
-    return enriched_config

dbt_platform_helper/utils/platform_config.py

@@ -18,10 +18,3 @@ def load_unvalidated_config_file():
         return yaml.safe_load(file_contents)
     except yaml.parser.ParserError:
         return {}
-
-
-def get_environment_pipeline_names():
-    pipelines_config = load_unvalidated_config_file().get("environment_pipelines")
-    if pipelines_config:
-        return pipelines_config.keys()
-    return {}

dbt_platform_helper/utils/template.py

@@ -5,6 +5,16 @@ import jinja2
 from dbt_platform_helper.jinja2_tags import ExtraHeaderTag
 from dbt_platform_helper.jinja2_tags import VersionTag
 
+S3_CROSS_ACCOUNT_POLICY = "addons/svc/s3-cross-account-policy.yml"
+
+ADDON_TEMPLATE_MAP = {
+    "s3": ["addons/svc/s3-policy.yml"],
+    "s3-policy": ["addons/svc/s3-policy.yml"],
+    "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
+    "subscription-filter": ["addons/svc/subscription-filter.yml"],
+    "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
+}
+
 
 def camel_case(s):
     s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "")