dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/providers/vpc.py

@@ -77,12 +77,43 @@ class VpcProvider:
         return vpc_id
 
     def _get_security_groups(self, app: str, env: str, vpc_id: str) -> list:
-
         vpc_filter = {"Name": "vpc-id", "Values": [vpc_id]}
-        tag_filter = {"Name": f"tag:Name", "Values": [f"copilot-{app}-{env}-env"]}
-        response = self.ec2_client.describe_security_groups(Filters=[vpc_filter, tag_filter])
-
-        return [sg.get("GroupId") for sg in response.get("SecurityGroups")]
+        platform_sg_name = f"platform-{app}-{env}-env-sg"
+        copilot_sg_name = f"copilot-{app}-{env}-env"
+        tag_filter = {"Name": f"tag:Name", "Values": [copilot_sg_name, platform_sg_name]}
+
+        filtered_security_groups = self.ec2_client.describe_security_groups(
+            Filters=[vpc_filter, tag_filter]
+        )
+
+        platform_security_groups = self._get_matching_security_groups(
+            filtered_security_groups, platform_sg_name
+        )
+
+        if platform_security_groups:
+            print(
+                f"using {platform_security_groups}"
+            )  # TODO remove this once decopilotiing has been completed
+            return platform_security_groups
+
+        copilot_security_groups = self._get_matching_security_groups(
+            filtered_security_groups, copilot_sg_name
+        )
+
+        print(
+            f"using {copilot_security_groups}"
+        )  # TODO remove this once decopilotiing has been completed
+        return copilot_security_groups
+
+    def _get_matching_security_groups(
+        self, filtered_security_groups: list[dict], security_group_name: str
+    ):
+        matching_sec_groups = filtered_security_groups.get("SecurityGroups")
+        return [
+            sg.get("GroupId")
+            for sg in matching_sec_groups
+            if {"Key": "Name", "Value": security_group_name} in sg.get("Tags", [])
+        ]
 
     def get_vpc(self, app: str, env: str, vpc_name: str) -> Vpc:
 

dbt_platform_helper/providers/yaml_file.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from pathlib import Path
 
 import yaml
@@ -29,6 +30,7 @@ class DuplicateKeysException(YamlFileProviderException):
 
 
 class YamlFileProvider:
+    @staticmethod
     def load(path: str) -> dict:
         """
         Raises:
@@ -45,15 +47,25 @@ class YamlFileProvider:
 
         if not yaml_content:
             return {}
-
         YamlFileProvider.lint_yaml_for_duplicate_keys(path)
 
         return yaml_content
 
+    @staticmethod
     def write(path: str, contents: dict, comment: str = ""):
         with open(path, "w") as file:
             file.write(comment)
-            yaml.dump(contents, file)
+            yaml.add_representer(str, account_number_representer)
+            yaml.add_representer(type(None), null_value_representer)
+
+            yaml.dump(
+                contents,
+                file,
+                canonical=False,
+                sort_keys=False,
+                default_style=None,
+                default_flow_style=False,
+            )
 
     @staticmethod
     def lint_yaml_for_duplicate_keys(path):
@@ -72,3 +84,47 @@ class YamlFileProvider:
             ]
         if duplicate_keys:
             raise DuplicateKeysException(",".join(duplicate_keys))
+
+    @staticmethod
+    def remove_empty_keys(config: (dict, OrderedDict)) -> (dict, OrderedDict):
+        cleaned = config.__class__()
+
+        for k, v in config.items():
+            if isinstance(v, (dict, OrderedDict)):
+                v = YamlFileProvider.remove_empty_keys(v)
+            if v not in (None, [], {}, ()):
+                cleaned[k] = v
+
+        return cleaned
+
+    @staticmethod
+    def find_and_replace(config, strings: list, replacements: list):
+        if len(strings) != len(replacements):
+            raise ValueError("'strings' and 'replacements' must be the same length.")
+        if not isinstance(strings, list) or not isinstance(replacements, list):
+            raise ValueError("'strings' and 'replacements' must both be lists.")
+        if isinstance(config, (dict, OrderedDict)):
+            return {
+                k: YamlFileProvider.find_and_replace(v, strings, replacements)
+                for k, v in config.items()
+            }
+        elif isinstance(config, list):
+            return [
+                YamlFileProvider.find_and_replace(item, strings, replacements) for item in config
+            ]
+        elif isinstance(config, str):
+            for s, r in zip(strings, replacements):
+                config = config.replace(s, r)
+            return config
+        else:
+            return replacements if config == strings else config
+
+
+def account_number_representer(dumper, data):
+    if data.isdigit():
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'")
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=None)
+
+
+def null_value_representer(dumper, data):
+    return dumper.represent_scalar("tag:yaml.org,2002:null", "")

dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -10,10 +10,10 @@ locals {
 provider "aws" {
   region                   = "eu-west-2"
   profile                  = "{{ aws_account }}"
-  alias                    = "{{ aws_account }}"
-  shared_credentials_files = ["~/.aws/config"]
+  allowed_account_ids      = ["{{ deploy_account_id }}"]
 }
 
+
 terraform {
   required_version = "{{ terraform_version }}"
   backend "s3" {
@@ -34,7 +34,7 @@ terraform {
 
 
 module "environment-pipelines" {
-  source = "git::https://github.com/uktrade/terraform-platform-modules.git//environment-pipelines?depth=1&ref={{ terraform_platform_modules_version }}"
+  source = "{{ module_source }}"
 
   for_each = local.pipelines
 
@@ -49,4 +49,5 @@ module "environment-pipelines" {
   slack_channel       = each.value.slack_channel
   trigger_on_push     = each.value.trigger_on_push
   pipeline_to_trigger = lookup(each.value, "pipeline_to_trigger", null)
+  pinned_version      = {% if pinned_version  %}"{{ pinned_version }}"{% else %}null{% endif %}
 }

dbt_platform_helper/templates/svc/overrides/cfn.patches.yml

@@ -24,3 +24,8 @@
     Condition:
       StringEquals:
           'ssm:ResourceTag/copilot-application': '__all__'
+
+- op: add
+  path: /Resources/TaskDefinition/Properties/pidMode
+  value:
+    task

dbt_platform_helper/utilities/decorators.py

@@ -0,0 +1,103 @@
+import functools
+import time
+from typing import Callable
+from typing import Optional
+
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+
+SECONDS_BEFORE_RETRY = 3
+RETRY_MAX_ATTEMPTS = 3
+
+
+class RetryException(PlatformException):
+
+    def __init__(
+        self, function_name: str, max_attempts: int, original_exception: Optional[Exception] = None
+    ):
+        message = f"Function: {function_name} failed after {max_attempts} attempts"
+        self.original_exception = original_exception
+        if original_exception:
+            message += f": \n{str(original_exception)}"
+        super().__init__(message)
+
+
+def retry(
+    exceptions_to_catch: tuple = (Exception,),
+    max_attempts: int = RETRY_MAX_ATTEMPTS,
+    delay: float = SECONDS_BEFORE_RETRY,
+    raise_custom_exception: bool = True,
+    custom_exception: type = RetryException,
+    io: ClickIOProvider = ClickIOProvider(),
+):
+    def decorator(func):
+        func.__wrapped_by__ = "retry"
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions_to_catch as e:
+                    last_exception = e
+                    io.debug(
+                        f"Attempt {attempt+1}/{max_attempts} for {func.__name__} failed with exception {str(last_exception)}"
+                    )
+                    if attempt < max_attempts - 1:
+                        time.sleep(delay)
+            if raise_custom_exception:
+                raise custom_exception(func.__name__, max_attempts, last_exception)
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+def wait_until(
+    exceptions_to_catch: tuple = (PlatformException,),
+    max_attempts: int = RETRY_MAX_ATTEMPTS,
+    delay: float = SECONDS_BEFORE_RETRY,
+    raise_custom_exception: bool = True,
+    custom_exception=RetryException,
+    message_on_false="Condition not met",
+    io: ClickIOProvider = ClickIOProvider(),
+):
+    """Wrap a function which returns a boolean."""
+
+    def decorator(func: Callable[..., bool]):
+        func.__wrapped_by__ = "wait_until"
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(max_attempts):
+                try:
+                    result = func(*args, **kwargs)
+                    if result:
+                        return result
+                    io.debug(
+                        f"Attempt {attempt+1}/{max_attempts} for {func.__name__} returned falsy"
+                    )
+                except exceptions_to_catch as e:
+                    last_exception = e
+                    io.debug(
+                        f"Attempt {attempt+1}/{max_attempts} for {func.__name__} failed with exception {str(last_exception)}"
+                    )
+
+                if attempt < max_attempts - 1:
+                    time.sleep(delay)
+
+            if not last_exception:  # If func returns false set last_exception
+                last_exception = PlatformException(message_on_false)
+            if (
+                not raise_custom_exception
+            ):  # Raise last_exception when you don't want custom exception
+                raise last_exception
+            else:
+                raise custom_exception(func.__name__, max_attempts, last_exception)
+
+        return wrapper
+
+    return decorator

dbt_platform_helper/utils/application.py

@@ -59,23 +59,11 @@ class Application:
         return str(self) == str(other)
 
 
-def load_application(app=None, default_session=None) -> Application:
+def load_application(app=None, default_session=None, env=None) -> Application:
     application = Application(app if app else get_application_name())
     current_session = default_session if default_session else get_aws_session_or_abort()
 
     ssm_client = current_session.client("ssm")
-
-    try:
-        ssm_client.get_parameter(
-            Name=f"/copilot/applications/{application.name}",
-            WithDecryption=False,
-        )
-    except ssm_client.exceptions.ParameterNotFound:
-        raise ApplicationNotFoundException(application.name)
-
-    path = f"/copilot/applications/{application.name}/environments"
-    secrets = get_ssm_secrets(app, None, current_session, path)
-
     sts_client = current_session.client("sts")
     account_id = sts_client.get_caller_identity()["Account"]
     sessions = {account_id: current_session}
@@ -86,20 +74,112 @@ def load_application(app=None, default_session=None) -> Application:
         nesting.
 
         e.g.
+         - /platform/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env/addons will not match.
         """
-        environment_key_regex = r"^/copilot/applications/{}/environments/[^/]*$".format(
+        environment_key_regex = r"^/(copilot|platform)/applications/{}/environments/[^/]*$".format(
             application.name
         )
         return bool(re.match(environment_key_regex, name))
 
-    environments = {
+    environments_data = []
+
+    # Try to load all /platform SSM parameters that are present
+    env_params = get_ssm_secrets(
+        app=app,
+        env=None,
+        session=current_session,
+        path=f"/platform/applications/{application.name}/environments",
+    )
+
+    if env_params:
+        for name, value in env_params:
+            try:
+                param_data = json.loads(value)
+            except json.JSONDecodeError:
+                continue
+
+            # Each /platform SSM parameter contains data about all the environments of an application
+            if "allEnvironments" in param_data:
+                environments_data = param_data["allEnvironments"]
+                break  # Only need one
+    else:
+        try:
+            # Check that the Copilot application exists
+            ssm_client.get_parameter(
+                Name=f"/copilot/applications/{application.name}",
+                WithDecryption=False,
+            )
+
+            # Legacy /copilot SSM parameters for each environment
+            env_params = get_ssm_secrets(
+                app, None, current_session, f"/copilot/applications/{application.name}/environments"
+            )
+
+            for name, value in env_params:
+                try:
+                    param_data = json.loads(value)
+                except json.JSONDecodeError:
+                    continue
+
+                if is_environment_key(name):
+                    environments_data.append(param_data)
+
+        except ssm_client.exceptions.ParameterNotFound:
+            raise ApplicationNotFoundException(
+                application_name=application.name, environment_name=env
+            )
+
+    application.environments = {
         env["name"]: Environment(env["name"], env["accountID"], sessions)
-        for env in [json.loads(s[1]) for s in secrets if is_environment_key(s[0])]
+        for env in environments_data
     }
-    application.environments = environments
 
+    application.services = _load_services(ssm_client, application)
+
+    return application
+
+
+def _load_services(ssm_client, application: Application) -> Dict[str, Service]:
+    """
+    Try to load
+    /platform/applications/{app}/environments/{env}/services/{service}
+    parameters if present.
+
+    Otherwise, fall back to legacy /copilot/applications/{app}/components
+    parameters.
+    """
+    services: Dict[str, Service] = {}
+
+    # Try /platform SSM parameter
+    for env_name in application.environments.keys():
+        params = dict(
+            Path=f"/platform/applications/{application.name}/environments/{env_name}/services",
+            Recursive=False,
+            WithDecryption=False,
+        )
+
+        while True:
+            response = ssm_client.get_parameters_by_path(**params)
+            for ssm_param in response.get("Parameters", []):
+                try:
+                    data = json.loads(ssm_param["Value"])
+                    name = data["name"]
+                    kind = data["type"]
+                    services.setdefault(name, Service(name, kind))  # Avoid duplicates
+                except (json.JSONDecodeError, KeyError):
+                    continue
+
+            if "NextToken" in response:
+                params["NextToken"] = response["NextToken"]
+            else:
+                break
+
+    if services:
+        return services
+
+    # Fallback to legacy /copilot SSM parameter
     response = ssm_client.get_parameters_by_path(
         Path=f"/copilot/applications/{application.name}/components",
         Recursive=False,
@@ -115,17 +195,17 @@ def load_application(app=None, default_session=None) -> Application:
         )
         results.extend(response["Parameters"])
 
-    application.services = {
+    legacy_services = {
         svc["name"]: Service(svc["name"], svc["type"])
         for svc in [json.loads(parameter["Value"]) for parameter in results]
     }
 
-    return application
+    return legacy_services
 
 
 def get_application_name(abort=abort_with_error):
     if Path(PLATFORM_CONFIG_FILE).exists():
-        config = ConfigProvider()
+        config = ConfigProvider(installed_version_provider="N/A")
         try:
             app_config = config.load_unvalidated_config_file()
             return app_config["application"]
@@ -142,7 +222,24 @@ class ApplicationException(PlatformException):
 
 
 class ApplicationNotFoundException(ApplicationException):
-    def __init__(self, application_name: str):
+    def __init__(self, application_name: str, environment_name: str):
+        super().__init__(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}". 
+Please ensure that the environment variable "AWS_PROFILE" is set correctly. If the issue persists, verify that one of the following AWS SSM parameters exists:
+ - /platform/applications/{application_name}/environments/{environment_name}
+ - /copilot/applications/{application_name}"""
+        )
+
+
+class ApplicationServiceNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, svc_name: str):
+        super().__init__(
+            f"""The service {svc_name} was not found in the application {application_name}. It either does not exist, or has not been deployed."""
+        )
+
+
+class ApplicationEnvironmentNotFoundException(ApplicationException):
+    def __init__(self, application_name: str, environment: str):
         super().__init__(
-            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}"; ensure you have set the environment variable "AWS_PROFILE" correctly."""
+            f"""The environment "{environment}" either does not exist or has not been deployed for the application {application_name}."""
         )