dbt-platform-helper 15.11.0__tar.gz → 15.12.0__tar.gz

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-platform-helper
-Version: 15.11.0
+Version: 15.12.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 License-File: LICENSE

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/commands/internal.py

@@ -74,6 +74,8 @@ def deploy(name, env, image_tag):
         s3_provider = S3Provider(client=s3_client)
         logs_provider = LogsProvider(client=logs_client)
 
+        account_id = application.environments.get(env).account_id
+
         service_manager = ServiceManager(
             ecs_provider=ecs_provider, s3_provider=s3_provider, logs_provider=logs_provider
         )
@@ -81,6 +83,7 @@ def deploy(name, env, image_tag):
             service=name,
             environment=env,
             application=application.name,
+            account_id=account_id,
             image_tag=image_tag,
         )
     except PlatformException as error:

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/constants.py

@@ -1,4 +1,5 @@
 # TODO: DBTP-1888: Move to Config provider
+REPO_ORG = "uktrade"
 PLATFORM_CONFIG_FILE = "platform-config.yml"
 SERVICE_DIRECTORY = "services"
 SERVICE_CONFIG_FILE = "service-config.yml"
@@ -59,3 +60,5 @@ STANDARD_PLATFORM_SSO_ROLES = [
     "DBTPlatformDeveloperWrite",
     "DBTPlatformDeveloperRead",
 ]
+ROUTED_TO_PLATFORM_MODES = ["platform", "dual-deploy-platform-traffic"]
+ROUTED_TO_COPILOT_MODES = ["copilot", "dual-deploy-copilot-traffic"]

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/domain/conduit.py

@@ -56,10 +56,16 @@ class TerraformConduitStrategy(ConduitECSStrategy):
 
     def get_data(self):
         self.io.info("Starting conduit in Terraform mode.")
-        return {
-            "cluster_arn": self.ecs_provider.get_cluster_arn_by_name(
+        try:
+            cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
+                f"{self.application.name}-{self.env}-cluster"
+            )
+        except:
+            cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
                 f"{self.application.name}-{self.env}"
-            ),
+            )
+        return {
+            "cluster_arn": cluster_arn,
             "task_def_family": self._generate_container_name(),
             "vpc_name": self._resolve_vpc_name(),
             "addon_type": self.addon_type,
@@ -94,8 +100,10 @@ class TerraformConduitStrategy(ConduitECSStrategy):
                 },
             ]
 
+        cluster_name = data_context["cluster_arn"].split("/")[-1]
+
         self.ecs_provider.start_ecs_task(
-            f"{self.application.name}-{self.env}",
+            cluster_name,
             self._generate_container_name(),
             data_context["task_def_family"],
             vpc_config,

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/domain/service.py

@@ -231,13 +231,17 @@ class ServiceManager:
                     f"{service_path}/service-config.yml", dict(service_manifest), message
                 )
 
-    def deploy(self, service: str, environment: str, application: str, image_tag: str = None):
+    def deploy(
+        self,
+        service: str,
+        environment: str,
+        application: str,
+        account_id: str,
+        image_tag: str = None,
+    ):
         """Register a new ECS task definition revision, update the ECS service
-        with it, and output Cloudwatch logs until deployment is complete."""
-
-        application_obj = self.load_application(app=application)
-        application_envs = application_obj.environments
-        account_id = application_envs.get(environment).account_id
+        with it, output a Cloudwatch logs URL, and wait until deployment is
+        complete."""
 
         s3_response = self.s3_provider.get_object(
             bucket_name=f"ecs-task-definitions-{application}-{environment}",

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/providers/load_balancers.py

@@ -1,11 +1,14 @@
+import json
 from typing import Dict
 from typing import List
 
 from boto3 import Session
 
 from dbt_platform_helper.constants import MANAGED_BY_PLATFORM_TERRAFORM
+from dbt_platform_helper.constants import ROUTED_TO_PLATFORM_MODES
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 
 
@@ -33,6 +36,7 @@ class LoadBalancerProvider:
         self.session = session
         self.evlb_client = self._get_client("elbv2")
         self.rg_tagging_client = self._get_client("resourcegroupstaggingapi")
+        self.parameter_store_provider = ParameterStore(self._get_client("ssm"))
         self.io = io
 
     def _get_client(self, client: str):
@@ -41,21 +45,44 @@ class LoadBalancerProvider:
         return self.session.client(client)
 
     def find_target_group(self, app: str, env: str, svc: str) -> str:
+
+        # TODO once copilot is gone this is no longer needed
+        try:
+            result = self.parameter_store_provider.get_ssm_parameter_by_name(
+                f"/platform/applications/{app}/environments/{env}"
+            )["Value"]
+            env_config = json.loads(result)
+            service_deployment_mode = env_config["service_deployment_mode"]
+        except Exception:
+            service_deployment_mode = "copilot"
+
+        if service_deployment_mode in ROUTED_TO_PLATFORM_MODES:
+            application_key = "application"
+            environment_key = "environment"
+            service_key = "service"
+        else:
+            application_key = "copilot-application"
+            environment_key = "copilot-environment"
+            service_key = "copilot-service"
         target_group_arn = None
 
         paginator = self.rg_tagging_client.get_paginator("get_resources")
         page_iterator = paginator.paginate(
             TagFilters=[
                 {
-                    "Key": "copilot-application",
+                    "Key": application_key,
                     "Values": [
                         app,
                     ],
-                    "Key": "copilot-environment",
+                },
+                {
+                    "Key": environment_key,
                     "Values": [
                         env,
                     ],
-                    "Key": "copilot-service",
+                },
+                {
+                    "Key": service_key,
                     "Values": [
                         svc,
                     ],
@@ -71,9 +98,9 @@ class LoadBalancerProvider:
                 tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
 
                 if (
-                    tags.get("copilot-service") == svc
-                    and tags.get("copilot-environment") == env
-                    and tags.get("copilot-application") == app
+                    tags.get(service_key) == svc
+                    and tags.get(environment_key) == env
+                    and tags.get(application_key) == app
                 ):
                     target_group_arn = resource["ResourceARN"]
 

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/dbt_platform_helper/utils/application.py

@@ -74,6 +74,7 @@ def load_application(app=None, default_session=None, env=None) -> Application:
         nesting.
 
         e.g.
+         - /platform/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env/addons will not match.
         """
@@ -84,20 +85,24 @@ def load_application(app=None, default_session=None, env=None) -> Application:
 
     environments_data = []
 
-    # Try to load the new /platform SSM parameter if present
-    platform_env_path = f"/platform/applications/{application.name}/environments"
-    secrets = get_ssm_secrets(app, None, current_session, platform_env_path)
+    # Try to load all /platform SSM parameters that are present
+    env_params = get_ssm_secrets(
+        app=app,
+        env=None,
+        session=current_session,
+        path=f"/platform/applications/{application.name}/environments",
+    )
 
-    if secrets:
-        for name, value in secrets:
+    if env_params:
+        for name, value in env_params:
             try:
-                data = json.loads(value)
+                param_data = json.loads(value)
             except json.JSONDecodeError:
                 continue
 
-            # New /platform SSM parameter contains data about all environments
-            if "allEnvironments" in data:
-                environments_data = data["allEnvironments"]
+            # Each /platform SSM parameter contains data about all the environments of an application
+            if "allEnvironments" in param_data:
+                environments_data = param_data["allEnvironments"]
                 break  # Only need one
     else:
         try:
@@ -106,19 +111,20 @@ def load_application(app=None, default_session=None, env=None) -> Application:
                 Name=f"/copilot/applications/{application.name}",
                 WithDecryption=False,
             )
-            secrets = get_ssm_secrets(
+
+            # Legacy /copilot SSM parameters for each environment
+            env_params = get_ssm_secrets(
                 app, None, current_session, f"/copilot/applications/{application.name}/environments"
             )
 
-            for name, value in secrets:
+            for name, value in env_params:
                 try:
-                    data = json.loads(value)
+                    param_data = json.loads(value)
                 except json.JSONDecodeError:
                     continue
 
                 if is_environment_key(name):
-                    # Legacy /copilot SSM parameter. An individual SSM param is present per environment - looping through all of them is needed to extract necessary data about each env.
-                    environments_data.append(data)
+                    environments_data.append(param_data)
 
         except ssm_client.exceptions.ParameterNotFound:
             raise ApplicationNotFoundException(
@@ -130,6 +136,50 @@ def load_application(app=None, default_session=None, env=None) -> Application:
         for env in environments_data
     }
 
+    application.services = _load_services(ssm_client, application)
+
+    return application
+
+
+def _load_services(ssm_client, application: Application) -> Dict[str, Service]:
+    """
+    Try to load
+    /platform/applications/{app}/environments/{env}/services/{service}
+    parameters if present.
+
+    Otherwise, fall back to legacy /copilot/applications/{app}/components
+    parameters.
+    """
+    services: Dict[str, Service] = {}
+
+    # Try /platform SSM parameter
+    for env_name in application.environments.keys():
+        params = dict(
+            Path=f"/platform/applications/{application.name}/environments/{env_name}/services",
+            Recursive=False,
+            WithDecryption=False,
+        )
+
+        while True:
+            response = ssm_client.get_parameters_by_path(**params)
+            for ssm_param in response.get("Parameters", []):
+                try:
+                    data = json.loads(ssm_param["Value"])
+                    name = data["name"]
+                    kind = data["type"]
+                    services.setdefault(name, Service(name, kind))  # Avoid duplicates
+                except (json.JSONDecodeError, KeyError):
+                    continue
+
+            if "NextToken" in response:
+                params["NextToken"] = response["NextToken"]
+            else:
+                break
+
+    if services:
+        return services
+
+    # Fallback to legacy /copilot SSM parameter
     response = ssm_client.get_parameters_by_path(
         Path=f"/copilot/applications/{application.name}/components",
         Recursive=False,
@@ -145,12 +195,12 @@ def load_application(app=None, default_session=None, env=None) -> Application:
         )
         results.extend(response["Parameters"])
 
-    application.services = {
+    legacy_services = {
         svc["name"]: Service(svc["name"], svc["type"])
         for svc in [json.loads(parameter["Value"]) for parameter in results]
     }
 
-    return application
+    return legacy_services
 
 
 def get_application_name(abort=abort_with_error):

{dbt_platform_helper-15.11.0 → dbt_platform_helper-15.12.0}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "15.11.0"
+version = "15.12.0"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"