dbt-platform-helper 15.0.0__tar.gz → 15.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (106) hide show
  1. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/PKG-INFO +1 -1
  2. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/codebase.py +12 -3
  3. dbt_platform_helper-15.2.0/dbt_platform_helper/domain/conduit.py +365 -0
  4. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/config.py +4 -4
  5. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/versioning.py +6 -6
  6. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/platform_exception.py +4 -0
  7. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/exceptions.py +20 -2
  8. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/config.py +2 -2
  9. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/copilot.py +20 -12
  10. dbt_platform_helper-15.2.0/dbt_platform_helper/providers/ecr.py +102 -0
  11. dbt_platform_helper-15.2.0/dbt_platform_helper/providers/ecs.py +174 -0
  12. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/io.py +6 -1
  13. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/terraform_manifest.py +7 -1
  14. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/version.py +1 -1
  15. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/version_status.py +1 -1
  16. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/vpc.py +1 -1
  17. dbt_platform_helper-15.2.0/dbt_platform_helper/utilities/decorators.py +103 -0
  18. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/validation.py +1 -1
  19. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/pyproject.toml +1 -1
  20. dbt_platform_helper-15.0.0/dbt_platform_helper/domain/conduit.py +0 -119
  21. dbt_platform_helper-15.0.0/dbt_platform_helper/providers/ecr.py +0 -77
  22. dbt_platform_helper-15.0.0/dbt_platform_helper/providers/ecs.py +0 -102
  23. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/LICENSE +0 -0
  24. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/COMMANDS.md +0 -0
  25. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/README.md +0 -0
  26. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/__init__.py +0 -0
  27. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/addon-plans.yml +0 -0
  28. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/__init__.py +0 -0
  29. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/application.py +0 -0
  30. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/codebase.py +0 -0
  31. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/conduit.py +0 -0
  32. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/config.py +0 -0
  33. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/copilot.py +0 -0
  34. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/database.py +0 -0
  35. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/environment.py +0 -0
  36. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/generate.py +0 -0
  37. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/notify.py +0 -0
  38. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/pipeline.py +0 -0
  39. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/secrets.py +0 -0
  40. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/commands/version.py +0 -0
  41. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/constants.py +0 -0
  42. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/default-extensions.yml +0 -0
  43. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/__init__.py +0 -0
  44. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/copilot.py +0 -0
  45. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/copilot_environment.py +0 -0
  46. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/database_copy.py +0 -0
  47. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/maintenance_page.py +0 -0
  48. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/notify.py +0 -0
  49. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/pipelines.py +0 -0
  50. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/domain/terraform_environment.py +0 -0
  51. {dbt_platform_helper-15.0.0/dbt_platform_helper/providers → dbt_platform_helper-15.2.0/dbt_platform_helper/entities}/platform_config_schema.py +0 -0
  52. {dbt_platform_helper-15.0.0/dbt_platform_helper/providers → dbt_platform_helper-15.2.0/dbt_platform_helper/entities}/semantic_version.py +0 -0
  53. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/jinja2_tags.py +0 -0
  54. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/__init__.py +0 -0
  55. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/__init__.py +0 -0
  56. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/interfaces.py +0 -0
  57. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/opensearch.py +0 -0
  58. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/redis.py +0 -0
  59. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/aws/sso_auth.py +0 -0
  60. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/cache.py +0 -0
  61. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/cloudformation.py +0 -0
  62. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/config_validator.py +0 -0
  63. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/files.py +0 -0
  64. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/kms.py +0 -0
  65. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/load_balancers.py +0 -0
  66. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/parameter_store.py +0 -0
  67. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
  68. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +0 -0
  69. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/schema_migrator.py +0 -0
  70. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/secrets.py +0 -0
  71. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/slack_channel_notifier.py +0 -0
  72. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/validation.py +0 -0
  73. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/providers/yaml_file.py +0 -0
  74. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/config.yml +0 -0
  75. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/image_build_run.sh +0 -0
  76. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/phases/build.sh +0 -0
  77. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/phases/install.sh +0 -0
  78. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/phases/post_build.sh +0 -0
  79. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/.copilot/phases/pre_build.sh +0 -0
  80. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/COMMANDS.md.jinja +0 -0
  81. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addon-instructions.txt +0 -0
  82. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/README.md +0 -0
  83. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/svc/appconfig-ipfilter.yml +0 -0
  84. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/svc/prometheus-policy.yml +0 -0
  85. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml +0 -0
  86. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/svc/s3-policy.yml +0 -0
  87. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/addons/svc/subscription-filter.yml +0 -0
  88. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/ci-codebuild-role-policy.json +0 -0
  89. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/create-codebuild-role.json +0 -0
  90. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/custom-codebuild-role-policy.json +0 -0
  91. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/env/manifest.yml +0 -0
  92. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml +0 -0
  93. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/environment-pipelines/main.tf +0 -0
  94. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/default.html +0 -0
  95. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/dmas-migration.html +0 -0
  96. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/migration.html +0 -0
  97. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +0 -0
  98. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/__init__.py +0 -0
  99. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/application.py +0 -0
  100. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/arn_parser.py +0 -0
  101. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/aws.py +0 -0
  102. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/click.py +0 -0
  103. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/git.py +0 -0
  104. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/messages.py +0 -0
  105. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/dbt_platform_helper/utils/template.py +0 -0
  106. {dbt_platform_helper-15.0.0 → dbt_platform_helper-15.2.0}/platform_helper.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: dbt-platform-helper
3
- Version: 15.0.0
3
+ Version: 15.2.0
4
4
  Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
5
5
  License: MIT
6
6
  Author: Department for Business and Trade Platform Team
@@ -167,13 +167,16 @@ class Codebase:
167
167
 
168
168
  image_ref = None
169
169
  if commit:
170
- image_ref = f"commit-{commit[0:7]}"
170
+ self._validate_sha_length(commit)
171
+ image_ref = f"commit-{commit}"
171
172
  elif tag:
172
173
  image_ref = f"tag-{tag}"
173
174
  elif branch:
174
175
  image_ref = f"branch-{branch}"
175
- image_details = self.ecr_provider.get_image_details(application, codebase, image_ref)
176
- image_ref = self.ecr_provider.find_commit_tag(image_details, image_ref)
176
+
177
+ image_ref = self.ecr_provider.get_commit_tag_for_reference(
178
+ application.name, codebase, image_ref
179
+ )
177
180
 
178
181
  codepipeline_client = session.client("codepipeline")
179
182
  pipeline_name = self.get_manual_release_pipeline(codepipeline_client, app, codebase)
@@ -284,6 +287,12 @@ class Codebase:
284
287
  return get_build_url_from_pipeline_execution_id(execution_id, build_options["name"])
285
288
  return None
286
289
 
290
+ def _validate_sha_length(self, commit):
291
+ if len(commit) < 7:
292
+ self.io.abort_with_error(
293
+ "Your commit reference is too short. Commit sha hashes specified by '--commit' must be at least 7 characters long."
294
+ )
295
+
287
296
 
288
297
  class ApplicationDeploymentNotTriggered(PlatformException):
289
298
  def __init__(self, codebase: str):
@@ -0,0 +1,365 @@
1
+ from abc import ABC
2
+ from abc import abstractmethod
3
+ from typing import Callable
4
+ from typing import Optional
5
+
6
+ from dbt_platform_helper.providers.cloudformation import CloudFormation
7
+ from dbt_platform_helper.providers.copilot import _normalise_secret_name
8
+ from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
9
+ from dbt_platform_helper.providers.copilot import create_addon_client_task
10
+ from dbt_platform_helper.providers.copilot import get_postgres_admin_connection_string
11
+ from dbt_platform_helper.providers.ecs import ECS
12
+ from dbt_platform_helper.providers.io import ClickIOProvider
13
+ from dbt_platform_helper.providers.secrets import Secrets
14
+ from dbt_platform_helper.providers.vpc import VpcProvider
15
+ from dbt_platform_helper.utils.application import Application
16
+
17
+
18
+ class ConduitECSStrategy(ABC):
19
+ @abstractmethod
20
+ def get_data(self):
21
+ pass
22
+
23
+ @abstractmethod
24
+ def start_task(self, data_context: dict):
25
+ pass
26
+
27
+ @abstractmethod
28
+ def exec_task(self, data_context: dict):
29
+ pass
30
+
31
+
32
+ class TerraformConduitStrategy(ConduitECSStrategy):
33
+ def __init__(
34
+ self,
35
+ clients,
36
+ ecs_provider: ECS,
37
+ application: Application,
38
+ addon_name: str,
39
+ addon_type: str,
40
+ access: str,
41
+ env: str,
42
+ io: ClickIOProvider,
43
+ vpc_provider: Callable,
44
+ get_postgres_admin_connection_string: Callable,
45
+ ):
46
+ self.clients = clients
47
+ self.ecs_provider = ecs_provider
48
+ self.io = io
49
+ self.vpc_provider = vpc_provider
50
+ self.access = access
51
+ self.addon_name = addon_name
52
+ self.addon_type = addon_type
53
+ self.application = application
54
+ self.env = env
55
+ self.get_postgres_admin_connection_string = get_postgres_admin_connection_string
56
+
57
+ def get_data(self):
58
+ self.io.info("Starting conduit in Terraform mode.")
59
+ return {
60
+ "cluster_arn": self.ecs_provider.get_cluster_arn_by_name(
61
+ f"{self.application.name}-{self.env}"
62
+ ),
63
+ "task_def_family": self._generate_container_name(),
64
+ "vpc_name": self._resolve_vpc_name(),
65
+ "addon_type": self.addon_type,
66
+ "access": self.access,
67
+ }
68
+
69
+ def start_task(self, data_context: dict):
70
+
71
+ environments = self.application.environments
72
+ environment = environments.get(self.env)
73
+ env_session = environment.session
74
+
75
+ vpc_provider = self.vpc_provider(env_session)
76
+ vpc_config = vpc_provider.get_vpc(
77
+ self.application.name,
78
+ self.env,
79
+ data_context["vpc_name"],
80
+ )
81
+
82
+ postgres_admin_env_vars = None
83
+ if data_context["addon_type"] == "postgres" and data_context["access"] == "admin":
84
+ postgres_admin_env_vars = [
85
+ {
86
+ "name": "CONNECTION_SECRET",
87
+ "value": self.get_postgres_admin_connection_string(
88
+ self.clients.get("ssm"),
89
+ f"/copilot/{self.application.name}/{self.env}/secrets/{_normalise_secret_name(self.addon_name)}",
90
+ self.application,
91
+ self.env,
92
+ self.addon_name,
93
+ ),
94
+ },
95
+ ]
96
+
97
+ self.ecs_provider.start_ecs_task(
98
+ f"{self.application.name}-{self.env}",
99
+ self._generate_container_name(),
100
+ data_context["task_def_family"],
101
+ vpc_config,
102
+ postgres_admin_env_vars,
103
+ )
104
+
105
+ def exec_task(self, data_context: dict):
106
+ self.ecs_provider.exec_task(data_context["cluster_arn"], data_context["task_arns"][0])
107
+
108
+ def _generate_container_name(self):
109
+ return f"conduit-{self.addon_type}-{self.access}-{self.application.name}-{self.env}-{self.addon_name}"
110
+
111
+ def _resolve_vpc_name(self):
112
+ ssm_client = self.clients["ssm"]
113
+ parameter_key = f"/conduit/{self.application.name}/{self.env}/{_normalise_secret_name(self.addon_name)}_VPC_NAME"
114
+
115
+ try:
116
+ response = ssm_client.get_parameter(Name=parameter_key)
117
+ return response["Parameter"]["Value"]
118
+ except ssm_client.exceptions.ParameterNotFound:
119
+ self.io.abort_with_error(
120
+ f"Could not find VPC name for {self.addon_name}. Missing SSM param: {parameter_key}"
121
+ )
122
+
123
+
124
+ class CopilotConduitStrategy(ConduitECSStrategy):
125
+ def __init__(
126
+ self,
127
+ clients,
128
+ ecs_provider: ECS,
129
+ secrets_provider: Secrets,
130
+ cloudformation_provider: CloudFormation,
131
+ application: Application,
132
+ addon_name: str,
133
+ access: str,
134
+ env: str,
135
+ io: ClickIOProvider,
136
+ connect_to_addon_client_task: Callable,
137
+ create_addon_client_task: Callable,
138
+ ):
139
+ self.clients = clients
140
+ self.cloudformation_provider = cloudformation_provider
141
+ self.ecs_provider = ecs_provider
142
+ self.secrets_provider = secrets_provider
143
+
144
+ self.io = io
145
+ self.access = access
146
+ self.addon_name = addon_name
147
+ self.application = application
148
+ self.env = env
149
+ self.connect_to_addon_client_task = connect_to_addon_client_task
150
+ self.create_addon_client_task = create_addon_client_task
151
+
152
+ def get_data(self):
153
+
154
+ addon_type = self.secrets_provider.get_addon_type(self.addon_name)
155
+ parameter_name = self.secrets_provider.get_parameter_name(
156
+ addon_type, self.addon_name, self.access
157
+ )
158
+ task_name = self.ecs_provider.get_or_create_task_name(self.addon_name, parameter_name)
159
+
160
+ return {
161
+ "cluster_arn": self.ecs_provider.get_cluster_arn_by_copilot_tag(),
162
+ "addon_type": addon_type,
163
+ "task_def_family": f"copilot-{task_name}",
164
+ "parameter_name": parameter_name,
165
+ "task_name": task_name,
166
+ }
167
+
168
+ def start_task(self, data_context: dict):
169
+ self.create_addon_client_task(
170
+ self.clients["iam"],
171
+ self.clients["ssm"],
172
+ self.application,
173
+ self.env,
174
+ data_context["addon_type"],
175
+ self.addon_name,
176
+ data_context["task_name"],
177
+ self.access,
178
+ )
179
+
180
+ self.io.info("Updating conduit task")
181
+ self.cloudformation_provider.add_stack_delete_policy_to_task_role(data_context["task_name"])
182
+ stack_name = self.cloudformation_provider.update_conduit_stack_resources(
183
+ self.application.name,
184
+ self.env,
185
+ data_context["addon_type"],
186
+ self.addon_name,
187
+ data_context["task_name"],
188
+ data_context["parameter_name"],
189
+ self.access,
190
+ )
191
+ self.io.info("Waiting for conduit task update to complete...")
192
+ self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
193
+ "stack_update_complete", stack_name
194
+ )
195
+
196
+ def exec_task(self, data_context: dict):
197
+ self.connect_to_addon_client_task(
198
+ self.clients["ecs"],
199
+ self.application.name,
200
+ self.env,
201
+ data_context["cluster_arn"],
202
+ data_context["task_name"],
203
+ )
204
+
205
+
206
+ class ConduitStrategyFactory:
207
+
208
+ @staticmethod
209
+ def detect_mode(
210
+ ecs_client,
211
+ application,
212
+ environment,
213
+ addon_name: str,
214
+ addon_type: str,
215
+ access: str,
216
+ io: ClickIOProvider,
217
+ ) -> str:
218
+ """Detect if Terraform-based conduit task definitions are present,
219
+ otherwise default to Copilot mode."""
220
+ paginator = ecs_client.get_paginator("list_task_definitions")
221
+ prefix = f"conduit-{addon_type}-{access}-{application}-{environment}-{addon_name}"
222
+
223
+ for page in paginator.paginate():
224
+ for arn in page["taskDefinitionArns"]:
225
+ if arn.split("/")[-1].startswith(prefix):
226
+ return "terraform"
227
+
228
+ io.info("Defaulting to copilot mode.")
229
+ return "copilot"
230
+
231
+ @staticmethod
232
+ def create_strategy(
233
+ mode: str,
234
+ clients,
235
+ ecs_provider: ECS,
236
+ secrets_provider: Secrets,
237
+ cloudformation_provider: CloudFormation,
238
+ application: Application,
239
+ addon_name: str,
240
+ addon_type: str,
241
+ access: str,
242
+ env: str,
243
+ io: ClickIOProvider,
244
+ ):
245
+
246
+ if mode == "terraform":
247
+ return TerraformConduitStrategy(
248
+ clients,
249
+ ecs_provider,
250
+ application,
251
+ addon_name,
252
+ addon_type,
253
+ access,
254
+ env,
255
+ io,
256
+ vpc_provider=VpcProvider,
257
+ get_postgres_admin_connection_string=get_postgres_admin_connection_string,
258
+ )
259
+ else:
260
+ return CopilotConduitStrategy(
261
+ clients,
262
+ ecs_provider,
263
+ secrets_provider,
264
+ cloudformation_provider,
265
+ application,
266
+ addon_name,
267
+ access,
268
+ env,
269
+ io,
270
+ connect_to_addon_client_task=connect_to_addon_client_task,
271
+ create_addon_client_task=create_addon_client_task,
272
+ )
273
+
274
+
275
+ class Conduit:
276
+ def __init__(
277
+ self,
278
+ application: Application,
279
+ secrets_provider: Secrets,
280
+ cloudformation_provider: CloudFormation,
281
+ ecs_provider: ECS,
282
+ io: ClickIOProvider = ClickIOProvider(),
283
+ vpc_provider=VpcProvider,
284
+ strategy_factory: Optional[ConduitStrategyFactory] = None,
285
+ ):
286
+
287
+ self.application = application
288
+ self.secrets_provider = secrets_provider
289
+ self.cloudformation_provider = cloudformation_provider
290
+ self.ecs_provider = ecs_provider
291
+ self.io = io
292
+ self.vpc_provider = vpc_provider
293
+ self.strategy_factory = strategy_factory or ConduitStrategyFactory()
294
+
295
+ def start(self, env: str, addon_name: str, access: str = "read"):
296
+ self.clients = self._initialise_clients(env)
297
+ addon_type = self.secrets_provider.get_addon_type(addon_name)
298
+
299
+ if (addon_type == "opensearch" or addon_type == "redis") and (access != "read"):
300
+ access = "read"
301
+
302
+ mode = self.strategy_factory.detect_mode(
303
+ self.clients.get("ecs"),
304
+ self.application.name,
305
+ env,
306
+ addon_name,
307
+ addon_type,
308
+ access,
309
+ self.io,
310
+ )
311
+
312
+ strategy = self.strategy_factory.create_strategy(
313
+ mode=mode,
314
+ clients=self.clients,
315
+ ecs_provider=self.ecs_provider,
316
+ secrets_provider=self.secrets_provider,
317
+ cloudformation_provider=self.cloudformation_provider,
318
+ application=self.application,
319
+ addon_name=addon_name,
320
+ addon_type=addon_type,
321
+ access=access,
322
+ env=env,
323
+ io=self.io,
324
+ )
325
+
326
+ data_context = strategy.get_data()
327
+
328
+ data_context["task_arns"] = self.ecs_provider.get_ecs_task_arns(
329
+ data_context["cluster_arn"], data_context["task_def_family"]
330
+ )
331
+
332
+ info_log = (
333
+ f"Checking if a conduit ECS task is already running for:\n"
334
+ f" Addon Name : {addon_name}\n"
335
+ f" Addon Type : {addon_type}"
336
+ )
337
+
338
+ if addon_type == "postgres":
339
+ info_log += f"\n Access Level : {access}"
340
+
341
+ self.io.info(info_log)
342
+
343
+ if not data_context["task_arns"]:
344
+ self.io.info("Creating conduit ECS task...")
345
+ strategy.start_task(data_context)
346
+ data_context["task_arns"] = self.ecs_provider.wait_for_task_to_register(
347
+ data_context["cluster_arn"], data_context["task_def_family"]
348
+ )
349
+ else:
350
+ self.io.info(f"Found a task already running: {data_context['task_arns'][0]}")
351
+
352
+ self.io.info(f"Waiting for ECS Exec agent to become available on the conduit task...")
353
+ self.ecs_provider.ecs_exec_is_available(
354
+ data_context["cluster_arn"], data_context["task_arns"]
355
+ )
356
+
357
+ self.io.info("Connecting to conduit task...")
358
+ strategy.exec_task(data_context)
359
+
360
+ def _initialise_clients(self, env):
361
+ return {
362
+ "ecs": self.application.environments[env].session.client("ecs"),
363
+ "iam": self.application.environments[env].session.client("iam"),
364
+ "ssm": self.application.environments[env].session.client("ssm"),
365
+ }
@@ -10,16 +10,16 @@ from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
10
10
  from dbt_platform_helper.domain.versioning import AWSVersioning
11
11
  from dbt_platform_helper.domain.versioning import CopilotVersioning
12
12
  from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
13
+ from dbt_platform_helper.entities.semantic_version import (
14
+ IncompatibleMajorVersionException,
15
+ )
16
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
13
17
  from dbt_platform_helper.platform_exception import PlatformException
14
18
  from dbt_platform_helper.providers.aws.sso_auth import SSOAuthProvider
15
19
  from dbt_platform_helper.providers.config import ConfigProvider
16
20
  from dbt_platform_helper.providers.io import ClickIOProvider
17
21
  from dbt_platform_helper.providers.schema_migrator import ALL_MIGRATIONS
18
22
  from dbt_platform_helper.providers.schema_migrator import Migrator
19
- from dbt_platform_helper.providers.semantic_version import (
20
- IncompatibleMajorVersionException,
21
- )
22
- from dbt_platform_helper.providers.semantic_version import SemanticVersion
23
23
  from dbt_platform_helper.providers.validation import ValidationException
24
24
  from dbt_platform_helper.providers.version_status import VersionStatus
25
25
 
@@ -1,15 +1,15 @@
1
1
  import os
2
2
 
3
- from dbt_platform_helper.platform_exception import PlatformException
4
- from dbt_platform_helper.providers.config import ConfigProvider
5
- from dbt_platform_helper.providers.io import ClickIOProvider
6
- from dbt_platform_helper.providers.semantic_version import (
3
+ from dbt_platform_helper.entities.semantic_version import (
7
4
  IncompatibleMajorVersionException,
8
5
  )
9
- from dbt_platform_helper.providers.semantic_version import (
6
+ from dbt_platform_helper.entities.semantic_version import (
10
7
  IncompatibleMinorVersionException,
11
8
  )
12
- from dbt_platform_helper.providers.semantic_version import SemanticVersion
9
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
10
+ from dbt_platform_helper.platform_exception import PlatformException
11
+ from dbt_platform_helper.providers.config import ConfigProvider
12
+ from dbt_platform_helper.providers.io import ClickIOProvider
13
13
  from dbt_platform_helper.providers.version import AWSCLIInstalledVersionProvider
14
14
  from dbt_platform_helper.providers.version import CopilotInstalledVersionProvider
15
15
  from dbt_platform_helper.providers.version import GithubLatestVersionProvider
@@ -3,3 +3,7 @@
3
3
  # error and abort.
4
4
  class PlatformException(Exception):
5
5
  pass
6
+
7
+
8
+ class ValidationException(PlatformException):
9
+ pass
@@ -12,16 +12,34 @@ class CreateTaskTimeoutException(AWSException):
12
12
  )
13
13
 
14
14
 
15
+ IMAGE_NOT_FOUND_TEMPLATE = """An image labelled "{image_ref}" could not be found in your image repository. Try the `platform-helper codebase build` command first."""
16
+
17
+
15
18
  class ImageNotFoundException(AWSException):
16
19
  def __init__(self, image_ref: str):
20
+ super().__init__(IMAGE_NOT_FOUND_TEMPLATE.format(image_ref=image_ref))
21
+
22
+
23
+ MULTIPLE_IMAGES_FOUND_TEMPLATE = (
24
+ 'Image reference "{image_ref}" is matched by the following images: {matching_images}'
25
+ )
26
+
27
+
28
+ class MultipleImagesFoundException(AWSException):
29
+ def __init__(self, image_ref: str, matching_images: list[str]):
17
30
  super().__init__(
18
- f"""An image labelled "{image_ref}" could not be found in your image repository. Try the `platform-helper codebase build` command first."""
31
+ MULTIPLE_IMAGES_FOUND_TEMPLATE.format(
32
+ image_ref=image_ref, matching_images=", ".join(sorted(matching_images))
33
+ )
19
34
  )
20
35
 
21
36
 
37
+ REPOSITORY_NOT_FOUND_TEMPLATE = """The ECR repository "{repository}" could not be found."""
38
+
39
+
22
40
  class RepositoryNotFoundException(AWSException):
23
41
  def __init__(self, repository: str):
24
- super().__init__(f"""The ECR repository "{repository}" could not be found.""")
42
+ super().__init__(REPOSITORY_NOT_FOUND_TEMPLATE.format(repository=repository))
25
43
 
26
44
 
27
45
  class LogGroupNotFoundException(AWSException):
@@ -8,11 +8,11 @@ from dbt_platform_helper.constants import FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR
8
8
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
9
9
  from dbt_platform_helper.constants import PLATFORM_CONFIG_SCHEMA_VERSION
10
10
  from dbt_platform_helper.constants import PLATFORM_HELPER_PACKAGE_NAME
11
+ from dbt_platform_helper.entities.platform_config_schema import PlatformConfigSchema
12
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
11
13
  from dbt_platform_helper.providers.config_validator import ConfigValidator
12
14
  from dbt_platform_helper.providers.config_validator import ConfigValidatorError
13
15
  from dbt_platform_helper.providers.io import ClickIOProvider
14
- from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
15
- from dbt_platform_helper.providers.semantic_version import SemanticVersion
16
16
  from dbt_platform_helper.providers.version import InstalledVersionProvider
17
17
  from dbt_platform_helper.providers.yaml_file import FileNotFoundException
18
18
  from dbt_platform_helper.providers.yaml_file import FileProviderException
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import subprocess
2
3
  import time
3
4
 
4
5
  from botocore.exceptions import ClientError
@@ -13,7 +14,6 @@ from dbt_platform_helper.utils.messages import abort_with_error
13
14
  def create_addon_client_task(
14
15
  iam_client,
15
16
  ssm_client,
16
- subprocess,
17
17
  application: Application,
18
18
  env: str,
19
19
  addon_type: str,
@@ -31,7 +31,6 @@ def create_addon_client_task(
31
31
  elif access == "admin":
32
32
  create_postgres_admin_task(
33
33
  ssm_client,
34
- subprocess,
35
34
  application,
36
35
  addon_name,
37
36
  addon_type,
@@ -71,15 +70,8 @@ def create_addon_client_task(
71
70
  )
72
71
 
73
72
 
74
- def create_postgres_admin_task(
75
- ssm_client,
76
- subprocess,
77
- app: Application,
78
- addon_name: str,
79
- addon_type: str,
80
- env: str,
81
- secret_name: str,
82
- task_name: str,
73
+ def get_postgres_admin_connection_string(
74
+ ssm_client, secret_name: str, app: Application, env: str, addon_name: str
83
75
  ):
84
76
  read_only_secret_name = secret_name + "_READ_ONLY_USER"
85
77
  master_secret_name = (
@@ -94,6 +86,23 @@ def create_postgres_admin_task(
94
86
  )
95
87
  )
96
88
 
89
+ return connection_string
90
+
91
+
92
+ def create_postgres_admin_task(
93
+ ssm_client,
94
+ app: Application,
95
+ addon_name: str,
96
+ addon_type: str,
97
+ env: str,
98
+ secret_name: str,
99
+ task_name: str,
100
+ ):
101
+
102
+ connection_string = get_postgres_admin_connection_string(
103
+ ssm_client, secret_name, app, env, addon_name
104
+ )
105
+
97
106
  subprocess.call(
98
107
  f"copilot task run --app {app.name} --env {env} "
99
108
  f"--task-group-name {task_name} "
@@ -121,7 +130,6 @@ def _temp_until_refactor_get_ecs_task_arns(ecs_client, cluster_arn: str, task_na
121
130
 
122
131
  def connect_to_addon_client_task(
123
132
  ecs_client,
124
- subprocess,
125
133
  application_name,
126
134
  env,
127
135
  cluster_arn,
@@ -0,0 +1,102 @@
1
+ from collections import defaultdict
2
+
3
+ import botocore
4
+ from boto3 import Session
5
+
6
+ from dbt_platform_helper.providers.aws.exceptions import AWSException
7
+ from dbt_platform_helper.providers.aws.exceptions import ImageNotFoundException
8
+ from dbt_platform_helper.providers.aws.exceptions import MultipleImagesFoundException
9
+ from dbt_platform_helper.providers.aws.exceptions import RepositoryNotFoundException
10
+ from dbt_platform_helper.providers.io import ClickIOProvider
11
+ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
12
+
13
+ NOT_A_UNIQUE_TAG_INFO = 'INFO: The tag "{image_ref}" is not a unique, commit-specific tag. Deploying the corresponding commit tag "{commit_tag}" instead.'
14
+ NO_ASSOCIATED_COMMIT_TAG_WARNING = 'WARNING: The AWS ECR image "{image_ref}" has no associated commit tag so deploying "{image_ref}". Note this could result in images with unintended or incompatible changes being deployed in new ECS Tasks for your service.'
15
+
16
+
17
+ class ECRProvider:
18
+ def __init__(self, session: Session = None, click_io: ClickIOProvider = ClickIOProvider()):
19
+ self.session = session
20
+ self.click_io = click_io
21
+
22
+ def get_ecr_repo_names(self) -> list[str]:
23
+ out = []
24
+ for page in self._get_client().get_paginator("describe_repositories").paginate():
25
+ out.extend([repo["repositoryName"] for repo in page.get("repositories", {})])
26
+ return out
27
+
28
+ def get_commit_tag_for_reference(self, application_name: str, codebase: str, image_ref: str):
29
+ repository = f"{application_name}/{codebase}"
30
+ next_page_token = None
31
+ tag_map = {}
32
+ digest_map = defaultdict(dict)
33
+
34
+ while True:
35
+ image_list = self._get_ecr_images(repository, image_ref, next_page_token)
36
+ next_page_token = image_list.get("nextToken")
37
+
38
+ for image in image_list["imageIds"]:
39
+ digest, tag = image["imageDigest"], image["imageTag"]
40
+ digest_map[digest][tag.split("-")[0]] = tag
41
+ tag_map[tag] = digest
42
+
43
+ if not next_page_token:
44
+ break
45
+
46
+ if image_ref.startswith("commit-"):
47
+ if image_ref in tag_map:
48
+ return image_ref
49
+ else:
50
+ candidates = [
51
+ tag
52
+ for tag in tag_map.keys()
53
+ if image_ref.startswith(tag) or tag.startswith(image_ref)
54
+ ]
55
+ if not candidates:
56
+ raise ImageNotFoundException(image_ref)
57
+ if len(candidates) > 1:
58
+ raise MultipleImagesFoundException(image_ref, candidates)
59
+ return candidates[0]
60
+ else:
61
+ digest = tag_map.get(image_ref)
62
+ if not digest:
63
+ raise ImageNotFoundException(image_ref)
64
+
65
+ commit_tag = digest_map.get(digest, dict()).get("commit")
66
+
67
+ if commit_tag:
68
+ self.click_io.info(
69
+ NOT_A_UNIQUE_TAG_INFO.format(image_ref=image_ref, commit_tag=commit_tag)
70
+ )
71
+ return commit_tag
72
+ else:
73
+ self.click_io.warn(NO_ASSOCIATED_COMMIT_TAG_WARNING.format(image_ref=image_ref))
74
+ return image_ref
75
+
76
+ def _get_ecr_images(self, repository, image_ref, next_page_token):
77
+ params = {"repositoryName": repository, "filter": {"tagStatus": "TAGGED"}}
78
+ if next_page_token:
79
+ params["nextToken"] = next_page_token
80
+ try:
81
+ image_list = self._get_client().list_images(**params)
82
+ return image_list
83
+ except botocore.exceptions.ClientError as e:
84
+ if e.response["Error"]["Code"] == "RepositoryNotFoundException":
85
+ raise RepositoryNotFoundException(repository)
86
+ else:
87
+ raise AWSException(
88
+ f"Unexpected error for repo '{repository}' and image reference '{image_ref}': {e}"
89
+ )
90
+
91
+ @staticmethod
92
+ def _check_image_details_exists(image_info: dict, image_ref: str):
93
+ """Error handling for any unexpected scenario where AWS ECR returns a
94
+ malformed response."""
95
+
96
+ if "imageDetails" not in image_info:
97
+ raise ImageNotFoundException(image_ref)
98
+
99
+ def _get_client(self):
100
+ if not self.session:
101
+ self.session = get_aws_session_or_abort()
102
+ return self.session.client("ecr")