dbt-platform-helper 12.5.0__py3-none-any.whl → 12.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. dbt_platform_helper/COMMANDS.md +39 -38
  2. dbt_platform_helper/commands/codebase.py +5 -8
  3. dbt_platform_helper/commands/conduit.py +2 -2
  4. dbt_platform_helper/commands/config.py +1 -1
  5. dbt_platform_helper/commands/copilot.py +4 -2
  6. dbt_platform_helper/commands/environment.py +40 -24
  7. dbt_platform_helper/commands/pipeline.py +6 -171
  8. dbt_platform_helper/constants.py +1 -0
  9. dbt_platform_helper/domain/codebase.py +20 -23
  10. dbt_platform_helper/domain/conduit.py +10 -12
  11. dbt_platform_helper/domain/config_validator.py +40 -7
  12. dbt_platform_helper/domain/copilot_environment.py +135 -131
  13. dbt_platform_helper/domain/database_copy.py +45 -42
  14. dbt_platform_helper/domain/maintenance_page.py +220 -183
  15. dbt_platform_helper/domain/pipelines.py +212 -0
  16. dbt_platform_helper/domain/terraform_environment.py +68 -35
  17. dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py +100 -0
  18. dbt_platform_helper/providers/cache.py +1 -2
  19. dbt_platform_helper/providers/cloudformation.py +12 -1
  20. dbt_platform_helper/providers/config.py +21 -13
  21. dbt_platform_helper/providers/copilot.py +2 -0
  22. dbt_platform_helper/providers/files.py +26 -0
  23. dbt_platform_helper/providers/io.py +31 -0
  24. dbt_platform_helper/providers/load_balancers.py +29 -3
  25. dbt_platform_helper/providers/platform_config_schema.py +10 -7
  26. dbt_platform_helper/providers/vpc.py +106 -0
  27. dbt_platform_helper/providers/yaml_file.py +3 -14
  28. dbt_platform_helper/templates/COMMANDS.md.jinja +5 -3
  29. dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +819 -623
  30. dbt_platform_helper/utils/application.py +32 -34
  31. dbt_platform_helper/utils/aws.py +0 -50
  32. dbt_platform_helper/utils/files.py +8 -23
  33. dbt_platform_helper/utils/messages.py +2 -3
  34. dbt_platform_helper/utils/platform_config.py +0 -7
  35. dbt_platform_helper/utils/versioning.py +12 -0
  36. {dbt_platform_helper-12.5.0.dist-info → dbt_platform_helper-12.6.0.dist-info}/METADATA +2 -2
  37. {dbt_platform_helper-12.5.0.dist-info → dbt_platform_helper-12.6.0.dist-info}/RECORD +40 -35
  38. {dbt_platform_helper-12.5.0.dist-info → dbt_platform_helper-12.6.0.dist-info}/WHEEL +1 -1
  39. {dbt_platform_helper-12.5.0.dist-info → dbt_platform_helper-12.6.0.dist-info}/LICENSE +0 -0
  40. {dbt_platform_helper-12.5.0.dist-info → dbt_platform_helper-12.6.0.dist-info}/entry_points.txt +0 -0
@@ -3,133 +3,297 @@ import random
3
3
  import re
4
4
  import string
5
5
  from pathlib import Path
6
+ from typing import Callable
6
7
  from typing import List
7
8
  from typing import Union
8
9
 
9
10
  import boto3
10
11
  import click
11
12
 
13
+ from dbt_platform_helper.platform_exception import PlatformException
14
+ from dbt_platform_helper.providers.io import ClickIOProvider
12
15
  from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
13
16
  from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
14
17
  from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
15
- from dbt_platform_helper.providers.load_balancers import find_https_listener
18
+ from dbt_platform_helper.providers.load_balancers import (
19
+ get_https_listener_for_application,
20
+ )
21
+ from dbt_platform_helper.utils.application import Application
16
22
  from dbt_platform_helper.utils.application import Environment
17
23
  from dbt_platform_helper.utils.application import Service
18
- from dbt_platform_helper.utils.application import load_application
19
24
 
20
25
 
21
- class MaintenancePageProvider:
22
- def activate(self, app, env, svc, template, vpc):
23
- application = load_application(app)
24
- application_environment = get_app_environment(app, env)
26
+ class MaintenancePageException(PlatformException):
27
+ pass
25
28
 
29
+
30
+ class LoadBalancedWebServiceNotFoundException(MaintenancePageException):
31
+ def __init__(self, application_name):
32
+ super().__init__(f"No services deployed yet to {application_name} ")
33
+
34
+
35
+ def get_maintenance_page_type(session: boto3.Session, listener_arn: str) -> Union[str, None]:
36
+ lb_client = session.client("elbv2")
37
+
38
+ rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
39
+ tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
40
+
41
+ maintenance_page_type = None
42
+ for description in tag_descriptions:
43
+ tags = {t["Key"]: t["Value"] for t in description["Tags"]}
44
+ if tags.get("name") == "MaintenancePage":
45
+ maintenance_page_type = tags.get("type")
46
+
47
+ return maintenance_page_type
48
+
49
+
50
+ def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
51
+ account_name = f"{application_environment.session.profile_name}-vpc"
52
+ vpc_name = vpc if vpc else account_name
53
+ ssm_client = application_environment.session.client("ssm")
54
+
55
+ try:
56
+ param_value = ssm_client.get_parameter(Name=f"/{vpc_name}/EGRESS_IPS")["Parameter"]["Value"]
57
+ except ssm_client.exceptions.ParameterNotFound:
58
+ click.secho(f"No parameter found with name: /{vpc_name}/EGRESS_IPS")
59
+ raise click.Abort
60
+
61
+ return [ip.strip() for ip in param_value.split(",")]
62
+
63
+
64
+ def add_maintenance_page(
65
+ session: boto3.Session,
66
+ listener_arn: str,
67
+ app: str,
68
+ env: str,
69
+ services: List[Service],
70
+ allowed_ips: tuple,
71
+ template: str = "default",
72
+ ):
73
+ lb_client = session.client("elbv2")
74
+ maintenance_page_content = get_maintenance_page_template(template)
75
+ bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
76
+
77
+ rule_priority = itertools.count(start=1)
78
+
79
+ for svc in services:
80
+ target_group_arn = find_target_group(app, env, svc.name, session)
81
+
82
+ # not all of an application's services are guaranteed to have been deployed to an environment
83
+ if not target_group_arn:
84
+ continue
85
+
86
+ for ip in allowed_ips:
87
+ create_header_rule(
88
+ lb_client,
89
+ listener_arn,
90
+ target_group_arn,
91
+ "X-Forwarded-For",
92
+ [ip],
93
+ "AllowedIps",
94
+ next(rule_priority),
95
+ )
96
+ create_source_ip_rule(
97
+ lb_client,
98
+ listener_arn,
99
+ target_group_arn,
100
+ [ip],
101
+ "AllowedSourceIps",
102
+ next(rule_priority),
103
+ )
104
+
105
+ create_header_rule(
106
+ lb_client,
107
+ listener_arn,
108
+ target_group_arn,
109
+ "Bypass-Key",
110
+ [bypass_value],
111
+ "BypassIpFilter",
112
+ next(rule_priority),
113
+ )
114
+
115
+ click.secho(
116
+ f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
117
+ )
118
+
119
+ lb_client.create_rule(
120
+ ListenerArn=listener_arn,
121
+ Priority=next(rule_priority),
122
+ Conditions=[
123
+ {
124
+ "Field": "path-pattern",
125
+ "PathPatternConfig": {"Values": ["/*"]},
126
+ }
127
+ ],
128
+ Actions=[
129
+ {
130
+ "Type": "fixed-response",
131
+ "FixedResponseConfig": {
132
+ "StatusCode": "503",
133
+ "ContentType": "text/html",
134
+ "MessageBody": maintenance_page_content,
135
+ },
136
+ }
137
+ ],
138
+ Tags=[
139
+ {"Key": "name", "Value": "MaintenancePage"},
140
+ {"Key": "type", "Value": template},
141
+ ],
142
+ )
143
+
144
+
145
+ def remove_maintenance_page(session: boto3.Session, listener_arn: str):
146
+ lb_client = session.client("elbv2")
147
+
148
+ rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
149
+ tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
150
+
151
+ for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
152
+ deleted = delete_listener_rule(tag_descriptions, name, lb_client)
153
+
154
+ if name == "MaintenancePage" and not deleted:
155
+ raise ListenerRuleNotFoundException()
156
+
157
+
158
+ class MaintenancePage:
159
+ def __init__(
160
+ self,
161
+ application: Application,
162
+ io: ClickIOProvider = ClickIOProvider(),
163
+ get_https_listener_for_application: Callable[
164
+ [boto3.Session, str, str], str
165
+ ] = get_https_listener_for_application,
166
+ # TODO refactor get_maintenance_page_type, add_maintenance_page, remove_maintenance_page into MaintenancePage class with LoadBalancerProvider as the dependency
167
+ get_maintenance_page_type: Callable[
168
+ [boto3.Session, str], Union[str, None]
169
+ ] = get_maintenance_page_type,
170
+ get_env_ips: Callable[[str, Environment], List[str]] = get_env_ips,
171
+ add_maintenance_page: Callable[
172
+ [boto3.Session, str, str, str, List[Service], tuple, str], None
173
+ ] = add_maintenance_page,
174
+ remove_maintenance_page: Callable[[boto3.Session, str], None] = remove_maintenance_page,
175
+ ):
176
+ self.application = application
177
+ self.get_https_listener_for_application = get_https_listener_for_application
178
+ self.io = io
179
+ self.get_maintenance_page_type = get_maintenance_page_type
180
+ self.get_env_ips = get_env_ips
181
+ self.add_maintenance_page = add_maintenance_page
182
+ self.remove_maintenance_page = remove_maintenance_page
183
+
184
+ def _get_deployed_load_balanced_web_services(self, app: Application, svc: List[str]):
26
185
  if "*" in svc:
27
- services = [
28
- s for s in application.services.values() if s.kind == "Load Balanced Web Service"
29
- ]
186
+ services = [s for s in app.services.values() if s.kind == "Load Balanced Web Service"]
30
187
  else:
31
188
  all_services = [get_app_service(app, s) for s in list(svc)]
32
189
  services = [s for s in all_services if s.kind == "Load Balanced Web Service"]
33
-
34
190
  if not services:
35
- click.secho(f"No services deployed yet to {app} environment {env}", fg="red")
36
- raise click.Abort
191
+ raise LoadBalancedWebServiceNotFoundException(app.name)
192
+ return services
193
+
194
+ def activate(self, env: str, services: List[str], template: str, vpc: Union[str, None]):
195
+
196
+ services = self._get_deployed_load_balanced_web_services(self.application, services)
197
+ application_environment = get_app_environment(self.application, env)
37
198
 
38
199
  try:
39
- https_listener = find_https_listener(application_environment.session, app, env)
40
- current_maintenance_page = get_maintenance_page(
200
+ https_listener = self.get_https_listener_for_application(
201
+ application_environment.session, self.application.name, env
202
+ )
203
+ current_maintenance_page = self.get_maintenance_page_type(
41
204
  application_environment.session, https_listener
42
205
  )
43
206
  remove_current_maintenance_page = False
44
207
  if current_maintenance_page:
45
- remove_current_maintenance_page = click.confirm(
208
+ remove_current_maintenance_page = self.io.confirm(
46
209
  f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
47
- f"environment in {app}.\nWould you like to replace it with a '{template}' "
210
+ f"environment in {self.application.name}.\nWould you like to replace it with a '{template}' "
48
211
  f"maintenance page?"
49
212
  )
50
213
  if not remove_current_maintenance_page:
51
- raise click.Abort
214
+ return
52
215
 
53
- if remove_current_maintenance_page or click.confirm(
216
+ if remove_current_maintenance_page or self.io.confirm(
54
217
  f"You are about to enable the '{template}' maintenance page for the {env} "
55
- f"environment in {app}.\nWould you like to continue?"
218
+ f"environment in {self.application.name}.\nWould you like to continue?"
56
219
  ):
57
220
  if current_maintenance_page and remove_current_maintenance_page:
58
- remove_maintenance_page(application_environment.session, https_listener)
221
+ self.remove_maintenance_page(application_environment.session, https_listener)
59
222
 
60
- allowed_ips = get_env_ips(vpc, application_environment)
223
+ allowed_ips = self.get_env_ips(vpc, application_environment)
61
224
 
62
- add_maintenance_page(
225
+ self.add_maintenance_page(
63
226
  application_environment.session,
64
227
  https_listener,
65
- app,
228
+ self.application.name,
66
229
  env,
67
230
  services,
68
231
  allowed_ips,
69
232
  template,
70
233
  )
71
- click.secho(
72
- f"Maintenance page '{template}' added for environment {env} in application {app}",
73
- fg="green",
234
+ self.io.info(
235
+ f"Maintenance page '{template}' added for environment {env} in application {self.application.name}",
74
236
  )
75
- else:
76
- raise click.Abort
77
237
 
78
238
  except LoadBalancerNotFoundException:
79
- click.secho(
80
- f"No load balancer found for environment {env} in the application {app}.", fg="red"
239
+ # TODO push exception to command layer
240
+ self.io.abort_with_error(
241
+ f"No load balancer found for environment {env} in the application {self.application.name}.",
81
242
  )
82
- raise click.Abort
83
243
 
84
244
  except ListenerNotFoundException:
85
- click.secho(
86
- f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
245
+ # TODO push exception to command layer
246
+ self.io.abort_with_error(
247
+ f"No HTTPS listener found for environment {env} in the application {self.application.name}.",
87
248
  )
88
- raise click.Abort
89
249
 
90
- def deactivate(self, app, env):
91
- application_environment = get_app_environment(app, env)
250
+ def deactivate(self, env: str):
251
+ application_environment = get_app_environment(self.application, env)
92
252
 
93
253
  try:
94
- https_listener = find_https_listener(application_environment.session, app, env)
95
- current_maintenance_page = get_maintenance_page(
254
+ https_listener = self.get_https_listener_for_application(
255
+ application_environment.session, self.application.name, env
256
+ )
257
+ current_maintenance_page = self.get_maintenance_page_type(
96
258
  application_environment.session, https_listener
97
259
  )
260
+
261
+ # TODO discuss, reduce number of return statements but more nested if statements
98
262
  if not current_maintenance_page:
99
- click.secho("There is no current maintenance page to remove", fg="red")
100
- raise click.Abort
263
+ self.io.warn("There is no current maintenance page to remove")
264
+ return
101
265
 
102
- if not click.confirm(
266
+ if not self.io.confirm(
103
267
  f"There is currently a '{current_maintenance_page}' maintenance page, "
104
268
  f"would you like to remove it?"
105
269
  ):
106
- raise click.Abort
270
+ return
107
271
 
108
- remove_maintenance_page(application_environment.session, https_listener)
109
- click.secho(
110
- f"Maintenance page removed from environment {env} in application {app}", fg="green"
272
+ self.remove_maintenance_page(application_environment.session, https_listener)
273
+ self.io.info(
274
+ f"Maintenance page removed from environment {env} in application {self.application.name}",
111
275
  )
112
276
 
113
277
  except LoadBalancerNotFoundException:
114
- click.secho(
115
- f"No load balancer found for environment {env} in the application {app}.", fg="red"
278
+ # TODO push exception to command layer
279
+ self.io.abort_with_error(
280
+ f"No load balancer found for environment {env} in the application {self.application.name}.",
116
281
  )
117
- raise click.Abort
118
282
 
119
283
  except ListenerNotFoundException:
120
- click.secho(
121
- f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
284
+ # TODO push exception to command layer
285
+ self.io.abort_with_error(
286
+ f"No HTTPS listener found for environment {env} in the application {self.application.name}.",
122
287
  )
123
- raise click.Abort
124
288
 
125
289
 
126
- def get_app_service(app_name: str, svc_name: str) -> Service:
127
- application = load_application(app_name)
290
+ def get_app_service(application: Application, svc_name: str) -> Service:
128
291
  application_service = application.services.get(svc_name)
129
292
 
130
293
  if not application_service:
294
+ # TODO raise exception instead of abort
131
295
  click.secho(
132
- f"The service {svc_name} was not found in the application {app_name}. "
296
+ f"The service {svc_name} was not found in the application {application.name}. "
133
297
  f"It either does not exist, or has not been deployed.",
134
298
  fg="red",
135
299
  )
@@ -138,13 +302,12 @@ def get_app_service(app_name: str, svc_name: str) -> Service:
138
302
  return application_service
139
303
 
140
304
 
141
- def get_app_environment(app_name: str, env_name: str) -> Environment:
142
- application = load_application(app_name)
305
+ def get_app_environment(application: Application, env_name: str) -> Environment:
143
306
  application_environment = application.environments.get(env_name)
144
307
 
145
308
  if not application_environment:
146
309
  click.secho(
147
- f"The environment {env_name} was not found in the application {app_name}. "
310
+ f"The environment {env_name} was not found in the application {application.name}. "
148
311
  f"It either does not exist, or has not been deployed.",
149
312
  fg="red",
150
313
  )
@@ -153,36 +316,6 @@ def get_app_environment(app_name: str, env_name: str) -> Environment:
153
316
  return application_environment
154
317
 
155
318
 
156
- def get_maintenance_page(session: boto3.Session, listener_arn: str) -> Union[str, None]:
157
- lb_client = session.client("elbv2")
158
-
159
- rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
160
- tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
161
-
162
- maintenance_page_type = None
163
- for description in tag_descriptions:
164
- tags = {t["Key"]: t["Value"] for t in description["Tags"]}
165
- if tags.get("name") == "MaintenancePage":
166
- maintenance_page_type = tags.get("type")
167
-
168
- return maintenance_page_type
169
-
170
-
171
- def remove_maintenance_page(session: boto3.Session, listener_arn: str):
172
- lb_client = session.client("elbv2")
173
-
174
- rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
175
- tag_descriptions = lb_client.describe_tags(ResourceArns=[r["RuleArn"] for r in rules])[
176
- "TagDescriptions"
177
- ]
178
-
179
- for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
180
- deleted = delete_listener_rule(tag_descriptions, name, lb_client)
181
-
182
- if name == "MaintenancePage" and not deleted:
183
- raise ListenerRuleNotFoundException()
184
-
185
-
186
319
  def get_rules_tag_descriptions(rules: list, lb_client):
187
320
  tag_descriptions = []
188
321
  chunk_size = 20
@@ -209,88 +342,6 @@ def delete_listener_rule(tag_descriptions: list, tag_name: str, lb_client: boto3
209
342
  return current_rule_arn
210
343
 
211
344
 
212
- def add_maintenance_page(
213
- session: boto3.Session,
214
- listener_arn: str,
215
- app: str,
216
- env: str,
217
- services: List[Service],
218
- allowed_ips: tuple,
219
- template: str = "default",
220
- ):
221
- lb_client = session.client("elbv2")
222
- maintenance_page_content = get_maintenance_page_template(template)
223
- bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
224
-
225
- rule_priority = itertools.count(start=1)
226
-
227
- for svc in services:
228
- target_group_arn = find_target_group(app, env, svc.name, session)
229
-
230
- # not all of an application's services are guaranteed to have been deployed to an environment
231
- if not target_group_arn:
232
- continue
233
-
234
- for ip in allowed_ips:
235
- create_header_rule(
236
- lb_client,
237
- listener_arn,
238
- target_group_arn,
239
- "X-Forwarded-For",
240
- [ip],
241
- "AllowedIps",
242
- next(rule_priority),
243
- )
244
- create_source_ip_rule(
245
- lb_client,
246
- listener_arn,
247
- target_group_arn,
248
- [ip],
249
- "AllowedSourceIps",
250
- next(rule_priority),
251
- )
252
-
253
- create_header_rule(
254
- lb_client,
255
- listener_arn,
256
- target_group_arn,
257
- "Bypass-Key",
258
- [bypass_value],
259
- "BypassIpFilter",
260
- next(rule_priority),
261
- )
262
-
263
- click.secho(
264
- f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
265
- fg="green",
266
- )
267
-
268
- lb_client.create_rule(
269
- ListenerArn=listener_arn,
270
- Priority=next(rule_priority),
271
- Conditions=[
272
- {
273
- "Field": "path-pattern",
274
- "PathPatternConfig": {"Values": ["/*"]},
275
- }
276
- ],
277
- Actions=[
278
- {
279
- "Type": "fixed-response",
280
- "FixedResponseConfig": {
281
- "StatusCode": "503",
282
- "ContentType": "text/html",
283
- "MessageBody": maintenance_page_content,
284
- },
285
- }
286
- ],
287
- Tags=[
288
- {"Key": "name", "Value": "MaintenancePage"},
289
- {"Key": "type", "Value": template},
290
- ],
291
- )
292
-
293
-
294
345
  def get_maintenance_page_template(template) -> str:
295
346
  template_contents = (
296
347
  Path(__file__)
@@ -448,17 +499,3 @@ def get_host_conditions(lb_client: boto3.client, listener_arn: str, target_group
448
499
  ]
449
500
 
450
501
  return conditions
451
-
452
-
453
- def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
454
- account_name = f"{application_environment.session.profile_name}-vpc"
455
- vpc_name = vpc if vpc else account_name
456
- ssm_client = application_environment.session.client("ssm")
457
-
458
- try:
459
- param_value = ssm_client.get_parameter(Name=f"/{vpc_name}/EGRESS_IPS")["Parameter"]["Value"]
460
- except ssm_client.exceptions.ParameterNotFound:
461
- click.secho(f"No parameter found with name: /{vpc_name}/EGRESS_IPS")
462
- raise click.Abort
463
-
464
- return [ip.strip() for ip in param_value.split(",")]