dbt-platform-helper 12.5.1__py3-none-any.whl → 13.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (47) hide show
  1. dbt_platform_helper/COMMANDS.md +45 -42
  2. dbt_platform_helper/commands/codebase.py +7 -10
  3. dbt_platform_helper/commands/conduit.py +2 -2
  4. dbt_platform_helper/commands/config.py +1 -1
  5. dbt_platform_helper/commands/environment.py +32 -18
  6. dbt_platform_helper/commands/notify.py +5 -3
  7. dbt_platform_helper/commands/pipeline.py +17 -11
  8. dbt_platform_helper/constants.py +3 -1
  9. dbt_platform_helper/domain/codebase.py +48 -36
  10. dbt_platform_helper/domain/conduit.py +10 -12
  11. dbt_platform_helper/domain/config_validator.py +42 -31
  12. dbt_platform_helper/domain/copilot_environment.py +133 -129
  13. dbt_platform_helper/domain/database_copy.py +38 -37
  14. dbt_platform_helper/domain/maintenance_page.py +243 -193
  15. dbt_platform_helper/domain/pipelines.py +60 -135
  16. dbt_platform_helper/domain/terraform_environment.py +7 -3
  17. dbt_platform_helper/providers/aws.py +5 -0
  18. dbt_platform_helper/providers/cloudformation.py +12 -1
  19. dbt_platform_helper/providers/config.py +12 -14
  20. dbt_platform_helper/providers/ecr.py +20 -0
  21. dbt_platform_helper/providers/files.py +1 -1
  22. dbt_platform_helper/providers/io.py +31 -0
  23. dbt_platform_helper/providers/load_balancers.py +29 -3
  24. dbt_platform_helper/providers/platform_config_schema.py +24 -22
  25. dbt_platform_helper/providers/terraform_manifest.py +120 -0
  26. dbt_platform_helper/providers/vpc.py +81 -32
  27. dbt_platform_helper/templates/COMMANDS.md.jinja +5 -3
  28. dbt_platform_helper/templates/environment-pipelines/main.tf +2 -2
  29. dbt_platform_helper/templates/environments/main.tf +3 -4
  30. dbt_platform_helper/utils/aws.py +16 -5
  31. dbt_platform_helper/utils/messages.py +2 -3
  32. {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/METADATA +2 -2
  33. {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/RECORD +36 -44
  34. dbt_platform_helper/templates/pipelines/codebase/manifest.yml +0 -56
  35. dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore +0 -12
  36. dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts +0 -8
  37. dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml +0 -29
  38. dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml +0 -48
  39. dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json +0 -20
  40. dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +0 -4232
  41. dbt_platform_helper/templates/pipelines/codebase/overrides/package.json +0 -27
  42. dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts +0 -521
  43. dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json +0 -30
  44. dbt_platform_helper/templates/pipelines/codebase/overrides/types.ts +0 -52
  45. {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/LICENSE +0 -0
  46. {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/WHEEL +0 -0
  47. {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/entry_points.txt +0 -0
@@ -3,22 +3,22 @@ from collections.abc import Callable
3
3
  from pathlib import Path
4
4
 
5
5
  import boto3
6
- import click
7
6
  from boto3 import Session
8
7
 
9
8
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
10
9
  from dbt_platform_helper.domain.config_validator import ConfigValidator
11
10
  from dbt_platform_helper.domain.maintenance_page import MaintenancePage
12
- from dbt_platform_helper.providers.aws import AWSException
13
11
  from dbt_platform_helper.providers.config import ConfigProvider
12
+ from dbt_platform_helper.providers.io import ClickIOProvider
13
+ from dbt_platform_helper.providers.io import ClickIOProviderException
14
14
  from dbt_platform_helper.providers.vpc import Vpc
15
15
  from dbt_platform_helper.providers.vpc import VpcProvider
16
+ from dbt_platform_helper.providers.vpc import VpcProviderException
16
17
  from dbt_platform_helper.utils.application import Application
17
18
  from dbt_platform_helper.utils.application import ApplicationNotFoundException
18
19
  from dbt_platform_helper.utils.application import load_application
19
20
  from dbt_platform_helper.utils.aws import get_connection_string
20
21
  from dbt_platform_helper.utils.aws import wait_for_log_group_to_exist
21
- from dbt_platform_helper.utils.messages import abort_with_error
22
22
 
23
23
 
24
24
  class DatabaseCopy:
@@ -33,12 +33,8 @@ class DatabaseCopy:
33
33
  db_connection_string: Callable[
34
34
  [Session, str, str, str, Callable], str
35
35
  ] = get_connection_string,
36
- maintenance_page_provider: Callable[
37
- [str, str, list[str], str, str], None
38
- ] = MaintenancePage(),
39
- input: Callable[[str], str] = click.prompt,
40
- echo: Callable[[str], str] = click.secho,
41
- abort: Callable[[str], None] = abort_with_error,
36
+ maintenance_page: Callable[[str, str, list[str], str, str], None] = MaintenancePage,
37
+ io: ClickIOProvider = ClickIOProvider(),
42
38
  config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
43
39
  ):
44
40
  self.app = app
@@ -46,15 +42,14 @@ class DatabaseCopy:
46
42
  self.auto_approve = auto_approve
47
43
  self.vpc_provider = vpc_provider
48
44
  self.db_connection_string = db_connection_string
49
- self.maintenance_page_provider = maintenance_page_provider
50
- self.input = input
51
- self.echo = echo
52
- self.abort = abort
45
+ self.io = io
53
46
  self.config_provider = config_provider
54
47
 
55
48
  if not self.app:
56
49
  if not Path(PLATFORM_CONFIG_FILE).exists():
57
- self.abort("You must either be in a deploy repo, or provide the --app option.")
50
+ self.io.abort_with_error(
51
+ "You must either be in a deploy repo, or provide the --app option."
52
+ )
58
53
 
59
54
  config = self.config_provider.load_and_validate_platform_config()
60
55
  self.app = config["application"]
@@ -62,7 +57,9 @@ class DatabaseCopy:
62
57
  try:
63
58
  self.application = load_application(self.app)
64
59
  except ApplicationNotFoundException:
65
- abort(f"No such application '{app}'.")
60
+ self.io.abort_with_error(f"No such application '{app}'.")
61
+
62
+ self.maintenance_page = maintenance_page(self.application)
66
63
 
67
64
  def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, filename: str):
68
65
  vpc_name = self.enrich_vpc_name(env, vpc_name)
@@ -70,7 +67,7 @@ class DatabaseCopy:
70
67
  environments = self.application.environments
71
68
  environment = environments.get(env)
72
69
  if not environment:
73
- self.abort(
70
+ self.io.abort_with_error(
74
71
  f"No such environment '{env}'. Available environments are: {', '.join(environments.keys())}"
75
72
  )
76
73
 
@@ -78,9 +75,9 @@ class DatabaseCopy:
78
75
 
79
76
  try:
80
77
  vpc_provider = self.vpc_provider(env_session)
81
- vpc_config = vpc_provider.get_vpc_info_by_name(self.app, env, vpc_name)
82
- except AWSException as ex:
83
- self.abort(str(ex))
78
+ vpc_config = vpc_provider.get_vpc(self.app, env, vpc_name)
79
+ except VpcProviderException as ex:
80
+ self.io.abort_with_error(str(ex))
84
81
 
85
82
  database_identifier = f"{self.app}-{env}-{self.database}"
86
83
 
@@ -89,31 +86,32 @@ class DatabaseCopy:
89
86
  env_session, self.app, env, database_identifier
90
87
  )
91
88
  except Exception as exc:
92
- self.abort(f"{exc} (Database: {database_identifier})")
89
+ self.io.abort_with_error(f"{exc} (Database: {database_identifier})")
93
90
 
94
91
  try:
95
92
  task_arn = self.run_database_copy_task(
96
93
  env_session, env, vpc_config, is_dump, db_connection_string, filename
97
94
  )
98
95
  except Exception as exc:
99
- self.abort(f"{exc} (Account id: {self.account_id(env)})")
96
+ self.io.abort_with_error(f"{exc} (Account id: {self.account_id(env)})")
100
97
 
101
98
  if is_dump:
102
99
  message = f"Dumping {self.database} from the {env} environment into S3"
103
100
  else:
104
101
  message = f"Loading data into {self.database} in the {env} environment from S3"
105
102
 
106
- self.echo(message, fg="white", bold=True)
107
- self.echo(
108
- f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
109
- fg="white",
103
+ self.io.info(message)
104
+ self.io.info(
105
+ f"Task {task_arn} started. Waiting for it to complete (this may take some time)..."
110
106
  )
111
107
  self.tail_logs(is_dump, env)
112
108
 
113
109
  def enrich_vpc_name(self, env, vpc_name):
114
110
  if not vpc_name:
115
111
  if not Path(PLATFORM_CONFIG_FILE).exists():
116
- self.abort("You must either be in a deploy repo, or provide the vpc name option.")
112
+ self.io.abort_with_error(
113
+ "You must either be in a deploy repo, or provide the vpc name option."
114
+ )
117
115
  config = self.config_provider.load_and_validate_platform_config()
118
116
  env_config = self.config_provider.apply_environment_defaults(config)["environments"]
119
117
  vpc_name = env_config.get(env, {}).get("vpc")
@@ -147,7 +145,7 @@ class DatabaseCopy:
147
145
  ],
148
146
  networkConfiguration={
149
147
  "awsvpcConfiguration": {
150
- "subnets": vpc_config.subnets,
148
+ "subnets": vpc_config.private_subnets,
151
149
  "securityGroups": vpc_config.security_groups,
152
150
  "assignPublicIp": "DISABLED",
153
151
  }
@@ -183,26 +181,27 @@ class DatabaseCopy:
183
181
  ):
184
182
  to_vpc = self.enrich_vpc_name(to_env, to_vpc)
185
183
  if not no_maintenance_page:
186
- self.maintenance_page_provider.activate(self.app, to_env, services, template, to_vpc)
184
+ self.maintenance_page.activate(to_env, services, template, to_vpc)
187
185
  self.dump(from_env, from_vpc, f"data_dump_{to_env}")
188
186
  self.load(to_env, to_vpc, f"data_dump_{to_env}")
189
187
  if not no_maintenance_page:
190
- self.maintenance_page_provider.deactivate(self.app, to_env)
188
+ self.maintenance_page.deactivate(to_env)
191
189
 
192
190
  def is_confirmed_ready_to_load(self, env: str) -> bool:
193
191
  if self.auto_approve:
194
192
  return True
195
-
196
- user_input = self.input(
197
- f"\nWARNING: the load operation is destructive and will delete the {self.database} database in the {env} environment. Continue? (y/n)"
198
- )
199
- return user_input.lower().strip() in ["y", "yes"]
193
+ try:
194
+ return self.io.confirm(
195
+ f"\nWARNING: the load operation is destructive and will delete the {self.database} database in the {env} environment. Continue?"
196
+ )
197
+ except ClickIOProviderException:
198
+ return False
200
199
 
201
200
  def tail_logs(self, is_dump: bool, env: str):
202
201
  action = "dump" if is_dump else "load"
203
202
  log_group_name = f"/ecs/{self.app}-{env}-{self.database}-{action}"
204
203
  log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id(env)}:log-group:{log_group_name}"
205
- self.echo(f"Tailing {log_group_name} logs", fg="yellow")
204
+ self.io.warn(f"Tailing {log_group_name} logs")
206
205
  session = self.application.environments[env].session
207
206
  log_client = session.client("logs")
208
207
  wait_for_log_group_to_exist(log_client, log_group_name)
@@ -220,9 +219,11 @@ class DatabaseCopy:
220
219
  match = re.match(r"(Stopping|Aborting) data (load|dump).*", message)
221
220
  if match:
222
221
  if match.group(1) == "Aborting":
223
- self.abort("Task aborted abnormally. See logs above for details.")
222
+ self.io.abort_with_error(
223
+ "Task aborted abnormally. See logs above for details."
224
+ )
224
225
  stopped = True
225
- self.echo(message)
226
+ self.io.info(message)
226
227
 
227
228
  def account_id(self, env):
228
229
  envs = self.application.environments