dbt-platform-helper 12.5.1__py3-none-any.whl → 13.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +45 -42
- dbt_platform_helper/commands/codebase.py +7 -10
- dbt_platform_helper/commands/conduit.py +2 -2
- dbt_platform_helper/commands/config.py +1 -1
- dbt_platform_helper/commands/environment.py +32 -18
- dbt_platform_helper/commands/notify.py +5 -3
- dbt_platform_helper/commands/pipeline.py +17 -11
- dbt_platform_helper/constants.py +3 -1
- dbt_platform_helper/domain/codebase.py +48 -36
- dbt_platform_helper/domain/conduit.py +10 -12
- dbt_platform_helper/domain/config_validator.py +42 -31
- dbt_platform_helper/domain/copilot_environment.py +133 -129
- dbt_platform_helper/domain/database_copy.py +38 -37
- dbt_platform_helper/domain/maintenance_page.py +243 -193
- dbt_platform_helper/domain/pipelines.py +60 -135
- dbt_platform_helper/domain/terraform_environment.py +7 -3
- dbt_platform_helper/providers/aws.py +5 -0
- dbt_platform_helper/providers/cloudformation.py +12 -1
- dbt_platform_helper/providers/config.py +12 -14
- dbt_platform_helper/providers/ecr.py +20 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +31 -0
- dbt_platform_helper/providers/load_balancers.py +29 -3
- dbt_platform_helper/providers/platform_config_schema.py +24 -22
- dbt_platform_helper/providers/terraform_manifest.py +120 -0
- dbt_platform_helper/providers/vpc.py +81 -32
- dbt_platform_helper/templates/COMMANDS.md.jinja +5 -3
- dbt_platform_helper/templates/environment-pipelines/main.tf +2 -2
- dbt_platform_helper/templates/environments/main.tf +3 -4
- dbt_platform_helper/utils/aws.py +16 -5
- dbt_platform_helper/utils/messages.py +2 -3
- {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/METADATA +2 -2
- {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/RECORD +36 -44
- dbt_platform_helper/templates/pipelines/codebase/manifest.yml +0 -56
- dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore +0 -12
- dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts +0 -8
- dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml +0 -29
- dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml +0 -48
- dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json +0 -20
- dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +0 -4232
- dbt_platform_helper/templates/pipelines/codebase/overrides/package.json +0 -27
- dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts +0 -521
- dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json +0 -30
- dbt_platform_helper/templates/pipelines/codebase/overrides/types.ts +0 -52
- {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-12.5.1.dist-info → dbt_platform_helper-13.0.0.dist-info}/entry_points.txt +0 -0
|
@@ -3,22 +3,22 @@ from collections.abc import Callable
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
import boto3
|
|
6
|
-
import click
|
|
7
6
|
from boto3 import Session
|
|
8
7
|
|
|
9
8
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
10
9
|
from dbt_platform_helper.domain.config_validator import ConfigValidator
|
|
11
10
|
from dbt_platform_helper.domain.maintenance_page import MaintenancePage
|
|
12
|
-
from dbt_platform_helper.providers.aws import AWSException
|
|
13
11
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
12
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
13
|
+
from dbt_platform_helper.providers.io import ClickIOProviderException
|
|
14
14
|
from dbt_platform_helper.providers.vpc import Vpc
|
|
15
15
|
from dbt_platform_helper.providers.vpc import VpcProvider
|
|
16
|
+
from dbt_platform_helper.providers.vpc import VpcProviderException
|
|
16
17
|
from dbt_platform_helper.utils.application import Application
|
|
17
18
|
from dbt_platform_helper.utils.application import ApplicationNotFoundException
|
|
18
19
|
from dbt_platform_helper.utils.application import load_application
|
|
19
20
|
from dbt_platform_helper.utils.aws import get_connection_string
|
|
20
21
|
from dbt_platform_helper.utils.aws import wait_for_log_group_to_exist
|
|
21
|
-
from dbt_platform_helper.utils.messages import abort_with_error
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class DatabaseCopy:
|
|
@@ -33,12 +33,8 @@ class DatabaseCopy:
|
|
|
33
33
|
db_connection_string: Callable[
|
|
34
34
|
[Session, str, str, str, Callable], str
|
|
35
35
|
] = get_connection_string,
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
] = MaintenancePage(),
|
|
39
|
-
input: Callable[[str], str] = click.prompt,
|
|
40
|
-
echo: Callable[[str], str] = click.secho,
|
|
41
|
-
abort: Callable[[str], None] = abort_with_error,
|
|
36
|
+
maintenance_page: Callable[[str, str, list[str], str, str], None] = MaintenancePage,
|
|
37
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
42
38
|
config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
|
|
43
39
|
):
|
|
44
40
|
self.app = app
|
|
@@ -46,15 +42,14 @@ class DatabaseCopy:
|
|
|
46
42
|
self.auto_approve = auto_approve
|
|
47
43
|
self.vpc_provider = vpc_provider
|
|
48
44
|
self.db_connection_string = db_connection_string
|
|
49
|
-
self.
|
|
50
|
-
self.input = input
|
|
51
|
-
self.echo = echo
|
|
52
|
-
self.abort = abort
|
|
45
|
+
self.io = io
|
|
53
46
|
self.config_provider = config_provider
|
|
54
47
|
|
|
55
48
|
if not self.app:
|
|
56
49
|
if not Path(PLATFORM_CONFIG_FILE).exists():
|
|
57
|
-
self.
|
|
50
|
+
self.io.abort_with_error(
|
|
51
|
+
"You must either be in a deploy repo, or provide the --app option."
|
|
52
|
+
)
|
|
58
53
|
|
|
59
54
|
config = self.config_provider.load_and_validate_platform_config()
|
|
60
55
|
self.app = config["application"]
|
|
@@ -62,7 +57,9 @@ class DatabaseCopy:
|
|
|
62
57
|
try:
|
|
63
58
|
self.application = load_application(self.app)
|
|
64
59
|
except ApplicationNotFoundException:
|
|
65
|
-
|
|
60
|
+
self.io.abort_with_error(f"No such application '{app}'.")
|
|
61
|
+
|
|
62
|
+
self.maintenance_page = maintenance_page(self.application)
|
|
66
63
|
|
|
67
64
|
def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, filename: str):
|
|
68
65
|
vpc_name = self.enrich_vpc_name(env, vpc_name)
|
|
@@ -70,7 +67,7 @@ class DatabaseCopy:
|
|
|
70
67
|
environments = self.application.environments
|
|
71
68
|
environment = environments.get(env)
|
|
72
69
|
if not environment:
|
|
73
|
-
self.
|
|
70
|
+
self.io.abort_with_error(
|
|
74
71
|
f"No such environment '{env}'. Available environments are: {', '.join(environments.keys())}"
|
|
75
72
|
)
|
|
76
73
|
|
|
@@ -78,9 +75,9 @@ class DatabaseCopy:
|
|
|
78
75
|
|
|
79
76
|
try:
|
|
80
77
|
vpc_provider = self.vpc_provider(env_session)
|
|
81
|
-
vpc_config = vpc_provider.
|
|
82
|
-
except
|
|
83
|
-
self.
|
|
78
|
+
vpc_config = vpc_provider.get_vpc(self.app, env, vpc_name)
|
|
79
|
+
except VpcProviderException as ex:
|
|
80
|
+
self.io.abort_with_error(str(ex))
|
|
84
81
|
|
|
85
82
|
database_identifier = f"{self.app}-{env}-{self.database}"
|
|
86
83
|
|
|
@@ -89,31 +86,32 @@ class DatabaseCopy:
|
|
|
89
86
|
env_session, self.app, env, database_identifier
|
|
90
87
|
)
|
|
91
88
|
except Exception as exc:
|
|
92
|
-
self.
|
|
89
|
+
self.io.abort_with_error(f"{exc} (Database: {database_identifier})")
|
|
93
90
|
|
|
94
91
|
try:
|
|
95
92
|
task_arn = self.run_database_copy_task(
|
|
96
93
|
env_session, env, vpc_config, is_dump, db_connection_string, filename
|
|
97
94
|
)
|
|
98
95
|
except Exception as exc:
|
|
99
|
-
self.
|
|
96
|
+
self.io.abort_with_error(f"{exc} (Account id: {self.account_id(env)})")
|
|
100
97
|
|
|
101
98
|
if is_dump:
|
|
102
99
|
message = f"Dumping {self.database} from the {env} environment into S3"
|
|
103
100
|
else:
|
|
104
101
|
message = f"Loading data into {self.database} in the {env} environment from S3"
|
|
105
102
|
|
|
106
|
-
self.
|
|
107
|
-
self.
|
|
108
|
-
f"Task {task_arn} started. Waiting for it to complete (this may take some time)..."
|
|
109
|
-
fg="white",
|
|
103
|
+
self.io.info(message)
|
|
104
|
+
self.io.info(
|
|
105
|
+
f"Task {task_arn} started. Waiting for it to complete (this may take some time)..."
|
|
110
106
|
)
|
|
111
107
|
self.tail_logs(is_dump, env)
|
|
112
108
|
|
|
113
109
|
def enrich_vpc_name(self, env, vpc_name):
|
|
114
110
|
if not vpc_name:
|
|
115
111
|
if not Path(PLATFORM_CONFIG_FILE).exists():
|
|
116
|
-
self.
|
|
112
|
+
self.io.abort_with_error(
|
|
113
|
+
"You must either be in a deploy repo, or provide the vpc name option."
|
|
114
|
+
)
|
|
117
115
|
config = self.config_provider.load_and_validate_platform_config()
|
|
118
116
|
env_config = self.config_provider.apply_environment_defaults(config)["environments"]
|
|
119
117
|
vpc_name = env_config.get(env, {}).get("vpc")
|
|
@@ -147,7 +145,7 @@ class DatabaseCopy:
|
|
|
147
145
|
],
|
|
148
146
|
networkConfiguration={
|
|
149
147
|
"awsvpcConfiguration": {
|
|
150
|
-
"subnets": vpc_config.
|
|
148
|
+
"subnets": vpc_config.private_subnets,
|
|
151
149
|
"securityGroups": vpc_config.security_groups,
|
|
152
150
|
"assignPublicIp": "DISABLED",
|
|
153
151
|
}
|
|
@@ -183,26 +181,27 @@ class DatabaseCopy:
|
|
|
183
181
|
):
|
|
184
182
|
to_vpc = self.enrich_vpc_name(to_env, to_vpc)
|
|
185
183
|
if not no_maintenance_page:
|
|
186
|
-
self.
|
|
184
|
+
self.maintenance_page.activate(to_env, services, template, to_vpc)
|
|
187
185
|
self.dump(from_env, from_vpc, f"data_dump_{to_env}")
|
|
188
186
|
self.load(to_env, to_vpc, f"data_dump_{to_env}")
|
|
189
187
|
if not no_maintenance_page:
|
|
190
|
-
self.
|
|
188
|
+
self.maintenance_page.deactivate(to_env)
|
|
191
189
|
|
|
192
190
|
def is_confirmed_ready_to_load(self, env: str) -> bool:
|
|
193
191
|
if self.auto_approve:
|
|
194
192
|
return True
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
193
|
+
try:
|
|
194
|
+
return self.io.confirm(
|
|
195
|
+
f"\nWARNING: the load operation is destructive and will delete the {self.database} database in the {env} environment. Continue?"
|
|
196
|
+
)
|
|
197
|
+
except ClickIOProviderException:
|
|
198
|
+
return False
|
|
200
199
|
|
|
201
200
|
def tail_logs(self, is_dump: bool, env: str):
|
|
202
201
|
action = "dump" if is_dump else "load"
|
|
203
202
|
log_group_name = f"/ecs/{self.app}-{env}-{self.database}-{action}"
|
|
204
203
|
log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id(env)}:log-group:{log_group_name}"
|
|
205
|
-
self.
|
|
204
|
+
self.io.warn(f"Tailing {log_group_name} logs")
|
|
206
205
|
session = self.application.environments[env].session
|
|
207
206
|
log_client = session.client("logs")
|
|
208
207
|
wait_for_log_group_to_exist(log_client, log_group_name)
|
|
@@ -220,9 +219,11 @@ class DatabaseCopy:
|
|
|
220
219
|
match = re.match(r"(Stopping|Aborting) data (load|dump).*", message)
|
|
221
220
|
if match:
|
|
222
221
|
if match.group(1) == "Aborting":
|
|
223
|
-
self.
|
|
222
|
+
self.io.abort_with_error(
|
|
223
|
+
"Task aborted abnormally. See logs above for details."
|
|
224
|
+
)
|
|
224
225
|
stopped = True
|
|
225
|
-
self.
|
|
226
|
+
self.io.info(message)
|
|
226
227
|
|
|
227
228
|
def account_id(self, env):
|
|
228
229
|
envs = self.application.environments
|