dbt-platform-helper 12.0.2__py3-none-any.whl → 12.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +2 -4
- dbt_platform_helper/commands/codebase.py +62 -228
- dbt_platform_helper/constants.py +1 -0
- dbt_platform_helper/domain/codebase.py +222 -0
- dbt_platform_helper/domain/database_copy.py +1 -1
- dbt_platform_helper/exceptions.py +28 -0
- dbt_platform_helper/utils/application.py +1 -4
- dbt_platform_helper/utils/aws.py +132 -0
- dbt_platform_helper/utils/files.py +70 -0
- dbt_platform_helper/utils/git.py +13 -0
- dbt_platform_helper/utils/validation.py +99 -2
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/METADATA +1 -1
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/RECORD +16 -24
- dbt_platform_helper/templates/env/overrides/.gitignore +0 -12
- dbt_platform_helper/templates/env/overrides/README.md +0 -11
- dbt_platform_helper/templates/env/overrides/bin/override.ts +0 -9
- dbt_platform_helper/templates/env/overrides/cdk.json +0 -20
- dbt_platform_helper/templates/env/overrides/log_resource_policy.json +0 -68
- dbt_platform_helper/templates/env/overrides/package-lock.json +0 -4307
- dbt_platform_helper/templates/env/overrides/package.json +0 -27
- dbt_platform_helper/templates/env/overrides/stack.ts +0 -51
- dbt_platform_helper/templates/env/overrides/tsconfig.json +0 -32
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/entry_points.txt +0 -0
dbt_platform_helper/COMMANDS.md
CHANGED
|
@@ -222,7 +222,7 @@ platform-helper codebase build --app <application> --codebase <codebase>
|
|
|
222
222
|
- `--app <text>`
|
|
223
223
|
- AWS application name
|
|
224
224
|
- `--codebase <text>`
|
|
225
|
-
- The codebase name as specified in the
|
|
225
|
+
- The codebase name as specified in the platform-config.yml file
|
|
226
226
|
- `--commit <text>`
|
|
227
227
|
- GitHub commit hash
|
|
228
228
|
- `--help <boolean>` _Defaults to False._
|
|
@@ -232,8 +232,6 @@ platform-helper codebase build --app <application> --codebase <codebase>
|
|
|
232
232
|
|
|
233
233
|
[↩ Parent](#platform-helper-codebase)
|
|
234
234
|
|
|
235
|
-
Trigger a CodePipeline pipeline based deployment.
|
|
236
|
-
|
|
237
235
|
## Usage
|
|
238
236
|
|
|
239
237
|
```
|
|
@@ -248,7 +246,7 @@ platform-helper codebase deploy --app <application> --env <environment> --codeba
|
|
|
248
246
|
- `--env <text>`
|
|
249
247
|
- AWS Copilot environment
|
|
250
248
|
- `--codebase <text>`
|
|
251
|
-
- The codebase name as specified in the
|
|
249
|
+
- The codebase name as specified in the platform-config.yml file
|
|
252
250
|
- `--commit <text>`
|
|
253
251
|
- GitHub commit hash
|
|
254
252
|
- `--help <boolean>` _Defaults to False._
|
|
@@ -1,21 +1,18 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
|
-
import stat
|
|
4
|
-
import subprocess
|
|
5
|
-
from pathlib import Path
|
|
6
3
|
|
|
7
4
|
import click
|
|
8
|
-
import requests
|
|
9
|
-
import yaml
|
|
10
|
-
from boto3 import Session
|
|
11
5
|
|
|
12
|
-
from dbt_platform_helper.
|
|
13
|
-
from dbt_platform_helper.
|
|
14
|
-
from dbt_platform_helper.
|
|
15
|
-
from dbt_platform_helper.
|
|
6
|
+
from dbt_platform_helper.domain.codebase import Codebase
|
|
7
|
+
from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
|
|
8
|
+
from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
|
|
9
|
+
from dbt_platform_helper.exceptions import ApplicationNotFoundError
|
|
10
|
+
from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
|
|
11
|
+
from dbt_platform_helper.exceptions import ImageNotFoundError
|
|
12
|
+
from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
|
|
13
|
+
from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
|
|
16
14
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
17
|
-
from dbt_platform_helper.utils.
|
|
18
|
-
from dbt_platform_helper.utils.template import setup_templates
|
|
15
|
+
from dbt_platform_helper.utils.git import CommitNotFoundError
|
|
19
16
|
from dbt_platform_helper.utils.versioning import (
|
|
20
17
|
check_platform_helper_version_needs_update,
|
|
21
18
|
)
|
|
@@ -30,92 +27,15 @@ def codebase():
|
|
|
30
27
|
@codebase.command()
|
|
31
28
|
def prepare():
|
|
32
29
|
"""Sets up an application codebase for use within a DBT platform project."""
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
.stdout.split("/")[-1]
|
|
38
|
-
.strip()
|
|
39
|
-
.removesuffix(".git")
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
if repository.endswith("-deploy") or Path("./copilot").exists():
|
|
30
|
+
try:
|
|
31
|
+
Codebase().prepare()
|
|
32
|
+
except NotInCodeBaseRepositoryError:
|
|
33
|
+
# TODO print error attached to exception
|
|
43
34
|
click.secho(
|
|
44
35
|
"You are in the deploy repository; make sure you are in the application codebase repository.",
|
|
45
36
|
fg="red",
|
|
46
37
|
)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
|
|
50
|
-
builder_configuration_response = requests.get(builder_configuration_url)
|
|
51
|
-
builder_configuration_content = yaml.safe_load(
|
|
52
|
-
builder_configuration_response.content.decode("utf-8")
|
|
53
|
-
)
|
|
54
|
-
builder_versions = next(
|
|
55
|
-
(
|
|
56
|
-
item
|
|
57
|
-
for item in builder_configuration_content["builders"]
|
|
58
|
-
if item["name"] == "paketobuildpacks/builder-jammy-base"
|
|
59
|
-
),
|
|
60
|
-
None,
|
|
61
|
-
)
|
|
62
|
-
builder_version = max(x["version"] for x in builder_versions["versions"])
|
|
63
|
-
# Temporary hack until https://uktrade.atlassian.net/browse/DBTP-351 is done
|
|
64
|
-
# Will need a change in tests/platform_helper/expected_files/.copilot/config.yml, when removed.
|
|
65
|
-
builder_version = min(builder_version, "0.4.240")
|
|
66
|
-
|
|
67
|
-
Path("./.copilot/phases").mkdir(parents=True, exist_ok=True)
|
|
68
|
-
image_build_run_contents = templates.get_template(f".copilot/image_build_run.sh").render()
|
|
69
|
-
|
|
70
|
-
config_contents = templates.get_template(f".copilot/config.yml").render(
|
|
71
|
-
repository=repository, builder_version=builder_version
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
click.echo(
|
|
75
|
-
mkfile(Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True)
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
image_build_run_file = Path(".copilot/image_build_run.sh")
|
|
79
|
-
image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
|
|
80
|
-
|
|
81
|
-
click.echo(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
|
|
82
|
-
|
|
83
|
-
for phase in ["build", "install", "post_build", "pre_build"]:
|
|
84
|
-
phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
|
|
85
|
-
|
|
86
|
-
click.echo(mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True))
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def list_latest_images(ecr_client, ecr_repository_name, codebase_repository):
|
|
90
|
-
paginator = ecr_client.get_paginator("describe_images")
|
|
91
|
-
describe_images_response_iterator = paginator.paginate(
|
|
92
|
-
repositoryName=ecr_repository_name,
|
|
93
|
-
filter={"tagStatus": "TAGGED"},
|
|
94
|
-
)
|
|
95
|
-
images = []
|
|
96
|
-
for page in describe_images_response_iterator:
|
|
97
|
-
images += page["imageDetails"]
|
|
98
|
-
|
|
99
|
-
sorted_images = sorted(
|
|
100
|
-
images,
|
|
101
|
-
key=lambda i: i["imagePushedAt"],
|
|
102
|
-
reverse=True,
|
|
103
|
-
)
|
|
104
|
-
|
|
105
|
-
MAX_RESULTS = 20
|
|
106
|
-
|
|
107
|
-
for image in sorted_images[:MAX_RESULTS]:
|
|
108
|
-
try:
|
|
109
|
-
commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
|
|
110
|
-
if not commit_tag:
|
|
111
|
-
continue
|
|
112
|
-
|
|
113
|
-
commit_hash = commit_tag.replace("commit-", "")
|
|
114
|
-
click.echo(
|
|
115
|
-
f" - https://github.com/{codebase_repository}/commit/{commit_hash} - published: {image['imagePushedAt']}"
|
|
116
|
-
)
|
|
117
|
-
except StopIteration:
|
|
118
|
-
continue
|
|
38
|
+
raise click.Abort
|
|
119
39
|
|
|
120
40
|
|
|
121
41
|
@codebase.command()
|
|
@@ -128,184 +48,98 @@ def list_latest_images(ecr_client, ecr_repository_name, codebase_repository):
|
|
|
128
48
|
)
|
|
129
49
|
def list(app, with_images):
|
|
130
50
|
"""List available codebases for the application."""
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
51
|
+
try:
|
|
52
|
+
Codebase().list(app, with_images)
|
|
53
|
+
except NoCopilotCodebasesFoundError:
|
|
54
|
+
click.secho(
|
|
55
|
+
f"""No codebases found for application "{app}""",
|
|
56
|
+
fg="red",
|
|
57
|
+
)
|
|
58
|
+
raise click.Abort
|
|
59
|
+
except ApplicationNotFoundError:
|
|
60
|
+
click.secho(
|
|
61
|
+
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
62
|
+
fg="red",
|
|
63
|
+
)
|
|
144
64
|
raise click.Abort
|
|
145
|
-
|
|
146
|
-
click.echo("The following codebases are available:")
|
|
147
|
-
|
|
148
|
-
for codebase in codebases:
|
|
149
|
-
click.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
|
|
150
|
-
if with_images:
|
|
151
|
-
list_latest_images(
|
|
152
|
-
ecr_client, f"{application.name}/{codebase['name']}", codebase["repository"]
|
|
153
|
-
)
|
|
154
|
-
|
|
155
|
-
click.echo("")
|
|
156
65
|
|
|
157
66
|
|
|
158
67
|
@codebase.command()
|
|
159
68
|
@click.option("--app", help="AWS application name", required=True)
|
|
160
69
|
@click.option(
|
|
161
|
-
"--codebase",
|
|
70
|
+
"--codebase",
|
|
71
|
+
help="The codebase name as specified in the platform-config.yml file",
|
|
72
|
+
required=True,
|
|
162
73
|
)
|
|
163
74
|
@click.option("--commit", help="GitHub commit hash", required=True)
|
|
164
75
|
def build(app, codebase, commit):
|
|
165
76
|
"""Trigger a CodePipeline pipeline based build."""
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
check_if_commit_exists = subprocess.run(
|
|
170
|
-
["git", "branch", "-r", "--contains", f"{commit}"], capture_output=True, text=True
|
|
171
|
-
)
|
|
172
|
-
|
|
173
|
-
if check_if_commit_exists.stderr:
|
|
77
|
+
try:
|
|
78
|
+
Codebase().build(app, codebase, commit)
|
|
79
|
+
except ApplicationNotFoundError:
|
|
174
80
|
click.secho(
|
|
175
|
-
f"""The
|
|
81
|
+
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
176
82
|
fg="red",
|
|
177
83
|
)
|
|
178
84
|
raise click.Abort
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
|
|
184
|
-
{
|
|
185
|
-
"projectName": f"codebuild-{app}-{codebase}",
|
|
186
|
-
"artifactsOverride": {"type": "NO_ARTIFACTS"},
|
|
187
|
-
"sourceVersion": commit,
|
|
188
|
-
},
|
|
189
|
-
)
|
|
190
|
-
|
|
191
|
-
if build_url:
|
|
192
|
-
return click.echo(
|
|
193
|
-
"Your build has been triggered. Check your build progress in the AWS Console: "
|
|
194
|
-
f"{build_url}",
|
|
85
|
+
except CommitNotFoundError:
|
|
86
|
+
click.secho(
|
|
87
|
+
f'The commit hash "{commit}" either does not exist or you need to run `git fetch`.',
|
|
88
|
+
fg="red",
|
|
195
89
|
)
|
|
196
|
-
|
|
197
|
-
|
|
90
|
+
raise click.Abort
|
|
91
|
+
except ApplicationDeploymentNotTriggered:
|
|
92
|
+
click.secho(
|
|
93
|
+
f"Your build for {codebase} was not triggered.",
|
|
94
|
+
fg="red",
|
|
95
|
+
)
|
|
96
|
+
raise click.Abort
|
|
198
97
|
|
|
199
98
|
|
|
200
99
|
@codebase.command()
|
|
201
100
|
@click.option("--app", help="AWS application name", required=True)
|
|
202
101
|
@click.option("--env", help="AWS Copilot environment", required=True)
|
|
203
102
|
@click.option(
|
|
204
|
-
"--codebase",
|
|
103
|
+
"--codebase",
|
|
104
|
+
help="The codebase name as specified in the platform-config.yml file",
|
|
105
|
+
required=True,
|
|
205
106
|
)
|
|
206
107
|
@click.option("--commit", help="GitHub commit hash", required=True)
|
|
207
108
|
def deploy(app, env, codebase, commit):
|
|
208
|
-
"""Trigger a CodePipeline pipeline based deployment."""
|
|
209
|
-
session = get_aws_session_or_abort()
|
|
210
|
-
application = load_application_with_environment(session, app, env)
|
|
211
|
-
check_codebase_exists(session, application, codebase)
|
|
212
|
-
check_image_exists(session, application, codebase, commit)
|
|
213
|
-
|
|
214
|
-
codebuild_client = session.client("codebuild")
|
|
215
|
-
build_url = start_build_with_confirmation(
|
|
216
|
-
codebuild_client,
|
|
217
|
-
f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
|
|
218
|
-
{
|
|
219
|
-
"projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
|
|
220
|
-
"artifactsOverride": {"type": "NO_ARTIFACTS"},
|
|
221
|
-
"sourceTypeOverride": "NO_SOURCE",
|
|
222
|
-
"environmentVariablesOverride": [
|
|
223
|
-
{"name": "COPILOT_ENVIRONMENT", "value": env},
|
|
224
|
-
{"name": "IMAGE_TAG", "value": f"commit-{commit}"},
|
|
225
|
-
],
|
|
226
|
-
},
|
|
227
|
-
)
|
|
228
|
-
|
|
229
|
-
if build_url:
|
|
230
|
-
return click.echo(
|
|
231
|
-
"Your deployment has been triggered. Check your build progress in the AWS Console: "
|
|
232
|
-
f"{build_url}",
|
|
233
|
-
)
|
|
234
|
-
|
|
235
|
-
return click.echo("Your deployment was not triggered.")
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
def load_application_or_abort(session: Session, app: str) -> Application:
|
|
239
109
|
try:
|
|
240
|
-
|
|
110
|
+
Codebase().deploy(app, env, codebase, commit)
|
|
241
111
|
except ApplicationNotFoundError:
|
|
242
112
|
click.secho(
|
|
243
113
|
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
244
114
|
fg="red",
|
|
245
115
|
)
|
|
246
116
|
raise click.Abort
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def check_image_exists(session: Session, application: Application, codebase: str, commit: str):
|
|
250
|
-
ecr_client = session.client("ecr")
|
|
251
|
-
try:
|
|
252
|
-
ecr_client.describe_images(
|
|
253
|
-
repositoryName=f"{application.name}/{codebase}",
|
|
254
|
-
imageIds=[{"imageTag": f"commit-{commit}"}],
|
|
255
|
-
)
|
|
256
|
-
except ecr_client.exceptions.RepositoryNotFoundException:
|
|
117
|
+
except ApplicationEnvironmentNotFoundError:
|
|
257
118
|
click.secho(
|
|
258
|
-
f
|
|
119
|
+
f"""The environment "{env}" either does not exist or has not been deployed.""",
|
|
259
120
|
fg="red",
|
|
260
121
|
)
|
|
261
122
|
raise click.Abort
|
|
262
|
-
|
|
123
|
+
# TODO: don't hide json decode error
|
|
124
|
+
except (
|
|
125
|
+
CopilotCodebaseNotFoundError,
|
|
126
|
+
json.JSONDecodeError,
|
|
127
|
+
):
|
|
263
128
|
click.secho(
|
|
264
|
-
f
|
|
265
|
-
"`platform-helper codebase build` command first.",
|
|
129
|
+
f"""The codebase "{codebase}" either does not exist or has not been deployed.""",
|
|
266
130
|
fg="red",
|
|
267
131
|
)
|
|
268
132
|
raise click.Abort
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
def check_codebase_exists(session: Session, application: Application, codebase: str):
|
|
272
|
-
ssm_client = session.client("ssm")
|
|
273
|
-
try:
|
|
274
|
-
parameter = ssm_client.get_parameter(
|
|
275
|
-
Name=f"/copilot/applications/{application.name}/codebases/{codebase}"
|
|
276
|
-
)
|
|
277
|
-
value = parameter["Parameter"]["Value"]
|
|
278
|
-
json.loads(value)
|
|
279
|
-
except (KeyError, ValueError, json.JSONDecodeError, ssm_client.exceptions.ParameterNotFound):
|
|
133
|
+
except ImageNotFoundError:
|
|
280
134
|
click.secho(
|
|
281
|
-
f
|
|
135
|
+
f'The commit hash "{commit}" has not been built into an image, try the '
|
|
136
|
+
"`platform-helper codebase build` command first.",
|
|
282
137
|
fg="red",
|
|
283
138
|
)
|
|
284
139
|
raise click.Abort
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
def load_application_with_environment(session: Session, app, env):
|
|
288
|
-
application = load_application_or_abort(session, app)
|
|
289
|
-
|
|
290
|
-
if not application.environments.get(env):
|
|
140
|
+
except ApplicationDeploymentNotTriggered:
|
|
291
141
|
click.secho(
|
|
292
|
-
f"
|
|
142
|
+
f"Your deployment for {codebase} was not triggered.",
|
|
293
143
|
fg="red",
|
|
294
144
|
)
|
|
295
145
|
raise click.Abort
|
|
296
|
-
return application
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
def get_build_url_from_arn(build_arn: str) -> str:
|
|
300
|
-
_, _, _, region, account_id, project_name, build_id = build_arn.split(":")
|
|
301
|
-
project_name = project_name.removeprefix("build/")
|
|
302
|
-
return (
|
|
303
|
-
f"https://eu-west-2.console.aws.amazon.com/codesuite/codebuild/{account_id}/projects/"
|
|
304
|
-
f"{project_name}/build/{project_name}%3A{build_id}"
|
|
305
|
-
)
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
def start_build_with_confirmation(codebuild_client, confirmation_message, build_options):
|
|
309
|
-
if click.confirm(confirmation_message):
|
|
310
|
-
response = codebuild_client.start_build(**build_options)
|
|
311
|
-
return get_build_url_from_arn(response["build"]["arn"])
|
dbt_platform_helper/constants.py
CHANGED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import stat
|
|
3
|
+
import subprocess
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
import requests
|
|
9
|
+
import yaml
|
|
10
|
+
from boto3 import Session
|
|
11
|
+
|
|
12
|
+
from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
|
|
13
|
+
from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
|
|
14
|
+
from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
|
|
15
|
+
from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
|
|
16
|
+
from dbt_platform_helper.utils.application import Application
|
|
17
|
+
from dbt_platform_helper.utils.application import load_application
|
|
18
|
+
from dbt_platform_helper.utils.aws import check_codebase_exists
|
|
19
|
+
from dbt_platform_helper.utils.aws import check_image_exists
|
|
20
|
+
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
21
|
+
from dbt_platform_helper.utils.aws import get_build_url_from_arn
|
|
22
|
+
from dbt_platform_helper.utils.aws import list_latest_images
|
|
23
|
+
from dbt_platform_helper.utils.aws import start_build_extraction
|
|
24
|
+
from dbt_platform_helper.utils.files import mkfile
|
|
25
|
+
from dbt_platform_helper.utils.git import check_if_commit_exists
|
|
26
|
+
from dbt_platform_helper.utils.template import setup_templates
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Codebase:
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
input_fn: Callable[[str], str] = click.prompt,
|
|
33
|
+
echo_fn: Callable[[str], str] = click.secho,
|
|
34
|
+
confirm_fn: Callable[[str], bool] = click.confirm,
|
|
35
|
+
load_application_fn: Callable[[str], Application] = load_application,
|
|
36
|
+
get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
|
|
37
|
+
check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
|
|
38
|
+
check_image_exists_fn: Callable[[str], str] = check_image_exists,
|
|
39
|
+
get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
|
|
40
|
+
list_latest_images_fn: Callable[[str], str] = list_latest_images,
|
|
41
|
+
start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
|
|
42
|
+
check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
|
|
43
|
+
subprocess: Callable[[str], str] = subprocess.run,
|
|
44
|
+
):
|
|
45
|
+
self.input_fn = input_fn
|
|
46
|
+
self.echo_fn = echo_fn
|
|
47
|
+
self.confirm_fn = confirm_fn
|
|
48
|
+
self.load_application_fn = load_application_fn
|
|
49
|
+
self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
|
|
50
|
+
self.check_codebase_exists_fn = check_codebase_exists_fn
|
|
51
|
+
self.check_image_exists_fn = check_image_exists_fn
|
|
52
|
+
self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
|
|
53
|
+
self.list_latest_images_fn = list_latest_images_fn
|
|
54
|
+
self.start_build_extraction_fn = start_build_extraction_fn
|
|
55
|
+
self.check_if_commit_exists_fn = check_if_commit_exists_fn
|
|
56
|
+
self.subprocess = subprocess
|
|
57
|
+
|
|
58
|
+
def prepare(self):
|
|
59
|
+
"""Sets up an application codebase for use within a DBT platform
|
|
60
|
+
project."""
|
|
61
|
+
templates = setup_templates()
|
|
62
|
+
|
|
63
|
+
repository = (
|
|
64
|
+
self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
|
|
65
|
+
.stdout.split("/")[-1]
|
|
66
|
+
.strip()
|
|
67
|
+
.removesuffix(".git")
|
|
68
|
+
)
|
|
69
|
+
if repository.endswith("-deploy") or Path("./copilot").exists():
|
|
70
|
+
raise NotInCodeBaseRepositoryError
|
|
71
|
+
|
|
72
|
+
builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
|
|
73
|
+
builder_configuration_response = requests.get(builder_configuration_url)
|
|
74
|
+
builder_configuration_content = yaml.safe_load(
|
|
75
|
+
builder_configuration_response.content.decode("utf-8")
|
|
76
|
+
)
|
|
77
|
+
builder_versions = next(
|
|
78
|
+
(
|
|
79
|
+
item
|
|
80
|
+
for item in builder_configuration_content["builders"]
|
|
81
|
+
if item["name"] == "paketobuildpacks/builder-jammy-base"
|
|
82
|
+
),
|
|
83
|
+
None,
|
|
84
|
+
)
|
|
85
|
+
builder_version = max(x["version"] for x in builder_versions["versions"])
|
|
86
|
+
builder_version = min(builder_version, "0.4.240")
|
|
87
|
+
|
|
88
|
+
Path("./.copilot/phases").mkdir(parents=True, exist_ok=True)
|
|
89
|
+
image_build_run_contents = templates.get_template(f".copilot/image_build_run.sh").render()
|
|
90
|
+
|
|
91
|
+
config_contents = templates.get_template(f".copilot/config.yml").render(
|
|
92
|
+
repository=repository, builder_version=builder_version
|
|
93
|
+
)
|
|
94
|
+
self.echo_fn(
|
|
95
|
+
mkfile(
|
|
96
|
+
Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
image_build_run_file = Path(".copilot/image_build_run.sh")
|
|
101
|
+
image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
|
|
102
|
+
|
|
103
|
+
self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
|
|
104
|
+
|
|
105
|
+
for phase in ["build", "install", "post_build", "pre_build"]:
|
|
106
|
+
phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
|
|
107
|
+
|
|
108
|
+
self.echo_fn(
|
|
109
|
+
mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def build(self, app: str, codebase: str, commit: str):
|
|
113
|
+
"""Trigger a CodePipeline pipeline based build."""
|
|
114
|
+
session = self.get_aws_session_or_abort_fn()
|
|
115
|
+
self.load_application_fn(app, default_session=session)
|
|
116
|
+
|
|
117
|
+
self.check_if_commit_exists_fn(commit)
|
|
118
|
+
|
|
119
|
+
codebuild_client = session.client("codebuild")
|
|
120
|
+
build_url = self.__start_build_with_confirmation(
|
|
121
|
+
self.confirm_fn,
|
|
122
|
+
codebuild_client,
|
|
123
|
+
self.get_build_url_from_arn_fn,
|
|
124
|
+
f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
|
|
125
|
+
{
|
|
126
|
+
"projectName": f"codebuild-{app}-{codebase}",
|
|
127
|
+
"artifactsOverride": {"type": "NO_ARTIFACTS"},
|
|
128
|
+
"sourceVersion": commit,
|
|
129
|
+
},
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
if build_url:
|
|
133
|
+
return self.echo_fn(
|
|
134
|
+
f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
raise ApplicationDeploymentNotTriggered()
|
|
138
|
+
|
|
139
|
+
def deploy(self, app, env, codebase, commit):
|
|
140
|
+
"""Trigger a CodePipeline pipeline based deployment."""
|
|
141
|
+
session = self.get_aws_session_or_abort_fn()
|
|
142
|
+
|
|
143
|
+
application = self.load_application_fn(app, default_session=session)
|
|
144
|
+
if not application.environments.get(env):
|
|
145
|
+
raise ApplicationEnvironmentNotFoundError()
|
|
146
|
+
|
|
147
|
+
json.loads(self.check_codebase_exists_fn(session, application, codebase))
|
|
148
|
+
|
|
149
|
+
self.check_image_exists_fn(session, application, codebase, commit)
|
|
150
|
+
|
|
151
|
+
codebuild_client = session.client("codebuild")
|
|
152
|
+
build_url = self.__start_build_with_confirmation(
|
|
153
|
+
self.confirm_fn,
|
|
154
|
+
codebuild_client,
|
|
155
|
+
self.get_build_url_from_arn_fn,
|
|
156
|
+
f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
|
|
157
|
+
{
|
|
158
|
+
"projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
|
|
159
|
+
"artifactsOverride": {"type": "NO_ARTIFACTS"},
|
|
160
|
+
"sourceTypeOverride": "NO_SOURCE",
|
|
161
|
+
"environmentVariablesOverride": [
|
|
162
|
+
{"name": "COPILOT_ENVIRONMENT", "value": env},
|
|
163
|
+
{"name": "IMAGE_TAG", "value": f"commit-{commit}"},
|
|
164
|
+
],
|
|
165
|
+
},
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
if build_url:
|
|
169
|
+
return self.echo_fn(
|
|
170
|
+
"Your deployment has been triggered. Check your build progress in the AWS Console: "
|
|
171
|
+
f"{build_url}",
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
raise ApplicationDeploymentNotTriggered()
|
|
175
|
+
|
|
176
|
+
def list(self, app: str, with_images: bool):
|
|
177
|
+
"""List available codebases for the application."""
|
|
178
|
+
session = self.get_aws_session_or_abort_fn()
|
|
179
|
+
application = self.load_application_fn(app, session)
|
|
180
|
+
ssm_client = session.client("ssm")
|
|
181
|
+
ecr_client = session.client("ecr")
|
|
182
|
+
codebases = self.__get_codebases(application, ssm_client)
|
|
183
|
+
|
|
184
|
+
self.echo_fn("The following codebases are available:")
|
|
185
|
+
|
|
186
|
+
for codebase in codebases:
|
|
187
|
+
self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
|
|
188
|
+
if with_images:
|
|
189
|
+
self.list_latest_images_fn(
|
|
190
|
+
ecr_client,
|
|
191
|
+
f"{application.name}/{codebase['name']}",
|
|
192
|
+
codebase["repository"],
|
|
193
|
+
self.echo_fn,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
self.echo_fn("")
|
|
197
|
+
|
|
198
|
+
# TODO return empty list without exception
|
|
199
|
+
def __get_codebases(self, application, ssm_client):
|
|
200
|
+
parameters = ssm_client.get_parameters_by_path(
|
|
201
|
+
Path=f"/copilot/applications/{application.name}/codebases",
|
|
202
|
+
Recursive=True,
|
|
203
|
+
)["Parameters"]
|
|
204
|
+
|
|
205
|
+
codebases = [json.loads(p["Value"]) for p in parameters]
|
|
206
|
+
|
|
207
|
+
if not codebases:
|
|
208
|
+
raise NoCopilotCodebasesFoundError
|
|
209
|
+
return codebases
|
|
210
|
+
|
|
211
|
+
def __start_build_with_confirmation(
|
|
212
|
+
self,
|
|
213
|
+
confirm_fn,
|
|
214
|
+
codebuild_client,
|
|
215
|
+
get_build_url_from_arn_fn,
|
|
216
|
+
confirmation_message,
|
|
217
|
+
build_options,
|
|
218
|
+
):
|
|
219
|
+
if confirm_fn(confirmation_message):
|
|
220
|
+
build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
|
|
221
|
+
return get_build_url_from_arn_fn(build_arn)
|
|
222
|
+
return None
|
|
@@ -8,9 +8,9 @@ from boto3 import Session
|
|
|
8
8
|
|
|
9
9
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
10
10
|
from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
|
|
11
|
+
from dbt_platform_helper.exceptions import ApplicationNotFoundError
|
|
11
12
|
from dbt_platform_helper.exceptions import AWSException
|
|
12
13
|
from dbt_platform_helper.utils.application import Application
|
|
13
|
-
from dbt_platform_helper.utils.application import ApplicationNotFoundError
|
|
14
14
|
from dbt_platform_helper.utils.application import load_application
|
|
15
15
|
from dbt_platform_helper.utils.aws import Vpc
|
|
16
16
|
from dbt_platform_helper.utils.aws import get_connection_string
|
|
@@ -18,3 +18,31 @@ class IncompatibleMinorVersion(ValidationException):
|
|
|
18
18
|
super().__init__()
|
|
19
19
|
self.app_version = app_version
|
|
20
20
|
self.check_version = check_version
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CopilotCodebaseNotFoundError(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class NotInCodeBaseRepositoryError(Exception):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class NoCopilotCodebasesFoundError(Exception):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ImageNotFoundError(Exception):
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ApplicationDeploymentNotTriggered(Exception):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ApplicationNotFoundError(Exception):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ApplicationEnvironmentNotFoundError(Exception):
|
|
48
|
+
pass
|