dbt-platform-helper 12.2.4__py3-none-any.whl → 12.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/commands/codebase.py +9 -80
- dbt_platform_helper/commands/conduit.py +25 -45
- dbt_platform_helper/commands/secrets.py +1 -1
- dbt_platform_helper/domain/codebase.py +60 -60
- dbt_platform_helper/domain/conduit.py +44 -89
- dbt_platform_helper/domain/database_copy.py +42 -37
- dbt_platform_helper/exceptions.py +87 -21
- dbt_platform_helper/providers/cloudformation.py +122 -100
- dbt_platform_helper/providers/copilot.py +32 -12
- dbt_platform_helper/providers/ecs.py +78 -70
- dbt_platform_helper/providers/secrets.py +74 -74
- dbt_platform_helper/utils/application.py +1 -1
- dbt_platform_helper/utils/aws.py +29 -6
- dbt_platform_helper/utils/validation.py +39 -14
- {dbt_platform_helper-12.2.4.dist-info → dbt_platform_helper-12.3.0.dist-info}/METADATA +1 -1
- {dbt_platform_helper-12.2.4.dist-info → dbt_platform_helper-12.3.0.dist-info}/RECORD +19 -19
- {dbt_platform_helper-12.2.4.dist-info → dbt_platform_helper-12.3.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.2.4.dist-info → dbt_platform_helper-12.3.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-12.2.4.dist-info → dbt_platform_helper-12.3.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,18 +1,8 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import os
|
|
3
|
-
|
|
4
1
|
import click
|
|
5
2
|
|
|
6
3
|
from dbt_platform_helper.domain.codebase import Codebase
|
|
7
|
-
from dbt_platform_helper.exceptions import
|
|
8
|
-
from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
|
|
9
|
-
from dbt_platform_helper.exceptions import ApplicationNotFoundError
|
|
10
|
-
from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
|
|
11
|
-
from dbt_platform_helper.exceptions import ImageNotFoundError
|
|
12
|
-
from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
|
|
13
|
-
from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
|
|
4
|
+
from dbt_platform_helper.exceptions import PlatformException
|
|
14
5
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
15
|
-
from dbt_platform_helper.utils.git import CommitNotFoundError
|
|
16
6
|
from dbt_platform_helper.utils.versioning import (
|
|
17
7
|
check_platform_helper_version_needs_update,
|
|
18
8
|
)
|
|
@@ -29,12 +19,8 @@ def prepare():
|
|
|
29
19
|
"""Sets up an application codebase for use within a DBT platform project."""
|
|
30
20
|
try:
|
|
31
21
|
Codebase().prepare()
|
|
32
|
-
except
|
|
33
|
-
|
|
34
|
-
click.secho(
|
|
35
|
-
"You are in the deploy repository; make sure you are in the application codebase repository.",
|
|
36
|
-
fg="red",
|
|
37
|
-
)
|
|
22
|
+
except PlatformException as err:
|
|
23
|
+
click.secho(str(err), fg="red")
|
|
38
24
|
raise click.Abort
|
|
39
25
|
|
|
40
26
|
|
|
@@ -50,17 +36,8 @@ def list(app, with_images):
|
|
|
50
36
|
"""List available codebases for the application."""
|
|
51
37
|
try:
|
|
52
38
|
Codebase().list(app, with_images)
|
|
53
|
-
except
|
|
54
|
-
click.secho(
|
|
55
|
-
f"""No codebases found for application "{app}""",
|
|
56
|
-
fg="red",
|
|
57
|
-
)
|
|
58
|
-
raise click.Abort
|
|
59
|
-
except ApplicationNotFoundError:
|
|
60
|
-
click.secho(
|
|
61
|
-
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
62
|
-
fg="red",
|
|
63
|
-
)
|
|
39
|
+
except PlatformException as err:
|
|
40
|
+
click.secho(str(err), fg="red")
|
|
64
41
|
raise click.Abort
|
|
65
42
|
|
|
66
43
|
|
|
@@ -76,23 +53,8 @@ def build(app, codebase, commit):
|
|
|
76
53
|
"""Trigger a CodePipeline pipeline based build."""
|
|
77
54
|
try:
|
|
78
55
|
Codebase().build(app, codebase, commit)
|
|
79
|
-
except
|
|
80
|
-
click.secho(
|
|
81
|
-
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
82
|
-
fg="red",
|
|
83
|
-
)
|
|
84
|
-
raise click.Abort
|
|
85
|
-
except CommitNotFoundError:
|
|
86
|
-
click.secho(
|
|
87
|
-
f'The commit hash "{commit}" either does not exist or you need to run `git fetch`.',
|
|
88
|
-
fg="red",
|
|
89
|
-
)
|
|
90
|
-
raise click.Abort
|
|
91
|
-
except ApplicationDeploymentNotTriggered:
|
|
92
|
-
click.secho(
|
|
93
|
-
f"Your build for {codebase} was not triggered.",
|
|
94
|
-
fg="red",
|
|
95
|
-
)
|
|
56
|
+
except PlatformException as err:
|
|
57
|
+
click.secho(str(err), fg="red")
|
|
96
58
|
raise click.Abort
|
|
97
59
|
|
|
98
60
|
|
|
@@ -108,39 +70,6 @@ def build(app, codebase, commit):
|
|
|
108
70
|
def deploy(app, env, codebase, commit):
|
|
109
71
|
try:
|
|
110
72
|
Codebase().deploy(app, env, codebase, commit)
|
|
111
|
-
except
|
|
112
|
-
|
|
113
|
-
click.secho(
|
|
114
|
-
f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
|
|
115
|
-
fg="red",
|
|
116
|
-
)
|
|
117
|
-
raise click.Abort
|
|
118
|
-
except ApplicationEnvironmentNotFoundError:
|
|
119
|
-
click.secho(
|
|
120
|
-
f"""The environment "{env}" either does not exist or has not been deployed.""",
|
|
121
|
-
fg="red",
|
|
122
|
-
)
|
|
123
|
-
raise click.Abort
|
|
124
|
-
except (
|
|
125
|
-
CopilotCodebaseNotFoundError,
|
|
126
|
-
# TODO: Catch this error earlier and throw a more meaningful error, maybe it's CopilotCodebaseNotFoundError?
|
|
127
|
-
json.JSONDecodeError,
|
|
128
|
-
):
|
|
129
|
-
click.secho(
|
|
130
|
-
f"""The codebase "{codebase}" either does not exist or has not been deployed.""",
|
|
131
|
-
fg="red",
|
|
132
|
-
)
|
|
133
|
-
raise click.Abort
|
|
134
|
-
except ImageNotFoundError:
|
|
135
|
-
click.secho(
|
|
136
|
-
f'The commit hash "{commit}" has not been built into an image, try the '
|
|
137
|
-
"`platform-helper codebase build` command first.",
|
|
138
|
-
fg="red",
|
|
139
|
-
)
|
|
140
|
-
raise click.Abort
|
|
141
|
-
except ApplicationDeploymentNotTriggered:
|
|
142
|
-
click.secho(
|
|
143
|
-
f"Your deployment for {codebase} was not triggered.",
|
|
144
|
-
fg="red",
|
|
145
|
-
)
|
|
73
|
+
except PlatformException as err:
|
|
74
|
+
click.secho(str(err), fg="red")
|
|
146
75
|
raise click.Abort
|
|
@@ -1,14 +1,10 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
|
-
from dbt_platform_helper.constants import CONDUIT_ADDON_TYPES
|
|
4
3
|
from dbt_platform_helper.domain.conduit import Conduit
|
|
5
|
-
from dbt_platform_helper.exceptions import
|
|
6
|
-
from dbt_platform_helper.
|
|
7
|
-
from dbt_platform_helper.
|
|
8
|
-
from dbt_platform_helper.
|
|
9
|
-
from dbt_platform_helper.exceptions import NoClusterError
|
|
10
|
-
from dbt_platform_helper.exceptions import ParameterNotFoundError
|
|
11
|
-
from dbt_platform_helper.providers.secrets import SecretNotFoundError
|
|
4
|
+
from dbt_platform_helper.exceptions import AWSException
|
|
5
|
+
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
6
|
+
from dbt_platform_helper.providers.ecs import ECS
|
|
7
|
+
from dbt_platform_helper.providers.secrets import Secrets
|
|
12
8
|
from dbt_platform_helper.utils.application import load_application
|
|
13
9
|
from dbt_platform_helper.utils.click import ClickDocOptCommand
|
|
14
10
|
from dbt_platform_helper.utils.versioning import (
|
|
@@ -35,44 +31,28 @@ def conduit(addon_name: str, app: str, env: str, access: str):
|
|
|
35
31
|
application = load_application(app)
|
|
36
32
|
|
|
37
33
|
try:
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
except SecretNotFoundError as err:
|
|
44
|
-
click.secho(
|
|
45
|
-
f"""No secret called "{err}" for "{app}" in "{env}" environment.""",
|
|
46
|
-
fg="red",
|
|
34
|
+
secrets_provider: Secrets = Secrets(
|
|
35
|
+
application.environments[env].session.client("ssm"),
|
|
36
|
+
application.environments[env].session.client("secretsmanager"),
|
|
37
|
+
application.name,
|
|
38
|
+
env,
|
|
47
39
|
)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
fg="red",
|
|
40
|
+
cloudformation_provider: CloudFormation = CloudFormation(
|
|
41
|
+
application.environments[env].session.client("cloudformation"),
|
|
42
|
+
application.environments[env].session.client("iam"),
|
|
43
|
+
application.environments[env].session.client("ssm"),
|
|
53
44
|
)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
exit(1)
|
|
61
|
-
except AddonNotFoundError:
|
|
62
|
-
click.secho(
|
|
63
|
-
f"""Addon "{addon_name}" does not exist.""",
|
|
64
|
-
fg="red",
|
|
65
|
-
)
|
|
66
|
-
exit(1)
|
|
67
|
-
except InvalidAddonTypeError as err:
|
|
68
|
-
click.secho(
|
|
69
|
-
f"""Addon type "{err.addon_type}" is not supported, we support: {", ".join(CONDUIT_ADDON_TYPES)}.""",
|
|
70
|
-
fg="red",
|
|
45
|
+
|
|
46
|
+
ecs_provider: ECS = ECS(
|
|
47
|
+
application.environments[env].session.client("ecs"),
|
|
48
|
+
application.environments[env].session.client("ssm"),
|
|
49
|
+
application.name,
|
|
50
|
+
env,
|
|
71
51
|
)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
f"""The configuration for the addon {addon_name}, is missconfigured and missing the addon type.""",
|
|
76
|
-
fg="red",
|
|
52
|
+
|
|
53
|
+
Conduit(application, secrets_provider, cloudformation_provider, ecs_provider).start(
|
|
54
|
+
env, addon_name, access
|
|
77
55
|
)
|
|
78
|
-
|
|
56
|
+
except AWSException as err:
|
|
57
|
+
click.secho(str(err), fg="red")
|
|
58
|
+
raise click.Abort
|
|
@@ -102,7 +102,6 @@ def list(app, env):
|
|
|
102
102
|
params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
|
|
103
103
|
secrets = []
|
|
104
104
|
|
|
105
|
-
# TODO: refactor shared code with get_ssm_secret_names - Check if this is still valid
|
|
106
105
|
while True:
|
|
107
106
|
response = client.get_parameters_by_path(**params)
|
|
108
107
|
|
|
@@ -114,6 +113,7 @@ def list(app, env):
|
|
|
114
113
|
else:
|
|
115
114
|
break
|
|
116
115
|
|
|
116
|
+
# Todo: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
|
|
117
117
|
print("\n".join(sorted(secrets)))
|
|
118
118
|
|
|
119
119
|
|
|
@@ -11,7 +11,6 @@ from boto3 import Session
|
|
|
11
11
|
|
|
12
12
|
from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
|
|
13
13
|
from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
|
|
14
|
-
from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
|
|
15
14
|
from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
|
|
16
15
|
from dbt_platform_helper.utils.application import Application
|
|
17
16
|
from dbt_platform_helper.utils.application import load_application
|
|
@@ -29,31 +28,31 @@ from dbt_platform_helper.utils.template import setup_templates
|
|
|
29
28
|
class Codebase:
|
|
30
29
|
def __init__(
|
|
31
30
|
self,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
31
|
+
input: Callable[[str], str] = click.prompt,
|
|
32
|
+
echo: Callable[[str], str] = click.secho,
|
|
33
|
+
confirm: Callable[[str], bool] = click.confirm,
|
|
34
|
+
load_application: Callable[[str], Application] = load_application,
|
|
35
|
+
get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
|
|
36
|
+
check_codebase_exists: Callable[[str], str] = check_codebase_exists,
|
|
37
|
+
check_image_exists: Callable[[str], str] = check_image_exists,
|
|
38
|
+
get_build_url_from_arn: Callable[[str], str] = get_build_url_from_arn,
|
|
39
|
+
list_latest_images: Callable[[str], str] = list_latest_images,
|
|
40
|
+
start_build_extraction: Callable[[str], str] = start_build_extraction,
|
|
41
|
+
check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
|
|
42
|
+
run_subprocess: Callable[[str], str] = subprocess.run,
|
|
44
43
|
):
|
|
45
|
-
self.
|
|
46
|
-
self.
|
|
47
|
-
self.
|
|
48
|
-
self.
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
52
|
-
self.
|
|
53
|
-
self.
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
56
|
-
self.
|
|
44
|
+
self.input = input
|
|
45
|
+
self.echo = echo
|
|
46
|
+
self.confirm = confirm
|
|
47
|
+
self.load_application = load_application
|
|
48
|
+
self.get_aws_session_or_abort = get_aws_session_or_abort
|
|
49
|
+
self.check_codebase_exists = check_codebase_exists
|
|
50
|
+
self.check_image_exists = check_image_exists
|
|
51
|
+
self.get_build_url_from_arn = get_build_url_from_arn
|
|
52
|
+
self.list_latest_images = list_latest_images
|
|
53
|
+
self.start_build_extraction = start_build_extraction
|
|
54
|
+
self.check_if_commit_exists = check_if_commit_exists
|
|
55
|
+
self.run_subprocess = run_subprocess
|
|
57
56
|
|
|
58
57
|
def prepare(self):
|
|
59
58
|
"""Sets up an application codebase for use within a DBT platform
|
|
@@ -61,13 +60,15 @@ class Codebase:
|
|
|
61
60
|
templates = setup_templates()
|
|
62
61
|
|
|
63
62
|
repository = (
|
|
64
|
-
self.
|
|
63
|
+
self.run_subprocess(
|
|
64
|
+
["git", "remote", "get-url", "origin"], capture_output=True, text=True
|
|
65
|
+
)
|
|
65
66
|
.stdout.split("/")[-1]
|
|
66
67
|
.strip()
|
|
67
68
|
.removesuffix(".git")
|
|
68
69
|
)
|
|
69
70
|
if repository.endswith("-deploy") or Path("./copilot").exists():
|
|
70
|
-
raise NotInCodeBaseRepositoryError
|
|
71
|
+
raise NotInCodeBaseRepositoryError()
|
|
71
72
|
|
|
72
73
|
builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
|
|
73
74
|
builder_configuration_response = requests.get(builder_configuration_url)
|
|
@@ -91,7 +92,7 @@ class Codebase:
|
|
|
91
92
|
config_contents = templates.get_template(f".copilot/config.yml").render(
|
|
92
93
|
repository=repository, builder_version=builder_version
|
|
93
94
|
)
|
|
94
|
-
self.
|
|
95
|
+
self.echo(
|
|
95
96
|
mkfile(
|
|
96
97
|
Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
|
|
97
98
|
)
|
|
@@ -100,27 +101,27 @@ class Codebase:
|
|
|
100
101
|
image_build_run_file = Path(".copilot/image_build_run.sh")
|
|
101
102
|
image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
|
|
102
103
|
|
|
103
|
-
self.
|
|
104
|
+
self.echo(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
|
|
104
105
|
|
|
105
106
|
for phase in ["build", "install", "post_build", "pre_build"]:
|
|
106
107
|
phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
|
|
107
108
|
|
|
108
|
-
self.
|
|
109
|
+
self.echo(
|
|
109
110
|
mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
|
|
110
111
|
)
|
|
111
112
|
|
|
112
113
|
def build(self, app: str, codebase: str, commit: str):
|
|
113
114
|
"""Trigger a CodePipeline pipeline based build."""
|
|
114
|
-
session = self.
|
|
115
|
-
self.
|
|
115
|
+
session = self.get_aws_session_or_abort()
|
|
116
|
+
self.load_application(app, default_session=session)
|
|
116
117
|
|
|
117
|
-
self.
|
|
118
|
+
self.check_if_commit_exists(commit)
|
|
118
119
|
|
|
119
120
|
codebuild_client = session.client("codebuild")
|
|
120
121
|
build_url = self.__start_build_with_confirmation(
|
|
121
|
-
self.
|
|
122
|
+
self.confirm,
|
|
122
123
|
codebuild_client,
|
|
123
|
-
self.
|
|
124
|
+
self.get_build_url_from_arn,
|
|
124
125
|
f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
|
|
125
126
|
{
|
|
126
127
|
"projectName": f"codebuild-{app}-{codebase}",
|
|
@@ -130,29 +131,29 @@ class Codebase:
|
|
|
130
131
|
)
|
|
131
132
|
|
|
132
133
|
if build_url:
|
|
133
|
-
return self.
|
|
134
|
+
return self.echo(
|
|
134
135
|
f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
|
|
135
136
|
)
|
|
136
137
|
|
|
137
|
-
raise ApplicationDeploymentNotTriggered()
|
|
138
|
+
raise ApplicationDeploymentNotTriggered(codebase)
|
|
138
139
|
|
|
139
140
|
def deploy(self, app, env, codebase, commit):
|
|
140
141
|
"""Trigger a CodePipeline pipeline based deployment."""
|
|
141
|
-
session = self.
|
|
142
|
+
session = self.get_aws_session_or_abort()
|
|
142
143
|
|
|
143
|
-
application = self.
|
|
144
|
+
application = self.load_application(app, default_session=session)
|
|
144
145
|
if not application.environments.get(env):
|
|
145
|
-
raise ApplicationEnvironmentNotFoundError()
|
|
146
|
+
raise ApplicationEnvironmentNotFoundError(env)
|
|
146
147
|
|
|
147
|
-
self.
|
|
148
|
+
self.check_codebase_exists(session, application, codebase)
|
|
148
149
|
|
|
149
|
-
self.
|
|
150
|
+
self.check_image_exists(session, application, codebase, commit)
|
|
150
151
|
|
|
151
152
|
codebuild_client = session.client("codebuild")
|
|
152
153
|
build_url = self.__start_build_with_confirmation(
|
|
153
|
-
self.
|
|
154
|
+
self.confirm,
|
|
154
155
|
codebuild_client,
|
|
155
|
-
self.
|
|
156
|
+
self.get_build_url_from_arn,
|
|
156
157
|
f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
|
|
157
158
|
{
|
|
158
159
|
"projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
|
|
@@ -166,34 +167,34 @@ class Codebase:
|
|
|
166
167
|
)
|
|
167
168
|
|
|
168
169
|
if build_url:
|
|
169
|
-
return self.
|
|
170
|
+
return self.echo(
|
|
170
171
|
"Your deployment has been triggered. Check your build progress in the AWS Console: "
|
|
171
172
|
f"{build_url}",
|
|
172
173
|
)
|
|
173
174
|
|
|
174
|
-
raise ApplicationDeploymentNotTriggered()
|
|
175
|
+
raise ApplicationDeploymentNotTriggered(codebase)
|
|
175
176
|
|
|
176
177
|
def list(self, app: str, with_images: bool):
|
|
177
178
|
"""List available codebases for the application."""
|
|
178
|
-
session = self.
|
|
179
|
-
application = self.
|
|
179
|
+
session = self.get_aws_session_or_abort()
|
|
180
|
+
application = self.load_application(app, session)
|
|
180
181
|
ssm_client = session.client("ssm")
|
|
181
182
|
ecr_client = session.client("ecr")
|
|
182
183
|
codebases = self.__get_codebases(application, ssm_client)
|
|
183
184
|
|
|
184
|
-
self.
|
|
185
|
+
self.echo("The following codebases are available:")
|
|
185
186
|
|
|
186
187
|
for codebase in codebases:
|
|
187
|
-
self.
|
|
188
|
+
self.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
|
|
188
189
|
if with_images:
|
|
189
|
-
self.
|
|
190
|
+
self.list_latest_images(
|
|
190
191
|
ecr_client,
|
|
191
192
|
f"{application.name}/{codebase['name']}",
|
|
192
193
|
codebase["repository"],
|
|
193
|
-
self.
|
|
194
|
+
self.echo,
|
|
194
195
|
)
|
|
195
196
|
|
|
196
|
-
self.
|
|
197
|
+
self.echo("")
|
|
197
198
|
|
|
198
199
|
def __get_codebases(self, application, ssm_client):
|
|
199
200
|
parameters = ssm_client.get_parameters_by_path(
|
|
@@ -204,19 +205,18 @@ class Codebase:
|
|
|
204
205
|
codebases = [json.loads(p["Value"]) for p in parameters]
|
|
205
206
|
|
|
206
207
|
if not codebases:
|
|
207
|
-
|
|
208
|
-
raise NoCopilotCodebasesFoundError
|
|
208
|
+
return []
|
|
209
209
|
return codebases
|
|
210
210
|
|
|
211
211
|
def __start_build_with_confirmation(
|
|
212
212
|
self,
|
|
213
|
-
|
|
213
|
+
confirm,
|
|
214
214
|
codebuild_client,
|
|
215
|
-
|
|
215
|
+
get_build_url_from_arn,
|
|
216
216
|
confirmation_message,
|
|
217
217
|
build_options,
|
|
218
218
|
):
|
|
219
|
-
if
|
|
220
|
-
build_arn = self.
|
|
221
|
-
return
|
|
219
|
+
if confirm(confirmation_message):
|
|
220
|
+
build_arn = self.start_build_extraction(codebuild_client, build_options)
|
|
221
|
+
return get_build_url_from_arn(build_arn)
|
|
222
222
|
return None
|
|
@@ -3,80 +3,54 @@ from collections.abc import Callable
|
|
|
3
3
|
|
|
4
4
|
import click
|
|
5
5
|
|
|
6
|
-
from dbt_platform_helper.
|
|
7
|
-
from dbt_platform_helper.providers.cloudformation import (
|
|
8
|
-
add_stack_delete_policy_to_task_role,
|
|
9
|
-
)
|
|
10
|
-
from dbt_platform_helper.providers.cloudformation import update_conduit_stack_resources
|
|
11
|
-
from dbt_platform_helper.providers.cloudformation import (
|
|
12
|
-
wait_for_cloudformation_to_reach_status,
|
|
13
|
-
)
|
|
6
|
+
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
14
7
|
from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
|
|
15
8
|
from dbt_platform_helper.providers.copilot import create_addon_client_task
|
|
16
9
|
from dbt_platform_helper.providers.copilot import create_postgres_admin_task
|
|
17
|
-
from dbt_platform_helper.providers.ecs import
|
|
18
|
-
from dbt_platform_helper.providers.
|
|
19
|
-
from dbt_platform_helper.providers.ecs import get_ecs_task_arns
|
|
20
|
-
from dbt_platform_helper.providers.ecs import get_or_create_task_name
|
|
21
|
-
from dbt_platform_helper.providers.secrets import get_addon_type
|
|
22
|
-
from dbt_platform_helper.providers.secrets import get_parameter_name
|
|
10
|
+
from dbt_platform_helper.providers.ecs import ECS
|
|
11
|
+
from dbt_platform_helper.providers.secrets import Secrets
|
|
23
12
|
from dbt_platform_helper.utils.application import Application
|
|
24
|
-
from dbt_platform_helper.utils.messages import abort_with_error
|
|
25
13
|
|
|
26
14
|
|
|
27
15
|
class Conduit:
|
|
28
16
|
def __init__(
|
|
29
17
|
self,
|
|
30
18
|
application: Application,
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
get_cluster_arn_fn=get_cluster_arn,
|
|
40
|
-
get_parameter_name_fn=get_parameter_name,
|
|
41
|
-
get_or_create_task_name_fn=get_or_create_task_name,
|
|
42
|
-
add_stack_delete_policy_to_task_role_fn=add_stack_delete_policy_to_task_role,
|
|
43
|
-
update_conduit_stack_resources_fn=update_conduit_stack_resources,
|
|
44
|
-
wait_for_cloudformation_to_reach_status_fn=wait_for_cloudformation_to_reach_status,
|
|
45
|
-
abort_fn=abort_with_error,
|
|
19
|
+
secrets_provider: Secrets,
|
|
20
|
+
cloudformation_provider: CloudFormation,
|
|
21
|
+
ecs_provider: ECS,
|
|
22
|
+
echo: Callable[[str], str] = click.secho,
|
|
23
|
+
subprocess: subprocess = subprocess,
|
|
24
|
+
connect_to_addon_client_task=connect_to_addon_client_task,
|
|
25
|
+
create_addon_client_task=create_addon_client_task,
|
|
26
|
+
create_postgres_admin_task=create_postgres_admin_task,
|
|
46
27
|
):
|
|
47
28
|
|
|
48
29
|
self.application = application
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
52
|
-
self.
|
|
53
|
-
self.
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
56
|
-
self.
|
|
57
|
-
self.get_cluster_arn_fn = get_cluster_arn_fn
|
|
58
|
-
self.get_parameter_name_fn = get_parameter_name_fn
|
|
59
|
-
self.get_or_create_task_name_fn = get_or_create_task_name_fn
|
|
60
|
-
self.add_stack_delete_policy_to_task_role_fn = add_stack_delete_policy_to_task_role_fn
|
|
61
|
-
self.update_conduit_stack_resources_fn = update_conduit_stack_resources_fn
|
|
62
|
-
self.wait_for_cloudformation_to_reach_status_fn = wait_for_cloudformation_to_reach_status_fn
|
|
63
|
-
self.abort_fn = abort_fn
|
|
30
|
+
self.secrets_provider = secrets_provider
|
|
31
|
+
self.cloudformation_provider = cloudformation_provider
|
|
32
|
+
self.ecs_provider = ecs_provider
|
|
33
|
+
self.subprocess = subprocess
|
|
34
|
+
self.echo = echo
|
|
35
|
+
self.connect_to_addon_client_task = connect_to_addon_client_task
|
|
36
|
+
self.create_addon_client_task = create_addon_client_task
|
|
37
|
+
self.create_postgres_admin_task = create_postgres_admin_task
|
|
64
38
|
|
|
65
39
|
def start(self, env: str, addon_name: str, access: str = "read"):
|
|
66
40
|
clients = self._initialise_clients(env)
|
|
67
41
|
addon_type, cluster_arn, parameter_name, task_name = self._get_addon_details(
|
|
68
|
-
|
|
42
|
+
addon_name, access
|
|
69
43
|
)
|
|
70
44
|
|
|
71
|
-
self.
|
|
72
|
-
|
|
73
|
-
if not
|
|
74
|
-
self.
|
|
75
|
-
self.
|
|
45
|
+
self.echo(f"Checking if a conduit task is already running for {addon_type}")
|
|
46
|
+
task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
|
|
47
|
+
if not task_arns:
|
|
48
|
+
self.echo("Creating conduit task")
|
|
49
|
+
self.create_addon_client_task(
|
|
76
50
|
clients["iam"],
|
|
77
51
|
clients["ssm"],
|
|
78
52
|
clients["secrets_manager"],
|
|
79
|
-
self.
|
|
53
|
+
self.subprocess,
|
|
80
54
|
self.application,
|
|
81
55
|
env,
|
|
82
56
|
addon_type,
|
|
@@ -85,11 +59,8 @@ class Conduit:
|
|
|
85
59
|
access,
|
|
86
60
|
)
|
|
87
61
|
|
|
88
|
-
self.
|
|
62
|
+
self.echo("Updating conduit task")
|
|
89
63
|
self._update_stack_resources(
|
|
90
|
-
clients["cloudformation"],
|
|
91
|
-
clients["iam"],
|
|
92
|
-
clients["ssm"],
|
|
93
64
|
self.application.name,
|
|
94
65
|
env,
|
|
95
66
|
addon_type,
|
|
@@ -99,21 +70,18 @@ class Conduit:
|
|
|
99
70
|
access,
|
|
100
71
|
)
|
|
101
72
|
|
|
102
|
-
|
|
73
|
+
task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
|
|
103
74
|
|
|
104
75
|
else:
|
|
105
|
-
self.
|
|
76
|
+
self.echo("Conduit task already running")
|
|
106
77
|
|
|
107
|
-
self.
|
|
78
|
+
self.echo(f"Checking if exec is available for conduit task...")
|
|
108
79
|
|
|
109
|
-
|
|
110
|
-
self.ecs_exec_is_available_fn(clients["ecs"], cluster_arn, task_arn)
|
|
111
|
-
except ECSAgentNotRunning:
|
|
112
|
-
self.abort_fn('ECS exec agent never reached "RUNNING" status')
|
|
80
|
+
self.ecs_provider.ecs_exec_is_available(cluster_arn, task_arns)
|
|
113
81
|
|
|
114
|
-
self.
|
|
115
|
-
self.
|
|
116
|
-
clients["ecs"], self.
|
|
82
|
+
self.echo("Connecting to conduit task")
|
|
83
|
+
self.connect_to_addon_client_task(
|
|
84
|
+
clients["ecs"], self.subprocess, self.application.name, env, cluster_arn, task_name
|
|
117
85
|
)
|
|
118
86
|
|
|
119
87
|
def _initialise_clients(self, env):
|
|
@@ -125,26 +93,16 @@ class Conduit:
|
|
|
125
93
|
"secrets_manager": self.application.environments[env].session.client("secretsmanager"),
|
|
126
94
|
}
|
|
127
95
|
|
|
128
|
-
def _get_addon_details(self,
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
cluster_arn = self.get_cluster_arn_fn(ecs_client, self.application.name, env)
|
|
134
|
-
parameter_name = self.get_parameter_name_fn(
|
|
135
|
-
self.application.name, env, addon_type, addon_name, access
|
|
136
|
-
)
|
|
137
|
-
task_name = self.get_or_create_task_name_fn(
|
|
138
|
-
ssm_client, self.application.name, env, addon_name, parameter_name
|
|
139
|
-
)
|
|
96
|
+
def _get_addon_details(self, addon_name, access):
|
|
97
|
+
addon_type = self.secrets_provider.get_addon_type(addon_name)
|
|
98
|
+
cluster_arn = self.ecs_provider.get_cluster_arn()
|
|
99
|
+
parameter_name = self.secrets_provider.get_parameter_name(addon_type, addon_name, access)
|
|
100
|
+
task_name = self.ecs_provider.get_or_create_task_name(addon_name, parameter_name)
|
|
140
101
|
|
|
141
102
|
return addon_type, cluster_arn, parameter_name, task_name
|
|
142
103
|
|
|
143
104
|
def _update_stack_resources(
|
|
144
105
|
self,
|
|
145
|
-
cloudformation_client,
|
|
146
|
-
iam_client,
|
|
147
|
-
ssm_client,
|
|
148
106
|
app_name,
|
|
149
107
|
env,
|
|
150
108
|
addon_type,
|
|
@@ -153,11 +111,8 @@ class Conduit:
|
|
|
153
111
|
parameter_name,
|
|
154
112
|
access,
|
|
155
113
|
):
|
|
156
|
-
self.
|
|
157
|
-
stack_name = self.
|
|
158
|
-
cloudformation_client,
|
|
159
|
-
iam_client,
|
|
160
|
-
ssm_client,
|
|
114
|
+
self.cloudformation_provider.add_stack_delete_policy_to_task_role(task_name)
|
|
115
|
+
stack_name = self.cloudformation_provider.update_conduit_stack_resources(
|
|
161
116
|
app_name,
|
|
162
117
|
env,
|
|
163
118
|
addon_type,
|
|
@@ -166,7 +121,7 @@ class Conduit:
|
|
|
166
121
|
parameter_name,
|
|
167
122
|
access,
|
|
168
123
|
)
|
|
169
|
-
self.
|
|
170
|
-
self.
|
|
171
|
-
|
|
124
|
+
self.echo("Waiting for conduit task update to complete...")
|
|
125
|
+
self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
|
|
126
|
+
"stack_update_complete", stack_name
|
|
172
127
|
)
|