dbt-platform-helper 15.1.0__py3-none-any.whl → 15.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/domain/codebase.py +12 -3
- dbt_platform_helper/domain/conduit.py +322 -76
- dbt_platform_helper/domain/copilot.py +4 -1
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/entities/platform_config_schema.py +7 -48
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/aws/exceptions.py +20 -2
- dbt_platform_helper/providers/copilot.py +20 -12
- dbt_platform_helper/providers/ecr.py +62 -37
- dbt_platform_helper/providers/ecs.py +112 -40
- dbt_platform_helper/providers/io.py +6 -1
- dbt_platform_helper/providers/terraform_manifest.py +7 -1
- dbt_platform_helper/providers/vpc.py +1 -1
- dbt_platform_helper/utilities/decorators.py +103 -0
- {dbt_platform_helper-15.1.0.dist-info → dbt_platform_helper-15.2.1.dist-info}/METADATA +1 -1
- {dbt_platform_helper-15.1.0.dist-info → dbt_platform_helper-15.2.1.dist-info}/RECORD +22 -18
- {dbt_platform_helper-15.1.0.dist-info → dbt_platform_helper-15.2.1.dist-info}/WHEEL +1 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- {dbt_platform_helper-15.1.0.dist-info → dbt_platform_helper-15.2.1.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-15.1.0.dist-info → dbt_platform_helper-15.2.1.dist-info}/entry_points.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import subprocess
|
|
2
3
|
import time
|
|
3
4
|
|
|
4
5
|
from botocore.exceptions import ClientError
|
|
@@ -13,7 +14,6 @@ from dbt_platform_helper.utils.messages import abort_with_error
|
|
|
13
14
|
def create_addon_client_task(
|
|
14
15
|
iam_client,
|
|
15
16
|
ssm_client,
|
|
16
|
-
subprocess,
|
|
17
17
|
application: Application,
|
|
18
18
|
env: str,
|
|
19
19
|
addon_type: str,
|
|
@@ -31,7 +31,6 @@ def create_addon_client_task(
|
|
|
31
31
|
elif access == "admin":
|
|
32
32
|
create_postgres_admin_task(
|
|
33
33
|
ssm_client,
|
|
34
|
-
subprocess,
|
|
35
34
|
application,
|
|
36
35
|
addon_name,
|
|
37
36
|
addon_type,
|
|
@@ -71,15 +70,8 @@ def create_addon_client_task(
|
|
|
71
70
|
)
|
|
72
71
|
|
|
73
72
|
|
|
74
|
-
def
|
|
75
|
-
ssm_client,
|
|
76
|
-
subprocess,
|
|
77
|
-
app: Application,
|
|
78
|
-
addon_name: str,
|
|
79
|
-
addon_type: str,
|
|
80
|
-
env: str,
|
|
81
|
-
secret_name: str,
|
|
82
|
-
task_name: str,
|
|
73
|
+
def get_postgres_admin_connection_string(
|
|
74
|
+
ssm_client, secret_name: str, app: Application, env: str, addon_name: str
|
|
83
75
|
):
|
|
84
76
|
read_only_secret_name = secret_name + "_READ_ONLY_USER"
|
|
85
77
|
master_secret_name = (
|
|
@@ -94,6 +86,23 @@ def create_postgres_admin_task(
|
|
|
94
86
|
)
|
|
95
87
|
)
|
|
96
88
|
|
|
89
|
+
return connection_string
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def create_postgres_admin_task(
|
|
93
|
+
ssm_client,
|
|
94
|
+
app: Application,
|
|
95
|
+
addon_name: str,
|
|
96
|
+
addon_type: str,
|
|
97
|
+
env: str,
|
|
98
|
+
secret_name: str,
|
|
99
|
+
task_name: str,
|
|
100
|
+
):
|
|
101
|
+
|
|
102
|
+
connection_string = get_postgres_admin_connection_string(
|
|
103
|
+
ssm_client, secret_name, app, env, addon_name
|
|
104
|
+
)
|
|
105
|
+
|
|
97
106
|
subprocess.call(
|
|
98
107
|
f"copilot task run --app {app.name} --env {env} "
|
|
99
108
|
f"--task-group-name {task_name} "
|
|
@@ -121,7 +130,6 @@ def _temp_until_refactor_get_ecs_task_arns(ecs_client, cluster_arn: str, task_na
|
|
|
121
130
|
|
|
122
131
|
def connect_to_addon_client_task(
|
|
123
132
|
ecs_client,
|
|
124
|
-
subprocess,
|
|
125
133
|
application_name,
|
|
126
134
|
env,
|
|
127
135
|
cluster_arn,
|
|
@@ -1,12 +1,18 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
|
|
1
3
|
import botocore
|
|
2
4
|
from boto3 import Session
|
|
3
5
|
|
|
6
|
+
from dbt_platform_helper.providers.aws.exceptions import AWSException
|
|
4
7
|
from dbt_platform_helper.providers.aws.exceptions import ImageNotFoundException
|
|
8
|
+
from dbt_platform_helper.providers.aws.exceptions import MultipleImagesFoundException
|
|
5
9
|
from dbt_platform_helper.providers.aws.exceptions import RepositoryNotFoundException
|
|
6
10
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
7
|
-
from dbt_platform_helper.utils.application import Application
|
|
8
11
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
9
12
|
|
|
13
|
+
NOT_A_UNIQUE_TAG_INFO = 'INFO: The tag "{image_ref}" is not a unique, commit-specific tag. Deploying the corresponding commit tag "{commit_tag}" instead.'
|
|
14
|
+
NO_ASSOCIATED_COMMIT_TAG_WARNING = 'WARNING: The AWS ECR image "{image_ref}" has no associated commit tag so deploying "{image_ref}". Note this could result in images with unintended or incompatible changes being deployed in new ECS Tasks for your service.'
|
|
15
|
+
|
|
10
16
|
|
|
11
17
|
class ECRProvider:
|
|
12
18
|
def __init__(self, session: Session = None, click_io: ClickIOProvider = ClickIOProvider()):
|
|
@@ -19,49 +25,68 @@ class ECRProvider:
|
|
|
19
25
|
out.extend([repo["repositoryName"] for repo in page.get("repositories", {})])
|
|
20
26
|
return out
|
|
21
27
|
|
|
22
|
-
def
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
def get_commit_tag_for_reference(self, application_name: str, codebase: str, image_ref: str):
|
|
29
|
+
repository = f"{application_name}/{codebase}"
|
|
30
|
+
next_page_token = None
|
|
31
|
+
tag_map = {}
|
|
32
|
+
digest_map = defaultdict(dict)
|
|
27
33
|
|
|
28
|
-
|
|
34
|
+
while True:
|
|
35
|
+
image_list = self._get_ecr_images(repository, image_ref, next_page_token)
|
|
36
|
+
next_page_token = image_list.get("nextToken")
|
|
29
37
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
38
|
+
for image in image_list["imageIds"]:
|
|
39
|
+
digest, tag = image["imageDigest"], image["imageTag"]
|
|
40
|
+
digest_map[digest][tag.split("-")[0]] = tag
|
|
41
|
+
tag_map[tag] = digest
|
|
35
42
|
|
|
36
|
-
|
|
43
|
+
if not next_page_token:
|
|
44
|
+
break
|
|
37
45
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
46
|
+
if image_ref.startswith("commit-"):
|
|
47
|
+
if image_ref in tag_map:
|
|
48
|
+
return image_ref
|
|
49
|
+
else:
|
|
50
|
+
candidates = [
|
|
51
|
+
tag
|
|
52
|
+
for tag in tag_map.keys()
|
|
53
|
+
if image_ref.startswith(tag) or tag.startswith(image_ref)
|
|
54
|
+
]
|
|
55
|
+
if not candidates:
|
|
56
|
+
raise ImageNotFoundException(image_ref)
|
|
57
|
+
if len(candidates) > 1:
|
|
58
|
+
raise MultipleImagesFoundException(image_ref, candidates)
|
|
59
|
+
return candidates[0]
|
|
60
|
+
else:
|
|
61
|
+
digest = tag_map.get(image_ref)
|
|
62
|
+
if not digest:
|
|
41
63
|
raise ImageNotFoundException(image_ref)
|
|
42
|
-
if e.response["Error"]["Code"] == "RepositoryNotFoundException":
|
|
43
|
-
raise RepositoryNotFoundException(repository)
|
|
44
64
|
|
|
45
|
-
|
|
46
|
-
"""Loop through imageTags list to query for an image tag starting with
|
|
47
|
-
'commit-', and return that value if found."""
|
|
65
|
+
commit_tag = digest_map.get(digest, dict()).get("commit")
|
|
48
66
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
67
|
+
if commit_tag:
|
|
68
|
+
self.click_io.info(
|
|
69
|
+
NOT_A_UNIQUE_TAG_INFO.format(image_ref=image_ref, commit_tag=commit_tag)
|
|
70
|
+
)
|
|
71
|
+
return commit_tag
|
|
72
|
+
else:
|
|
73
|
+
self.click_io.warn(NO_ASSOCIATED_COMMIT_TAG_WARNING.format(image_ref=image_ref))
|
|
74
|
+
return image_ref
|
|
75
|
+
|
|
76
|
+
def _get_ecr_images(self, repository, image_ref, next_page_token):
|
|
77
|
+
params = {"repositoryName": repository, "filter": {"tagStatus": "TAGGED"}}
|
|
78
|
+
if next_page_token:
|
|
79
|
+
params["nextToken"] = next_page_token
|
|
80
|
+
try:
|
|
81
|
+
image_list = self._get_client().list_images(**params)
|
|
82
|
+
return image_list
|
|
83
|
+
except botocore.exceptions.ClientError as e:
|
|
84
|
+
if e.response["Error"]["Code"] == "RepositoryNotFoundException":
|
|
85
|
+
raise RepositoryNotFoundException(repository)
|
|
86
|
+
else:
|
|
87
|
+
raise AWSException(
|
|
88
|
+
f"Unexpected error for repo '{repository}' and image reference '{image_ref}': {e}"
|
|
89
|
+
)
|
|
65
90
|
|
|
66
91
|
@staticmethod
|
|
67
92
|
def _check_image_details_exists(image_info: dict, image_ref: str):
|
|
@@ -1,9 +1,29 @@
|
|
|
1
1
|
import random
|
|
2
2
|
import string
|
|
3
|
-
import
|
|
3
|
+
import subprocess
|
|
4
4
|
from typing import List
|
|
5
5
|
|
|
6
6
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
7
|
+
from dbt_platform_helper.platform_exception import ValidationException
|
|
8
|
+
from dbt_platform_helper.providers.vpc import Vpc
|
|
9
|
+
from dbt_platform_helper.utilities.decorators import retry
|
|
10
|
+
from dbt_platform_helper.utilities.decorators import wait_until
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ECSException(PlatformException):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ECSAgentNotRunningException(ECSException):
|
|
18
|
+
def __init__(self):
|
|
19
|
+
super().__init__("""ECS exec agent never reached "RUNNING" status""")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class NoClusterException(ECSException):
|
|
23
|
+
def __init__(self, application_name: str, environment: str):
|
|
24
|
+
super().__init__(
|
|
25
|
+
f"""No ECS cluster found for "{application_name}" in "{environment}" environment."""
|
|
26
|
+
)
|
|
7
27
|
|
|
8
28
|
|
|
9
29
|
class ECS:
|
|
@@ -13,7 +33,49 @@ class ECS:
|
|
|
13
33
|
self.application_name = application_name
|
|
14
34
|
self.env = env
|
|
15
35
|
|
|
16
|
-
def
|
|
36
|
+
def start_ecs_task(
|
|
37
|
+
self,
|
|
38
|
+
cluster_name: str,
|
|
39
|
+
container_name: str,
|
|
40
|
+
task_def_arn: str,
|
|
41
|
+
vpc_config: Vpc,
|
|
42
|
+
env_vars: List[dict] = None,
|
|
43
|
+
):
|
|
44
|
+
container_override = {"name": container_name}
|
|
45
|
+
if env_vars:
|
|
46
|
+
container_override["environment"] = env_vars
|
|
47
|
+
|
|
48
|
+
response = self.ecs_client.run_task(
|
|
49
|
+
taskDefinition=task_def_arn,
|
|
50
|
+
cluster=cluster_name,
|
|
51
|
+
capacityProviderStrategy=[
|
|
52
|
+
{"capacityProvider": "FARGATE", "weight": 1, "base": 0},
|
|
53
|
+
],
|
|
54
|
+
enableExecuteCommand=True,
|
|
55
|
+
networkConfiguration={
|
|
56
|
+
"awsvpcConfiguration": {
|
|
57
|
+
"subnets": vpc_config.public_subnets,
|
|
58
|
+
"securityGroups": vpc_config.security_groups,
|
|
59
|
+
"assignPublicIp": "ENABLED",
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
overrides={"containerOverrides": [container_override]},
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
return response.get("tasks", [{}])[0].get("taskArn")
|
|
66
|
+
|
|
67
|
+
def get_cluster_arn_by_name(self, cluster_name: str) -> str:
|
|
68
|
+
clusters = self.ecs_client.describe_clusters(
|
|
69
|
+
clusters=[
|
|
70
|
+
cluster_name,
|
|
71
|
+
],
|
|
72
|
+
)["clusters"]
|
|
73
|
+
if len(clusters) == 1 and "clusterArn" in clusters[0]:
|
|
74
|
+
return clusters[0]["clusterArn"]
|
|
75
|
+
|
|
76
|
+
raise NoClusterException(self.application_name, self.env)
|
|
77
|
+
|
|
78
|
+
def get_cluster_arn_by_copilot_tag(self) -> str:
|
|
17
79
|
"""Returns the ARN of the ECS cluster for the given application and
|
|
18
80
|
environment."""
|
|
19
81
|
for cluster_arn in self.ecs_client.list_clusters()["clusterArns"]:
|
|
@@ -45,12 +107,12 @@ class ECS:
|
|
|
45
107
|
random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
|
|
46
108
|
return f"conduit-{self.application_name}-{self.env}-{addon_name}-{random_id}"
|
|
47
109
|
|
|
48
|
-
def get_ecs_task_arns(self, cluster_arn: str,
|
|
110
|
+
def get_ecs_task_arns(self, cluster_arn: str, task_def_family: str):
|
|
49
111
|
"""Gets the ECS task ARNs for a given task name and cluster ARN."""
|
|
50
112
|
tasks = self.ecs_client.list_tasks(
|
|
51
113
|
cluster=cluster_arn,
|
|
52
114
|
desiredStatus="RUNNING",
|
|
53
|
-
family=
|
|
115
|
+
family=task_def_family,
|
|
54
116
|
)
|
|
55
117
|
|
|
56
118
|
if not tasks["taskArns"]:
|
|
@@ -58,45 +120,55 @@ class ECS:
|
|
|
58
120
|
|
|
59
121
|
return tasks["taskArns"]
|
|
60
122
|
|
|
61
|
-
|
|
123
|
+
@retry()
|
|
124
|
+
def exec_task(self, cluster_arn: str, task_arn: str, subprocess_call=subprocess.call):
|
|
125
|
+
result = subprocess_call(
|
|
126
|
+
f"aws ecs execute-command --cluster {cluster_arn} "
|
|
127
|
+
f"--task {task_arn} "
|
|
128
|
+
f"--interactive --command bash ",
|
|
129
|
+
shell=True,
|
|
130
|
+
)
|
|
131
|
+
if result != 0:
|
|
132
|
+
raise PlatformException("Failed to exec into ECS task.")
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
@wait_until(
|
|
136
|
+
max_attempts=25,
|
|
137
|
+
exceptions_to_catch=(ECSException,),
|
|
138
|
+
message_on_false="ECS Agent Not running",
|
|
139
|
+
)
|
|
140
|
+
def ecs_exec_is_available(self, cluster_arn: str, task_arns: List[str]) -> bool:
|
|
62
141
|
"""
|
|
63
142
|
Checks if the ExecuteCommandAgent is running on the specified ECS task.
|
|
64
143
|
|
|
65
144
|
Waits for up to 25 attempts, then raises ECSAgentNotRunning if still not
|
|
66
145
|
running.
|
|
67
146
|
"""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
managed_agents =
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
class NoClusterException(ECSException):
|
|
99
|
-
def __init__(self, application_name: str, environment: str):
|
|
100
|
-
super().__init__(
|
|
101
|
-
f"""No ECS cluster found for "{application_name}" in "{environment}" environment."""
|
|
102
|
-
)
|
|
147
|
+
if not task_arns:
|
|
148
|
+
raise ValidationException("No task ARNs provided")
|
|
149
|
+
task_details = self.ecs_client.describe_tasks(cluster=cluster_arn, tasks=task_arns)
|
|
150
|
+
|
|
151
|
+
if not task_details["tasks"]:
|
|
152
|
+
raise ECSException("No ECS tasks returned.")
|
|
153
|
+
container_details = task_details["tasks"][0]["containers"][0]
|
|
154
|
+
if container_details.get("managedAgents", None):
|
|
155
|
+
managed_agents = container_details["managedAgents"]
|
|
156
|
+
else:
|
|
157
|
+
raise ECSException("No managed agent on ecs task.")
|
|
158
|
+
|
|
159
|
+
execute_command_agent = [
|
|
160
|
+
agent for agent in managed_agents if agent["name"] == "ExecuteCommandAgent"
|
|
161
|
+
]
|
|
162
|
+
if not execute_command_agent:
|
|
163
|
+
raise ECSException("No ExecuteCommandAgent on ecs task.")
|
|
164
|
+
return execute_command_agent[0]["lastStatus"] == "RUNNING"
|
|
165
|
+
|
|
166
|
+
@wait_until(
|
|
167
|
+
max_attempts=20,
|
|
168
|
+
message_on_false="ECS task did not register in time",
|
|
169
|
+
)
|
|
170
|
+
def wait_for_task_to_register(self, cluster_arn: str, task_family: str) -> list[str]:
|
|
171
|
+
task_arns = self.get_ecs_task_arns(cluster_arn, task_family)
|
|
172
|
+
if task_arns:
|
|
173
|
+
return task_arns
|
|
174
|
+
return False
|
|
@@ -1,14 +1,19 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
import click
|
|
2
4
|
|
|
3
5
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
4
6
|
|
|
7
|
+
DEBUG = os.environ.get("DEBUG", False)
|
|
8
|
+
|
|
5
9
|
|
|
6
10
|
class ClickIOProvider:
|
|
7
11
|
def warn(self, message: str):
|
|
8
12
|
click.secho(message, fg="magenta")
|
|
9
13
|
|
|
10
14
|
def debug(self, message: str):
|
|
11
|
-
|
|
15
|
+
if DEBUG == "True":
|
|
16
|
+
click.secho(message, fg="green")
|
|
12
17
|
|
|
13
18
|
def error(self, message: str):
|
|
14
19
|
click.secho(f"Error: {message}", fg="red")
|
|
@@ -140,7 +140,12 @@ class TerraformManifestProvider:
|
|
|
140
140
|
def _add_extensions_module(terraform: dict, platform_helper_version: str, env: str):
|
|
141
141
|
source = f"git::https://github.com/uktrade/platform-tools.git//terraform/extensions?depth=1&ref={platform_helper_version}"
|
|
142
142
|
terraform["module"] = {
|
|
143
|
-
"extensions": {
|
|
143
|
+
"extensions": {
|
|
144
|
+
"source": source,
|
|
145
|
+
"args": "${local.args}",
|
|
146
|
+
"environment": env,
|
|
147
|
+
"repos": "${local.codebase_pipeline_repos != null ? (distinct(values(local.codebase_pipeline_repos))) : null}",
|
|
148
|
+
}
|
|
144
149
|
}
|
|
145
150
|
|
|
146
151
|
@staticmethod
|
|
@@ -163,6 +168,7 @@ class TerraformManifestProvider:
|
|
|
163
168
|
"services": '${local.config["extensions"]}',
|
|
164
169
|
"env_config": "${local.env_config}",
|
|
165
170
|
},
|
|
171
|
+
"codebase_pipeline_repos": '${try({for k, v in local.config["codebase_pipelines"]: k => v.repository}, null)}',
|
|
166
172
|
}
|
|
167
173
|
|
|
168
174
|
@staticmethod
|
|
@@ -77,8 +77,8 @@ class VpcProvider:
|
|
|
77
77
|
return vpc_id
|
|
78
78
|
|
|
79
79
|
def _get_security_groups(self, app: str, env: str, vpc_id: str) -> list:
|
|
80
|
-
|
|
81
80
|
vpc_filter = {"Name": "vpc-id", "Values": [vpc_id]}
|
|
81
|
+
# TODO Handle terraformed environment SG https://uktrade.atlassian.net/browse/DBTP-2074
|
|
82
82
|
tag_filter = {"Name": f"tag:Name", "Values": [f"copilot-{app}-{env}-env"]}
|
|
83
83
|
response = self.ec2_client.describe_security_groups(Filters=[vpc_filter, tag_filter])
|
|
84
84
|
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import time
|
|
3
|
+
from typing import Callable
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
7
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
8
|
+
|
|
9
|
+
SECONDS_BEFORE_RETRY = 3
|
|
10
|
+
RETRY_MAX_ATTEMPTS = 3
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RetryException(PlatformException):
|
|
14
|
+
|
|
15
|
+
def __init__(
|
|
16
|
+
self, function_name: str, max_attempts: int, original_exception: Optional[Exception] = None
|
|
17
|
+
):
|
|
18
|
+
message = f"Function: {function_name} failed after {max_attempts} attempts"
|
|
19
|
+
self.original_exception = original_exception
|
|
20
|
+
if original_exception:
|
|
21
|
+
message += f": \n{str(original_exception)}"
|
|
22
|
+
super().__init__(message)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def retry(
|
|
26
|
+
exceptions_to_catch: tuple = (Exception,),
|
|
27
|
+
max_attempts: int = RETRY_MAX_ATTEMPTS,
|
|
28
|
+
delay: float = SECONDS_BEFORE_RETRY,
|
|
29
|
+
raise_custom_exception: bool = True,
|
|
30
|
+
custom_exception: type = RetryException,
|
|
31
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
32
|
+
):
|
|
33
|
+
def decorator(func):
|
|
34
|
+
func.__wrapped_by__ = "retry"
|
|
35
|
+
|
|
36
|
+
@functools.wraps(func)
|
|
37
|
+
def wrapper(*args, **kwargs):
|
|
38
|
+
last_exception = None
|
|
39
|
+
for attempt in range(max_attempts):
|
|
40
|
+
try:
|
|
41
|
+
return func(*args, **kwargs)
|
|
42
|
+
except exceptions_to_catch as e:
|
|
43
|
+
last_exception = e
|
|
44
|
+
io.debug(
|
|
45
|
+
f"Attempt {attempt+1}/{max_attempts} for {func.__name__} failed with exception {str(last_exception)}"
|
|
46
|
+
)
|
|
47
|
+
if attempt < max_attempts - 1:
|
|
48
|
+
time.sleep(delay)
|
|
49
|
+
if raise_custom_exception:
|
|
50
|
+
raise custom_exception(func.__name__, max_attempts, last_exception)
|
|
51
|
+
raise last_exception
|
|
52
|
+
|
|
53
|
+
return wrapper
|
|
54
|
+
|
|
55
|
+
return decorator
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def wait_until(
|
|
59
|
+
exceptions_to_catch: tuple = (PlatformException,),
|
|
60
|
+
max_attempts: int = RETRY_MAX_ATTEMPTS,
|
|
61
|
+
delay: float = SECONDS_BEFORE_RETRY,
|
|
62
|
+
raise_custom_exception: bool = True,
|
|
63
|
+
custom_exception=RetryException,
|
|
64
|
+
message_on_false="Condition not met",
|
|
65
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
66
|
+
):
|
|
67
|
+
"""Wrap a function which returns a boolean."""
|
|
68
|
+
|
|
69
|
+
def decorator(func: Callable[..., bool]):
|
|
70
|
+
func.__wrapped_by__ = "wait_until"
|
|
71
|
+
|
|
72
|
+
@functools.wraps(func)
|
|
73
|
+
def wrapper(*args, **kwargs):
|
|
74
|
+
last_exception = None
|
|
75
|
+
for attempt in range(max_attempts):
|
|
76
|
+
try:
|
|
77
|
+
result = func(*args, **kwargs)
|
|
78
|
+
if result:
|
|
79
|
+
return result
|
|
80
|
+
io.debug(
|
|
81
|
+
f"Attempt {attempt+1}/{max_attempts} for {func.__name__} returned falsy"
|
|
82
|
+
)
|
|
83
|
+
except exceptions_to_catch as e:
|
|
84
|
+
last_exception = e
|
|
85
|
+
io.debug(
|
|
86
|
+
f"Attempt {attempt+1}/{max_attempts} for {func.__name__} failed with exception {str(last_exception)}"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
if attempt < max_attempts - 1:
|
|
90
|
+
time.sleep(delay)
|
|
91
|
+
|
|
92
|
+
if not last_exception: # If func returns false set last_exception
|
|
93
|
+
last_exception = PlatformException(message_on_false)
|
|
94
|
+
if (
|
|
95
|
+
not raise_custom_exception
|
|
96
|
+
): # Raise last_exception when you don't want custom exception
|
|
97
|
+
raise last_exception
|
|
98
|
+
else:
|
|
99
|
+
raise custom_exception(func.__name__, max_attempts, last_exception)
|
|
100
|
+
|
|
101
|
+
return wrapper
|
|
102
|
+
|
|
103
|
+
return decorator
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dbt-platform-helper
|
|
3
|
-
Version: 15.1
|
|
3
|
+
Version: 15.2.1
|
|
4
4
|
Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Department for Business and Trade Platform Team
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
dbt_platform_helper/COMMANDS.md,sha256=szFwoNuKlrTfGv10jA0zLG_HDgRnzPaI4rBBSgFXtu8,23883
|
|
2
2
|
dbt_platform_helper/README.md,sha256=B0qN2_u_ASqqgkGDWY2iwNGZt_9tUgMb9XqtaTuzYjw,1530
|
|
3
3
|
dbt_platform_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
dbt_platform_helper/addon-plans.yml,sha256=O46a_ODsGG9KXmQY_1XbSGqrpSaHSLDe-SdROzHx8Go,4545
|
|
5
4
|
dbt_platform_helper/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
5
|
dbt_platform_helper/commands/application.py,sha256=OUQsahXXHSEKxmXAmK8fSy_bTLNwM_TdLuv6CvffRPk,10126
|
|
7
6
|
dbt_platform_helper/commands/codebase.py,sha256=oNlZcP2w3XE5YP-JVl0rdqoJuXUrfe1ELZ5xAdgPvBk,3166
|
|
@@ -18,24 +17,25 @@ dbt_platform_helper/commands/version.py,sha256=2GltWeeN7cqhVj9FhYWSbXSQSyNILHVNO
|
|
|
18
17
|
dbt_platform_helper/constants.py,sha256=Ao2uvVRcxRN5SXqvW6Jq2srd7LuyGz1jPy4fg2N6XSk,1153
|
|
19
18
|
dbt_platform_helper/default-extensions.yml,sha256=SU1ZitskbuEBpvE7efc3s56eAUF11j70brhj_XrNMMo,493
|
|
20
19
|
dbt_platform_helper/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
-
dbt_platform_helper/domain/codebase.py,sha256=
|
|
22
|
-
dbt_platform_helper/domain/conduit.py,sha256=
|
|
20
|
+
dbt_platform_helper/domain/codebase.py,sha256=2hJoBiDB2ciOudT_YUR44XV0ZQPWUJld_UIuds4XOt8,12481
|
|
21
|
+
dbt_platform_helper/domain/conduit.py,sha256=0aX5rhynkkJj8rJUwfyLENyCwlAI67_Vkky1lOEl6rw,12496
|
|
23
22
|
dbt_platform_helper/domain/config.py,sha256=Iyf-lV4YDD6BHH-RRaTvp-7qPS8BYeHM_SkSfeU7si4,13802
|
|
24
|
-
dbt_platform_helper/domain/copilot.py,sha256=
|
|
23
|
+
dbt_platform_helper/domain/copilot.py,sha256=g8W2LaskyhOvtNoCoNbwucGTrfdAzj-AJ0J98tgLbhA,15138
|
|
25
24
|
dbt_platform_helper/domain/copilot_environment.py,sha256=fL3XJCOfO0BJRCrCoBPFCcshrQoX1FeSYNTziOEaH4A,9093
|
|
26
25
|
dbt_platform_helper/domain/database_copy.py,sha256=AedcBTfKDod0OlMqVP6zb9c_9VIc3vqro0oUUhh7nwc,9497
|
|
27
26
|
dbt_platform_helper/domain/maintenance_page.py,sha256=0_dgM5uZvjVNBKcqScspjutinMh-7Hdm7jBEgUPujrk,14529
|
|
28
27
|
dbt_platform_helper/domain/notify.py,sha256=_BWj5znDWtrSdJ5xzDBgnao4ukliBA5wiUZGobIDyiI,1894
|
|
29
28
|
dbt_platform_helper/domain/pipelines.py,sha256=BUoXlV4pIKSw3Ry6oVMzd0mBU6tfl_tvqp-1zxHrQdk,6552
|
|
29
|
+
dbt_platform_helper/domain/plans.py,sha256=X5-jKGiJDVWn0CRH1k5aV74fTH0E41HqFQcCo5kB4hI,1160
|
|
30
30
|
dbt_platform_helper/domain/terraform_environment.py,sha256=kPfA44KCNnF_7ihQPuxaShLjEnVShrbruLwr5xoCeRc,1825
|
|
31
31
|
dbt_platform_helper/domain/versioning.py,sha256=pIL8VPAJHqX5kJBp3QIxII5vmUo4aIYW_U9u_KxUJd0,5494
|
|
32
|
-
dbt_platform_helper/entities/platform_config_schema.py,sha256=
|
|
32
|
+
dbt_platform_helper/entities/platform_config_schema.py,sha256=s7NiCKpI0WpwqEp3AgNTPC0J-0tgP7Ee2yvKC6CP9co,26665
|
|
33
33
|
dbt_platform_helper/entities/semantic_version.py,sha256=VgQ6V6OgSaleuVmMB8Kl_yLoakXl2auapJTDbK00mfc,2679
|
|
34
34
|
dbt_platform_helper/jinja2_tags.py,sha256=hKG6RS3zlxJHQ-Op9r2U2-MhWp4s3lZir4Ihe24ApJ0,540
|
|
35
|
-
dbt_platform_helper/platform_exception.py,sha256=
|
|
35
|
+
dbt_platform_helper/platform_exception.py,sha256=HGfCYRD20REsynqMKmyZndTfdkMd5dLSIEB2qGGCeP8,244
|
|
36
36
|
dbt_platform_helper/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
37
|
dbt_platform_helper/providers/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
|
-
dbt_platform_helper/providers/aws/exceptions.py,sha256=
|
|
38
|
+
dbt_platform_helper/providers/aws/exceptions.py,sha256=7zrbzGuZhZypmT7LB4pK7TrYD4kn5H0L_lgx07o2VPg,2332
|
|
39
39
|
dbt_platform_helper/providers/aws/interfaces.py,sha256=0JFggcUTJ8zERdxNVVpIiKvaaZeT2c-VECDG--MOi8E,285
|
|
40
40
|
dbt_platform_helper/providers/aws/opensearch.py,sha256=Qne2SoPllmacVSc7AxtjBlEbSBsRMbR_ySEkEymSF9k,581
|
|
41
41
|
dbt_platform_helper/providers/aws/redis.py,sha256=i3Kb00_BdqssjQg1wgZ-8GRXcEWQiORWnIEq6qkAXjQ,551
|
|
@@ -44,11 +44,11 @@ dbt_platform_helper/providers/cache.py,sha256=1hEwp0y9WYbEfgsp-RU9MyzIgCt1-4BxAp
|
|
|
44
44
|
dbt_platform_helper/providers/cloudformation.py,sha256=syMH6xc-ALRbsYQvlw9RcjX7c1MufFzwEdEzp_ucWig,5359
|
|
45
45
|
dbt_platform_helper/providers/config.py,sha256=8eK6txDTTF3s3iy7WxKszlaE33pHVhbJ55UDUJ9_nYw,9861
|
|
46
46
|
dbt_platform_helper/providers/config_validator.py,sha256=uF1GB-fl0ZuXVCtLNANgnY22UbiWZniBg1PiXgzGzuU,9923
|
|
47
|
-
dbt_platform_helper/providers/copilot.py,sha256=
|
|
48
|
-
dbt_platform_helper/providers/ecr.py,sha256=
|
|
49
|
-
dbt_platform_helper/providers/ecs.py,sha256=
|
|
47
|
+
dbt_platform_helper/providers/copilot.py,sha256=voFVGhvtOElulx6Cgd1KQGkybrg8v4oGkJTr_xRpF18,5582
|
|
48
|
+
dbt_platform_helper/providers/ecr.py,sha256=eYXSY1-pFN6F3Es1WSZgv3dmvX2oD-baqhHDO-QzgVg,4382
|
|
49
|
+
dbt_platform_helper/providers/ecs.py,sha256=4XRpOgcl7KFiTp9lhNrp4Lvmje0ZFYuUh9Z_eEqhyhA,6538
|
|
50
50
|
dbt_platform_helper/providers/files.py,sha256=cJdOV6Eupi-COmGUMxZMF10BZnMi3MCCipTVUnE_NPA,857
|
|
51
|
-
dbt_platform_helper/providers/io.py,sha256=
|
|
51
|
+
dbt_platform_helper/providers/io.py,sha256=5C7XUxy3XNqSWgxryr4Uy0l4J9np_lryGskqE0TRpmQ,1459
|
|
52
52
|
dbt_platform_helper/providers/kms.py,sha256=JR2EU3icXePoJCtr7QnqDPj1wWbyn5Uf9CRFq3_4lRs,647
|
|
53
53
|
dbt_platform_helper/providers/load_balancers.py,sha256=G-gqhthaO6ZmpKq6zAqnY1AUtc5YjnI99sQzpeaM0ec,10644
|
|
54
54
|
dbt_platform_helper/providers/parameter_store.py,sha256=klxDhcQ65Yc2KAc4Gf5P0vhpZOW7_vZalAVb-LLAA4s,1568
|
|
@@ -57,11 +57,11 @@ dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py,sha
|
|
|
57
57
|
dbt_platform_helper/providers/schema_migrator.py,sha256=qk14k3hMz1av9VrxHyJw2OKJLQnCBv_ugOoxZr3tFXQ,2854
|
|
58
58
|
dbt_platform_helper/providers/secrets.py,sha256=mOTIrcRRxxV2tS40U8onAjWekfPS3NzCvvyCMjr_yrU,5327
|
|
59
59
|
dbt_platform_helper/providers/slack_channel_notifier.py,sha256=G8etEcaBQSNHg8BnyC5UPv6l3vUB14cYWjcaAQksaEk,2135
|
|
60
|
-
dbt_platform_helper/providers/terraform_manifest.py,sha256=
|
|
60
|
+
dbt_platform_helper/providers/terraform_manifest.py,sha256=gfluve8mcHSkZqq4nNhazte48I7LFbJoQAVBMJBnNk4,9660
|
|
61
61
|
dbt_platform_helper/providers/validation.py,sha256=i2g-Mrd4hy_fGIfGa6ZQy4vTJ40OM44Fe_XpEifGWxs,126
|
|
62
62
|
dbt_platform_helper/providers/version.py,sha256=QNGrV5nyJi0JysXowYUU4OrXGDn27WmFezlV8benpdY,4251
|
|
63
63
|
dbt_platform_helper/providers/version_status.py,sha256=qafnhZrEc9k1cvXJpvJhkGj6WtkzcsoQhqS_Y6JXy48,929
|
|
64
|
-
dbt_platform_helper/providers/vpc.py,sha256=
|
|
64
|
+
dbt_platform_helper/providers/vpc.py,sha256=V8kXXzy-JuRpuzZhI9xyfcNky-eD42K0v_uM2WejLoo,3048
|
|
65
65
|
dbt_platform_helper/providers/yaml_file.py,sha256=LZ8eCPDQRr1wlck13My5hQa0eE2OVhSomm-pOIuZ9h0,2881
|
|
66
66
|
dbt_platform_helper/templates/.copilot/config.yml,sha256=J_bA9sCtBdCPBRImpCBRnYvhQd4vpLYIXIU-lq9vbkA,158
|
|
67
67
|
dbt_platform_helper/templates/.copilot/image_build_run.sh,sha256=adYucYXEB-kAgZNjTQo0T6EIAY8sh_xCEvVhWKKQ8mw,164
|
|
@@ -87,6 +87,7 @@ dbt_platform_helper/templates/svc/maintenance_pages/default.html,sha256=OTZ-qwwS
|
|
|
87
87
|
dbt_platform_helper/templates/svc/maintenance_pages/dmas-migration.html,sha256=qvI6tHuI0UQbMBCuvPgK1a_zLANB6w7KVo9N5d8r-i0,829
|
|
88
88
|
dbt_platform_helper/templates/svc/maintenance_pages/migration.html,sha256=GiQsOiuaMFb7jG5_wU3V7BMcByHBl9fOBgrNf8quYlw,783
|
|
89
89
|
dbt_platform_helper/templates/svc/overrides/cfn.patches.yml,sha256=W7-d017akuUq9kda64DQxazavcRcCPDjaAik6t1EZqM,742
|
|
90
|
+
dbt_platform_helper/utilities/decorators.py,sha256=rS6ohsuo0bc6fkZP98Qwaeh0c_v2MDqn9hCvqfoz2w8,3548
|
|
90
91
|
dbt_platform_helper/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
91
92
|
dbt_platform_helper/utils/application.py,sha256=d7Tg5odZMy9e3o4R0mmU19hrxJuIfS_ATDH4hY8zJvk,5480
|
|
92
93
|
dbt_platform_helper/utils/arn_parser.py,sha256=BaXzIxSOLdFmP_IfAxRq-0j-0Re1iCN7L4j2Zi5-CRQ,1304
|
|
@@ -97,8 +98,11 @@ dbt_platform_helper/utils/messages.py,sha256=nWA7BWLb7ND0WH5TejDN4OQUJSKYBxU4tyC
|
|
|
97
98
|
dbt_platform_helper/utils/template.py,sha256=g-Db-0I6a6diOHkgK1nYA0IxJSO4TRrjqOvlyeOR32o,950
|
|
98
99
|
dbt_platform_helper/utils/validation.py,sha256=W5jKC2zp5Q7cJ0PT57GB-s9FkJXrNt1jmWojXRFymcY,1187
|
|
99
100
|
platform_helper.py,sha256=_YNNGtMkH5BcpC_mQQYJrmlf2mt7lkxTYeH7ZgflPoA,1925
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
dbt_platform_helper-15.1.
|
|
104
|
-
dbt_platform_helper-15.1.
|
|
101
|
+
terraform/elasticache-redis/plans.yml,sha256=efJfkLuLC_5TwhLb9DalKHOuZFO79y6iei6Dg_tqKjI,1831
|
|
102
|
+
terraform/opensearch/plans.yml,sha256=lQbUSNMGfvUeDMcGx8mSwzGQhMJU3EZ4J4tPzPKaq6c,1471
|
|
103
|
+
terraform/postgres/plans.yml,sha256=plwCklW1VB_tNJFyUduRMZx9UANgiWH_7TGLWUaUEus,2553
|
|
104
|
+
dbt_platform_helper-15.2.1.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
|
|
105
|
+
dbt_platform_helper-15.2.1.dist-info/METADATA,sha256=65eTVvxQ1_HxDTh7aspHS24KzvbzNGMZfaOmJD6dP-4,3293
|
|
106
|
+
dbt_platform_helper-15.2.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
107
|
+
dbt_platform_helper-15.2.1.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
|
|
108
|
+
dbt_platform_helper-15.2.1.dist-info/RECORD,,
|