dbt-platform-helper 15.10.0__py3-none-any.whl → 15.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +0 -91
- dbt_platform_helper/commands/internal.py +111 -0
- dbt_platform_helper/constants.py +14 -0
- dbt_platform_helper/domain/conduit.py +1 -1
- dbt_platform_helper/domain/config.py +30 -1
- dbt_platform_helper/domain/maintenance_page.py +10 -8
- dbt_platform_helper/domain/service.py +270 -66
- dbt_platform_helper/domain/update_alb_rules.py +346 -0
- dbt_platform_helper/entities/platform_config_schema.py +0 -3
- dbt_platform_helper/entities/service.py +139 -13
- dbt_platform_helper/providers/aws/exceptions.py +5 -0
- dbt_platform_helper/providers/aws/sso_auth.py +14 -0
- dbt_platform_helper/providers/config.py +0 -11
- dbt_platform_helper/providers/ecs.py +104 -11
- dbt_platform_helper/providers/load_balancers.py +86 -8
- dbt_platform_helper/providers/logs.py +57 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/terraform_manifest.py +3 -5
- dbt_platform_helper/providers/yaml_file.py +13 -5
- {dbt_platform_helper-15.10.0.dist-info → dbt_platform_helper-15.11.0.dist-info}/METADATA +1 -1
- {dbt_platform_helper-15.10.0.dist-info → dbt_platform_helper-15.11.0.dist-info}/RECORD +25 -23
- platform_helper.py +0 -2
- dbt_platform_helper/commands/service.py +0 -53
- {dbt_platform_helper-15.10.0.dist-info → dbt_platform_helper-15.11.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-15.10.0.dist-info → dbt_platform_helper-15.11.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-15.10.0.dist-info → dbt_platform_helper-15.11.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from dataclasses import field
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import List
|
|
6
|
+
|
|
7
|
+
from dbt_platform_helper.constants import COPILOT_RULE_PRIORITY
|
|
8
|
+
from dbt_platform_helper.constants import DUMMY_RULE_REASON
|
|
9
|
+
from dbt_platform_helper.constants import MAINTENANCE_PAGE_REASON
|
|
10
|
+
from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
|
|
11
|
+
from dbt_platform_helper.constants import MANAGED_BY_SERVICE_TERRAFORM
|
|
12
|
+
from dbt_platform_helper.constants import PLATFORM_RULE_STARTING_PRIORITY
|
|
13
|
+
from dbt_platform_helper.constants import RULE_PRIORITY_INCREMENT
|
|
14
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
15
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
16
|
+
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
17
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
18
|
+
from dbt_platform_helper.providers.load_balancers import LoadBalancerProvider
|
|
19
|
+
from dbt_platform_helper.utils.application import load_application
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RollbackException(PlatformException):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class RuleType(Enum):
|
|
27
|
+
PLATFORM = "platform"
|
|
28
|
+
MAINTENANCE = "maintenance"
|
|
29
|
+
DEFAULT = "default"
|
|
30
|
+
COPILOT = "copilot"
|
|
31
|
+
MANUAL = "manual"
|
|
32
|
+
DUMMY = "dummy"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Deployment(Enum):
|
|
36
|
+
PLATFORM = "platform"
|
|
37
|
+
COPILOT = "copilot"
|
|
38
|
+
DUAL_DEPLOY_PLATFORM = "dual-deploy-platform-traffic"
|
|
39
|
+
DUAL_DEPLOY_COPILOT = "dual-deploy-copilot-traffic"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class OperationState:
|
|
44
|
+
created_rules: List[str] = field(default_factory=list)
|
|
45
|
+
deleted_rules: List[object] = field(default_factory=list)
|
|
46
|
+
listener_arn: str = ""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class UpdateALBRules:
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
session,
|
|
54
|
+
config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
|
|
55
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
56
|
+
load_application=load_application,
|
|
57
|
+
load_balancer_p: LoadBalancerProvider = LoadBalancerProvider,
|
|
58
|
+
):
|
|
59
|
+
self.config_provider = config_provider
|
|
60
|
+
self.io = io
|
|
61
|
+
self.load_application = load_application
|
|
62
|
+
self.load_balancer: LoadBalancerProvider = load_balancer_p(session, io=self.io)
|
|
63
|
+
|
|
64
|
+
def update_alb_rules(
|
|
65
|
+
self,
|
|
66
|
+
environment: str,
|
|
67
|
+
):
|
|
68
|
+
"""
|
|
69
|
+
Change ALB rules for a given environment.
|
|
70
|
+
|
|
71
|
+
Attempt to rollback the rules created/deleted if a failure occurs.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
operation_state = OperationState()
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
self._execute_rule_updates(environment, operation_state)
|
|
78
|
+
if operation_state.created_rules:
|
|
79
|
+
self.io.info(f"Created rules: {operation_state.created_rules}")
|
|
80
|
+
if operation_state.deleted_rules:
|
|
81
|
+
deleted_arns = [rule["RuleArn"] for rule in operation_state.deleted_rules]
|
|
82
|
+
self.io.info(f"Deleted rules: {deleted_arns}")
|
|
83
|
+
except Exception as e:
|
|
84
|
+
if operation_state.created_rules or operation_state.deleted_rules:
|
|
85
|
+
self.io.error(f"Error during rule update: {str(e)}")
|
|
86
|
+
self.io.warn("Rolling back")
|
|
87
|
+
self.io.info("Attempting to rollback changes ...")
|
|
88
|
+
try:
|
|
89
|
+
self._rollback_changes(operation_state)
|
|
90
|
+
except RollbackException as rollback_error:
|
|
91
|
+
raise PlatformException(f"Rollback failed: \n{str(rollback_error)}")
|
|
92
|
+
else:
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
def _execute_rule_updates(self, environment: str, operation_state: OperationState):
|
|
96
|
+
platform_config = self.config_provider.get_enriched_config()
|
|
97
|
+
|
|
98
|
+
application_name = platform_config.get("application", "")
|
|
99
|
+
application = self.load_application(app=application_name)
|
|
100
|
+
|
|
101
|
+
service_deployment_mode = (
|
|
102
|
+
platform_config.get("environments")
|
|
103
|
+
.get(environment, {})
|
|
104
|
+
.get("service-deployment-mode", Deployment.COPILOT.value)
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
self.io.info(f"Deployment Mode: {service_deployment_mode}")
|
|
108
|
+
|
|
109
|
+
listener_arn = self.load_balancer.get_https_listener_for_application(
|
|
110
|
+
application.name, environment
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
operation_state.listener_arn = listener_arn
|
|
114
|
+
|
|
115
|
+
self.io.debug(f"Listener ARN: {listener_arn}")
|
|
116
|
+
|
|
117
|
+
rules = self.load_balancer.get_rules_with_tags_by_listener_arn(listener_arn)
|
|
118
|
+
|
|
119
|
+
mapped_rules = {
|
|
120
|
+
rule_type: [rule for rule in rules if self._filter_rule_type(rule) == rule_type]
|
|
121
|
+
for rule_type in set(self._filter_rule_type(rule) for rule in rules)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if mapped_rules.get(RuleType.MANUAL.value, []):
|
|
125
|
+
rule_arns = [rule["RuleArn"] for rule in mapped_rules[RuleType.MANUAL.value]]
|
|
126
|
+
message = f"""The following rules have been created manually please review and if required set
|
|
127
|
+
the rules priority to the copilot range after priority: {COPILOT_RULE_PRIORITY}.\n
|
|
128
|
+
Rules: {rule_arns}"""
|
|
129
|
+
raise PlatformException(message)
|
|
130
|
+
|
|
131
|
+
if (
|
|
132
|
+
service_deployment_mode == Deployment.PLATFORM.value
|
|
133
|
+
or service_deployment_mode == Deployment.DUAL_DEPLOY_PLATFORM.value
|
|
134
|
+
):
|
|
135
|
+
|
|
136
|
+
if len(mapped_rules.get(RuleType.PLATFORM.value, [])) > 0 and len(
|
|
137
|
+
mapped_rules.get(RuleType.PLATFORM.value, [])
|
|
138
|
+
) != len(mapped_rules.get(RuleType.COPILOT.value, [])):
|
|
139
|
+
raise PlatformException("Platform rules are partially created, please review.")
|
|
140
|
+
|
|
141
|
+
if len(mapped_rules.get(RuleType.PLATFORM.value, [])) == len(
|
|
142
|
+
mapped_rules.get(RuleType.COPILOT.value, [])
|
|
143
|
+
):
|
|
144
|
+
self.io.info("Platform rules already exist, skipping creation")
|
|
145
|
+
return # early exit
|
|
146
|
+
|
|
147
|
+
grouped = dict()
|
|
148
|
+
|
|
149
|
+
service_mapped_tgs = self._get_tg_arns_for_platform_services(
|
|
150
|
+
application_name, environment
|
|
151
|
+
)
|
|
152
|
+
for copilot_rule in mapped_rules.get(RuleType.COPILOT.value, []):
|
|
153
|
+
rule_arn = copilot_rule["RuleArn"]
|
|
154
|
+
self.io.debug(f"Building platform rule for corresponding copilot rule: {rule_arn}")
|
|
155
|
+
sorted_hosts = sorted(copilot_rule["Conditions"].get("host-header", []))
|
|
156
|
+
# Depth represents the specificity of the path condition, allowing us to sort in decreasing complexity.
|
|
157
|
+
path_depth = max(
|
|
158
|
+
[
|
|
159
|
+
len([sub_path for sub_path in path.split("/") if sub_path])
|
|
160
|
+
for path in copilot_rule["Conditions"].get("path-pattern", [])
|
|
161
|
+
]
|
|
162
|
+
)
|
|
163
|
+
list_conditions = [
|
|
164
|
+
{"Field": key, "Values": value}
|
|
165
|
+
for key, value in copilot_rule["Conditions"].items()
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
service_name = self._get_service_from_tg(copilot_rule)
|
|
169
|
+
|
|
170
|
+
tg_arn = service_mapped_tgs[service_name]
|
|
171
|
+
|
|
172
|
+
actions = self._create_new_actions(copilot_rule["Actions"], tg_arn)
|
|
173
|
+
self.io.debug(f"Updated forward action for service {service_name} to use: {tg_arn}")
|
|
174
|
+
if grouped.get(",".join(sorted_hosts)):
|
|
175
|
+
grouped[",".join(sorted_hosts)].append(
|
|
176
|
+
{
|
|
177
|
+
"copilot_rule": rule_arn,
|
|
178
|
+
"actions": actions,
|
|
179
|
+
"conditions": list_conditions,
|
|
180
|
+
"path_depth": path_depth,
|
|
181
|
+
"service_name": service_name,
|
|
182
|
+
}
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
grouped[",".join(sorted_hosts)] = [
|
|
186
|
+
{
|
|
187
|
+
"copilot_rule": rule_arn,
|
|
188
|
+
"actions": actions,
|
|
189
|
+
"conditions": list_conditions,
|
|
190
|
+
"path_depth": path_depth,
|
|
191
|
+
"service_name": service_name,
|
|
192
|
+
}
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
rule_priority = PLATFORM_RULE_STARTING_PRIORITY
|
|
196
|
+
for hosts, rules in grouped.items():
|
|
197
|
+
rules.sort(key=lambda x: x["path_depth"], reverse=True)
|
|
198
|
+
|
|
199
|
+
for rule in rules:
|
|
200
|
+
# Create rule with priority
|
|
201
|
+
copilot_rule = rule.get("copilot_rule", "")
|
|
202
|
+
self.io.debug(
|
|
203
|
+
f"Creating platform rule for corresponding copilot rule: {copilot_rule}"
|
|
204
|
+
)
|
|
205
|
+
rule_arn = self.load_balancer.create_rule(
|
|
206
|
+
listener_arn,
|
|
207
|
+
rule["actions"],
|
|
208
|
+
rule["conditions"],
|
|
209
|
+
rule_priority,
|
|
210
|
+
tags=[
|
|
211
|
+
{"Key": "application", "Value": application_name},
|
|
212
|
+
{"Key": "environment", "Value": environment},
|
|
213
|
+
{"Key": "service", "Value": rule["service_name"]},
|
|
214
|
+
{"Key": "reason", "Value": "service"},
|
|
215
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
216
|
+
],
|
|
217
|
+
)["Rules"][0]["RuleArn"]
|
|
218
|
+
operation_state.created_rules.append(rule_arn)
|
|
219
|
+
rule_priority += RULE_PRIORITY_INCREMENT
|
|
220
|
+
|
|
221
|
+
next_thousandth = lambda x: ((x // 1000) + 1) * 1000
|
|
222
|
+
rule_priority = next_thousandth(rule_priority)
|
|
223
|
+
|
|
224
|
+
if (
|
|
225
|
+
service_deployment_mode == Deployment.COPILOT.value
|
|
226
|
+
or service_deployment_mode == Deployment.DUAL_DEPLOY_COPILOT.value
|
|
227
|
+
):
|
|
228
|
+
self._delete_rules(mapped_rules.get(RuleType.PLATFORM.value, []), operation_state)
|
|
229
|
+
|
|
230
|
+
def _delete_rules(self, rules: List[dict], operation_state: OperationState):
|
|
231
|
+
for rule in rules:
|
|
232
|
+
rule_arn = rule["RuleArn"]
|
|
233
|
+
try:
|
|
234
|
+
self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
|
|
235
|
+
operation_state.deleted_rules.append(rule)
|
|
236
|
+
self.io.debug(f"Deleted existing rule: {rule_arn}")
|
|
237
|
+
except Exception as e:
|
|
238
|
+
self.io.error(f"Failed to delete existing rule {rule_arn}: {str(e)}")
|
|
239
|
+
raise
|
|
240
|
+
|
|
241
|
+
def _filter_rule_type(self, rule: dict) -> str:
|
|
242
|
+
if rule["Tags"]:
|
|
243
|
+
if rule["Tags"].get("managed-by", "") == MANAGED_BY_PLATFORM:
|
|
244
|
+
return RuleType.PLATFORM.value
|
|
245
|
+
if rule["Tags"].get("reason", None) == MAINTENANCE_PAGE_REASON:
|
|
246
|
+
return RuleType.MAINTENANCE.value
|
|
247
|
+
if rule["Tags"].get("reason", None) == DUMMY_RULE_REASON:
|
|
248
|
+
return RuleType.DUMMY.value
|
|
249
|
+
|
|
250
|
+
if rule["Priority"] == "default":
|
|
251
|
+
return RuleType.DEFAULT.value
|
|
252
|
+
if int(rule["Priority"]) >= COPILOT_RULE_PRIORITY:
|
|
253
|
+
return RuleType.COPILOT.value
|
|
254
|
+
|
|
255
|
+
return RuleType.MANUAL.value
|
|
256
|
+
|
|
257
|
+
def _get_service_from_tg(self, rule: dict) -> str:
|
|
258
|
+
target_group_arn = None
|
|
259
|
+
|
|
260
|
+
for action in rule["Actions"]:
|
|
261
|
+
if action["Type"] == "forward":
|
|
262
|
+
target_group_arn = action["TargetGroupArn"]
|
|
263
|
+
|
|
264
|
+
if target_group_arn:
|
|
265
|
+
try:
|
|
266
|
+
tgs = self.load_balancer.get_target_groups_with_tags([target_group_arn])
|
|
267
|
+
return tgs[0]["Tags"].get("copilot-service", None)
|
|
268
|
+
except IndexError:
|
|
269
|
+
raise PlatformException(f"No target group found for arn: {target_group_arn}")
|
|
270
|
+
else:
|
|
271
|
+
rule_arn = rule["RuleArn"]
|
|
272
|
+
raise PlatformException(f"No target group arn found in rule: {rule_arn}")
|
|
273
|
+
|
|
274
|
+
def _get_tg_arns_for_platform_services(
|
|
275
|
+
self, application: str, environment: str, managed_by: str = MANAGED_BY_SERVICE_TERRAFORM
|
|
276
|
+
) -> dict:
|
|
277
|
+
tgs = self.load_balancer.get_target_groups_with_tags([])
|
|
278
|
+
service_mapped_tgs = {}
|
|
279
|
+
for tg in tgs:
|
|
280
|
+
if (
|
|
281
|
+
tg["Tags"].get("application", "") == application
|
|
282
|
+
and tg["Tags"].get("environment", "") == environment
|
|
283
|
+
and tg["Tags"].get("managed-by", "") == managed_by
|
|
284
|
+
):
|
|
285
|
+
if tg["Tags"].get("service", ""):
|
|
286
|
+
service_mapped_tgs[tg["Tags"].get("service")] = tg["TargetGroupArn"]
|
|
287
|
+
else:
|
|
288
|
+
tg_name = tg["TargetGroupName"]
|
|
289
|
+
self.io.warn(f"Target group {tg_name} has no 'service' tag")
|
|
290
|
+
return service_mapped_tgs
|
|
291
|
+
|
|
292
|
+
def _create_new_actions(self, actions: dict, tg_arn: str) -> dict:
|
|
293
|
+
|
|
294
|
+
updated_actions = copy.deepcopy(actions)
|
|
295
|
+
for action in updated_actions:
|
|
296
|
+
if action.get("Type") == "forward" and "TargetGroupArn" in action:
|
|
297
|
+
action["TargetGroupArn"] = tg_arn
|
|
298
|
+
for tg in action["ForwardConfig"]["TargetGroups"]:
|
|
299
|
+
tg["TargetGroupArn"] = tg_arn
|
|
300
|
+
return updated_actions
|
|
301
|
+
|
|
302
|
+
def _rollback_changes(self, operation_state: OperationState) -> bool:
|
|
303
|
+
rollback_errors = []
|
|
304
|
+
delete_rollbacks = []
|
|
305
|
+
create_rollbacks = []
|
|
306
|
+
for rule_arn in operation_state.created_rules:
|
|
307
|
+
try:
|
|
308
|
+
self.io.debug(f"Rolling back: Deleting created rule {rule_arn}")
|
|
309
|
+
self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
|
|
310
|
+
delete_rollbacks.append(rule_arn)
|
|
311
|
+
except Exception as e:
|
|
312
|
+
error_msg = f"Failed to delete rule {rule_arn} during rollback: {str(e)}"
|
|
313
|
+
rollback_errors.append(error_msg)
|
|
314
|
+
|
|
315
|
+
for rule_snapshot in operation_state.deleted_rules:
|
|
316
|
+
rule_arn = rule_snapshot["RuleArn"]
|
|
317
|
+
try:
|
|
318
|
+
self.io.debug(f"Rolling back: Recreating deleted rule {rule_arn}")
|
|
319
|
+
create_rollbacks.append(
|
|
320
|
+
self.load_balancer.create_rule(
|
|
321
|
+
operation_state.listener_arn,
|
|
322
|
+
actions=rule_snapshot["Actions"],
|
|
323
|
+
conditions=[
|
|
324
|
+
{"Field": key, "Values": value}
|
|
325
|
+
for key, value in rule_snapshot["Conditions"].items()
|
|
326
|
+
],
|
|
327
|
+
priority=int(rule_snapshot["Priority"]),
|
|
328
|
+
tags=[
|
|
329
|
+
{"Key": key, "Value": value}
|
|
330
|
+
for key, value in rule_snapshot["Tags"].items()
|
|
331
|
+
],
|
|
332
|
+
)["Rules"][0]["RuleArn"]
|
|
333
|
+
)
|
|
334
|
+
except Exception as e:
|
|
335
|
+
error_msg = f"Failed to recreate rule {rule_arn} during rollback: {str(e)}"
|
|
336
|
+
rollback_errors.append(error_msg)
|
|
337
|
+
|
|
338
|
+
if rollback_errors:
|
|
339
|
+
self.io.warn("Some rollback operations failed. Manual intervention may be required.")
|
|
340
|
+
errors = "\n".join(rollback_errors)
|
|
341
|
+
raise RollbackException(f"Rollback partially failed: {errors}")
|
|
342
|
+
else:
|
|
343
|
+
self.io.info("Rollback completed successfully")
|
|
344
|
+
self.io.info(
|
|
345
|
+
f"Rolledback rules by creating: {create_rollbacks} \n and deleting {delete_rollbacks}"
|
|
346
|
+
)
|
|
@@ -321,9 +321,6 @@ class PlatformConfigSchema:
|
|
|
321
321
|
Optional("deletion_protection"): bool,
|
|
322
322
|
Optional("multi_az"): bool,
|
|
323
323
|
Optional("storage_type"): _valid_postgres_storage_types,
|
|
324
|
-
Optional("backup_retention_days"): PlatformConfigSchema.is_integer_between(
|
|
325
|
-
1, 35
|
|
326
|
-
),
|
|
327
324
|
}
|
|
328
325
|
},
|
|
329
326
|
Optional("database_copy"): [_valid_postgres_database_copy],
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import re
|
|
1
2
|
from typing import ClassVar
|
|
2
3
|
from typing import Dict
|
|
3
4
|
from typing import Optional
|
|
@@ -5,6 +6,10 @@ from typing import Union
|
|
|
5
6
|
|
|
6
7
|
from pydantic import BaseModel
|
|
7
8
|
from pydantic import Field
|
|
9
|
+
from pydantic import field_validator
|
|
10
|
+
from pydantic import model_validator
|
|
11
|
+
|
|
12
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
8
13
|
|
|
9
14
|
|
|
10
15
|
class HealthCheck(BaseModel):
|
|
@@ -21,14 +26,22 @@ class HealthCheck(BaseModel):
|
|
|
21
26
|
timeout: Optional[str] = Field(
|
|
22
27
|
description="""The timeout for a healthcheck call""", default=None
|
|
23
28
|
)
|
|
24
|
-
grace_period: Optional[str] = Field(
|
|
29
|
+
grace_period: Optional[str] = Field(
|
|
30
|
+
description="""The time the service ignores unhealthy ALB and container health checks""",
|
|
31
|
+
default=None,
|
|
32
|
+
)
|
|
25
33
|
|
|
26
34
|
|
|
27
35
|
class Http(BaseModel):
|
|
36
|
+
path: str = Field(description="""Requests to this path will be forwarded to your service.""")
|
|
37
|
+
target_container: str = Field(description="""Target container for the requests""")
|
|
38
|
+
healthcheck: Optional[HealthCheck] = Field(default=None)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class HttpOverride(BaseModel):
|
|
28
42
|
path: Optional[str] = Field(
|
|
29
43
|
description="""Requests to this path will be forwarded to your service.""", default=None
|
|
30
44
|
)
|
|
31
|
-
alb: Optional[str] = Field(default=None)
|
|
32
45
|
target_container: Optional[str] = Field(
|
|
33
46
|
description="""Target container for the requests""", default=None
|
|
34
47
|
)
|
|
@@ -52,35 +65,135 @@ class SidecarOverride(BaseModel):
|
|
|
52
65
|
|
|
53
66
|
|
|
54
67
|
class Image(BaseModel):
|
|
55
|
-
|
|
56
|
-
location: Optional[str] = Field(default=None)
|
|
68
|
+
location: str = Field()
|
|
57
69
|
port: Optional[int] = Field(default=None)
|
|
70
|
+
depends_on: Optional[dict[str, str]] = Field(default=None)
|
|
58
71
|
|
|
59
72
|
|
|
60
73
|
class VPC(BaseModel):
|
|
61
|
-
placement: str = Field()
|
|
74
|
+
placement: Optional[str] = Field(default=None)
|
|
62
75
|
|
|
63
76
|
|
|
64
77
|
class Network(BaseModel):
|
|
65
|
-
connect: bool = Field(
|
|
66
|
-
description="Enable Service Connect for intra-environment traffic between services."
|
|
78
|
+
connect: Optional[bool] = Field(
|
|
79
|
+
description="Enable Service Connect for intra-environment traffic between services.",
|
|
80
|
+
default=None,
|
|
67
81
|
)
|
|
68
82
|
vpc: Optional[VPC] = Field(default=None)
|
|
69
83
|
|
|
70
84
|
|
|
71
85
|
class Storage(BaseModel):
|
|
72
|
-
readonly_fs: bool
|
|
86
|
+
readonly_fs: Optional[bool] = Field(default=None)
|
|
87
|
+
writable_directories: Optional[list[str]] = Field(default=None)
|
|
88
|
+
|
|
89
|
+
@field_validator("writable_directories", mode="after")
|
|
90
|
+
@classmethod
|
|
91
|
+
def has_leading_forward_slash(cls, value: Union[list, None]) -> Union[list, None]:
|
|
92
|
+
if value is not None:
|
|
93
|
+
for path in value:
|
|
94
|
+
if not path.startswith("/"):
|
|
95
|
+
raise PlatformException(
|
|
96
|
+
"All writable directory paths must be absolute (starts with a /)"
|
|
97
|
+
)
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Cooldown(BaseModel):
|
|
102
|
+
in_: int = Field(
|
|
103
|
+
alias="in",
|
|
104
|
+
description="Number of seconds to wait before scaling in (down) after a drop in load.",
|
|
105
|
+
) # Can't use 'in' because it's a reserved keyword
|
|
106
|
+
out: int = Field(
|
|
107
|
+
description="Number of seconds to wait before scaling out (up) after a spike in load."
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
@field_validator("in_", "out", mode="before")
|
|
111
|
+
@classmethod
|
|
112
|
+
def parse_seconds(cls, value):
|
|
113
|
+
if isinstance(value, str) and value.endswith("s"):
|
|
114
|
+
value = value.removesuffix("s") # remove the trailing 's'
|
|
115
|
+
try:
|
|
116
|
+
return int(value)
|
|
117
|
+
except (ValueError, TypeError):
|
|
118
|
+
raise PlatformException("Cooldown values must be integers or strings like '30s'")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class CpuPercentage(BaseModel):
|
|
122
|
+
value: int = Field(description="Target CPU utilisation percentage that triggers autoscaling.")
|
|
123
|
+
cooldown: Optional[Cooldown] = Field(
|
|
124
|
+
default=None, description="Optional CPU cooldown that overrides the global cooldown policy."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class MemoryPercentage(BaseModel):
|
|
129
|
+
value: int = Field(description="Target CPU utilisation percentage that triggers autoscaling.")
|
|
130
|
+
cooldown: Optional[Cooldown] = Field(
|
|
131
|
+
default=None,
|
|
132
|
+
description="Optional memory cooldown that overrides the global cooldown policy.",
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class RequestsPerMinute(BaseModel):
|
|
137
|
+
value: int = Field(
|
|
138
|
+
description="Number of incoming requests per minute that triggers autoscaling."
|
|
139
|
+
)
|
|
140
|
+
cooldown: Optional[Cooldown] = Field(
|
|
141
|
+
default=None,
|
|
142
|
+
description="Optional requests cooldown that overrides the global cooldown policy.",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Count(BaseModel):
|
|
147
|
+
range: str = Field(
|
|
148
|
+
description="Minimum and maximum number of ECS tasks to maintain e.g. '1-2'."
|
|
149
|
+
)
|
|
150
|
+
cooldown: Optional[Cooldown] = Field(
|
|
151
|
+
default=None,
|
|
152
|
+
description="Global cooldown applied to all autoscaling metrics unless overridden per metric.",
|
|
153
|
+
)
|
|
154
|
+
cpu_percentage: Optional[Union[int, CpuPercentage]] = Field(
|
|
155
|
+
default=None,
|
|
156
|
+
description="CPU utilisation threshold (0–100). Either a plain integer or a map with 'value' and 'cooldown'.",
|
|
157
|
+
)
|
|
158
|
+
memory_percentage: Optional[Union[int, MemoryPercentage]] = Field(
|
|
159
|
+
default=None,
|
|
160
|
+
description="Memory utilisation threshold (0–100). Either a plain integer or a map with 'value' and 'cooldown'.",
|
|
161
|
+
)
|
|
162
|
+
requests_per_minute: Optional[Union[int, RequestsPerMinute]] = Field(
|
|
163
|
+
default=None,
|
|
164
|
+
description="Request-rate threshold. Either a plain integer or a map with 'value' and 'cooldown'.",
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
@model_validator(mode="after")
|
|
168
|
+
def at_least_one_autoscaling_metric(self):
|
|
169
|
+
|
|
170
|
+
if not any([self.cpu_percentage, self.memory_percentage, self.requests_per_minute]):
|
|
171
|
+
raise PlatformException(
|
|
172
|
+
"If autoscaling is enabled, you must define at least one metric: "
|
|
173
|
+
"cpu_percentage, memory_percentage, or requests_per_minute"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
if not re.match(r"^(\d+)-(\d+)$", self.range):
|
|
177
|
+
raise PlatformException("Range must be in the format 'int-int' e.g. '1-2'")
|
|
178
|
+
|
|
179
|
+
range_split = self.range.split("-")
|
|
180
|
+
if range_split[0] >= range_split[1]:
|
|
181
|
+
raise PlatformException("Range minimum value must be less than the maximum value.")
|
|
182
|
+
|
|
183
|
+
return self
|
|
73
184
|
|
|
74
185
|
|
|
75
186
|
class ServiceConfigEnvironmentOverride(BaseModel):
|
|
76
|
-
http: Optional[
|
|
187
|
+
http: Optional[HttpOverride] = Field(default=None)
|
|
77
188
|
sidecars: Optional[Dict[str, SidecarOverride]] = Field(default=None)
|
|
78
189
|
image: Optional[Image] = Field(default=None)
|
|
79
190
|
|
|
80
191
|
cpu: Optional[int] = Field(default=None)
|
|
81
192
|
memory: Optional[int] = Field(default=None)
|
|
82
|
-
count: Optional[int] = Field(default=None)
|
|
193
|
+
count: Optional[Union[int, Count]] = Field(default=None)
|
|
83
194
|
exec: Optional[bool] = Field(default=None)
|
|
195
|
+
entrypoint: Optional[list[str]] = Field(default=None)
|
|
196
|
+
essential: Optional[bool] = Field(default=None)
|
|
84
197
|
network: Optional[Network] = Field(default=None)
|
|
85
198
|
|
|
86
199
|
storage: Optional[Storage] = Field(default=None)
|
|
@@ -93,14 +206,27 @@ class ServiceConfig(BaseModel):
|
|
|
93
206
|
name: str = Field(description="""Name of the Service.""")
|
|
94
207
|
type: str = Field(description="""The type of service""")
|
|
95
208
|
|
|
96
|
-
http: Optional[Http] = Field(default=None)
|
|
209
|
+
http: Optional[Http] = Field(default=None)
|
|
210
|
+
|
|
211
|
+
@model_validator(mode="after")
|
|
212
|
+
def check_http_for_web_service(self):
|
|
213
|
+
if self.type == "Load Balanced Web Service" and self.http is None:
|
|
214
|
+
raise PlatformException(
|
|
215
|
+
"A 'http' block must be provided when service type == 'Load Balanced Web Service'"
|
|
216
|
+
)
|
|
217
|
+
return self
|
|
218
|
+
|
|
97
219
|
sidecars: Optional[Dict[str, Sidecar]] = Field(default=None)
|
|
98
220
|
image: Image = Field()
|
|
99
221
|
|
|
100
222
|
cpu: int = Field()
|
|
101
223
|
memory: int = Field()
|
|
102
|
-
count: int = Field(
|
|
224
|
+
count: Union[int, Count] = Field(
|
|
225
|
+
description="Desired task count — either a fixed integer or an autoscaling policy map with 'range', 'cooldown', and at least one of 'cpu_percentage', 'memory_percentage', or 'requests_per_minute' metrics."
|
|
226
|
+
)
|
|
103
227
|
exec: Optional[bool] = Field(default=None)
|
|
228
|
+
entrypoint: Optional[list[str]] = Field(default=None)
|
|
229
|
+
essential: Optional[bool] = Field(default=None)
|
|
104
230
|
network: Optional[Network] = Field(default=None)
|
|
105
231
|
|
|
106
232
|
storage: Optional[Storage] = Field(default=None)
|
|
@@ -110,5 +236,5 @@ class ServiceConfig(BaseModel):
|
|
|
110
236
|
# Environment overrides can override almost the full config
|
|
111
237
|
environments: Optional[Dict[str, ServiceConfigEnvironmentOverride]] = Field(default=None)
|
|
112
238
|
|
|
113
|
-
# Class based variable used when handling the
|
|
239
|
+
# Class based variable used when handling the object
|
|
114
240
|
local_terraform_source: ClassVar[str] = "../../../../../platform-tools/terraform/ecs-service"
|
|
@@ -63,3 +63,8 @@ class CreateAccessTokenException(AWSException):
|
|
|
63
63
|
class UnableToRetrieveSSOAccountList(AWSException):
|
|
64
64
|
def __init__(self):
|
|
65
65
|
super().__init__("Unable to retrieve AWS SSO account list")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class UnableToRetrieveSSOAccountRolesList(AWSException):
|
|
69
|
+
def __init__(self, account_id: str):
|
|
70
|
+
super().__init__(f"Unable to retrieve AWS SSO roles list for AWS account {account_id}")
|
|
@@ -3,6 +3,9 @@ from boto3 import Session
|
|
|
3
3
|
|
|
4
4
|
from dbt_platform_helper.providers.aws.exceptions import CreateAccessTokenException
|
|
5
5
|
from dbt_platform_helper.providers.aws.exceptions import UnableToRetrieveSSOAccountList
|
|
6
|
+
from dbt_platform_helper.providers.aws.exceptions import (
|
|
7
|
+
UnableToRetrieveSSOAccountRolesList,
|
|
8
|
+
)
|
|
6
9
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
7
10
|
|
|
8
11
|
|
|
@@ -55,6 +58,17 @@ class SSOAuthProvider:
|
|
|
55
58
|
raise UnableToRetrieveSSOAccountList()
|
|
56
59
|
return aws_accounts_response.get("accountList")
|
|
57
60
|
|
|
61
|
+
def list_account_roles(self, access_token, account_id, max_results=100):
|
|
62
|
+
aws_account_roles_response = self.sso.list_account_roles(
|
|
63
|
+
accessToken=access_token,
|
|
64
|
+
accountId=account_id,
|
|
65
|
+
maxResults=max_results,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
if len(aws_account_roles_response.get("roleList", [])) == 0:
|
|
69
|
+
raise UnableToRetrieveSSOAccountRolesList(account_id=account_id)
|
|
70
|
+
return aws_account_roles_response.get("roleList")
|
|
71
|
+
|
|
58
72
|
def _get_client(self, client: str):
|
|
59
73
|
if not self.session:
|
|
60
74
|
self.session = get_aws_session_or_abort()
|
|
@@ -30,17 +30,6 @@ class ConfigLoader:
|
|
|
30
30
|
self.io = io
|
|
31
31
|
self.file_provider = file_provider
|
|
32
32
|
|
|
33
|
-
def load_into_model(self, path, model):
|
|
34
|
-
try:
|
|
35
|
-
file_content = self.file_provider.load(path)
|
|
36
|
-
return model(**file_content)
|
|
37
|
-
except FileNotFoundException as e:
|
|
38
|
-
self.io.abort_with_error(
|
|
39
|
-
f"{e} Please check it exists and you are in the root directory of your -deploy repository."
|
|
40
|
-
)
|
|
41
|
-
except FileProviderException as e:
|
|
42
|
-
self.io.abort_with_error(f"Error loading configuration from {path}: {e}")
|
|
43
|
-
|
|
44
33
|
def load(self, path):
|
|
45
34
|
try:
|
|
46
35
|
file_content = self.file_provider.load(path)
|