dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +107 -27
- dbt_platform_helper/commands/application.py +5 -6
- dbt_platform_helper/commands/codebase.py +31 -10
- dbt_platform_helper/commands/conduit.py +3 -5
- dbt_platform_helper/commands/config.py +20 -311
- dbt_platform_helper/commands/copilot.py +18 -391
- dbt_platform_helper/commands/database.py +17 -9
- dbt_platform_helper/commands/environment.py +20 -14
- dbt_platform_helper/commands/generate.py +0 -3
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/notify.py +58 -78
- dbt_platform_helper/commands/pipeline.py +23 -19
- dbt_platform_helper/commands/secrets.py +39 -93
- dbt_platform_helper/commands/version.py +7 -12
- dbt_platform_helper/constants.py +52 -7
- dbt_platform_helper/domain/codebase.py +89 -39
- dbt_platform_helper/domain/conduit.py +335 -76
- dbt_platform_helper/domain/config.py +381 -0
- dbt_platform_helper/domain/copilot.py +398 -0
- dbt_platform_helper/domain/copilot_environment.py +8 -8
- dbt_platform_helper/domain/database_copy.py +2 -2
- dbt_platform_helper/domain/maintenance_page.py +254 -430
- dbt_platform_helper/domain/notify.py +64 -0
- dbt_platform_helper/domain/pipelines.py +43 -35
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +14 -13
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +249 -0
- dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
- dbt_platform_helper/entities/semantic_version.py +83 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/__init__.py +0 -0
- dbt_platform_helper/providers/aws/exceptions.py +70 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/aws/sso_auth.py +75 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/cloudformation.py +1 -1
- dbt_platform_helper/providers/config.py +137 -19
- dbt_platform_helper/providers/config_validator.py +112 -51
- dbt_platform_helper/providers/copilot.py +24 -16
- dbt_platform_helper/providers/ecr.py +89 -7
- dbt_platform_helper/providers/ecs.py +228 -36
- dbt_platform_helper/providers/environment_variable.py +24 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +36 -4
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +402 -42
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +134 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
- dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
- dbt_platform_helper/providers/schema_migrator.py +77 -0
- dbt_platform_helper/providers/secrets.py +5 -5
- dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
- dbt_platform_helper/providers/terraform_manifest.py +121 -19
- dbt_platform_helper/providers/version.py +106 -23
- dbt_platform_helper/providers/version_status.py +27 -0
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +58 -2
- dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utilities/decorators.py +103 -0
- dbt_platform_helper/utils/application.py +119 -22
- dbt_platform_helper/utils/aws.py +39 -150
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -14
- dbt_platform_helper/utils/validation.py +1 -1
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
- dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +3 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- dbt_platform_helper/providers/aws.py +0 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/providers/semantic_version.py +0 -126
- dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
- dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
- dbt_platform_helper/utils/cloudfoundry.py +0 -14
- dbt_platform_helper/utils/files.py +0 -53
- dbt_platform_helper/utils/manifests.py +0 -18
- dbt_platform_helper/utils/versioning.py +0 -238
- dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from dataclasses import field
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import List
|
|
6
|
+
|
|
7
|
+
from botocore.exceptions import ClientError
|
|
8
|
+
|
|
9
|
+
from dbt_platform_helper.constants import COPILOT_RULE_PRIORITY
|
|
10
|
+
from dbt_platform_helper.constants import DUMMY_RULE_REASON
|
|
11
|
+
from dbt_platform_helper.constants import HTTP_SERVICE_TYPES
|
|
12
|
+
from dbt_platform_helper.constants import MAINTENANCE_PAGE_REASON
|
|
13
|
+
from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
|
|
14
|
+
from dbt_platform_helper.constants import MANAGED_BY_SERVICE_TERRAFORM
|
|
15
|
+
from dbt_platform_helper.constants import PLATFORM_RULE_STARTING_PRIORITY
|
|
16
|
+
from dbt_platform_helper.constants import RULE_PRIORITY_INCREMENT
|
|
17
|
+
from dbt_platform_helper.domain.service import ServiceManager
|
|
18
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
19
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
20
|
+
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
21
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
22
|
+
from dbt_platform_helper.providers.load_balancers import LoadBalancerProvider
|
|
23
|
+
from dbt_platform_helper.utils.application import load_application
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class RollbackException(PlatformException):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RuleType(Enum):
|
|
31
|
+
PLATFORM = "platform"
|
|
32
|
+
MAINTENANCE = "maintenance"
|
|
33
|
+
DEFAULT = "default"
|
|
34
|
+
COPILOT = "copilot"
|
|
35
|
+
MANUAL = "manual"
|
|
36
|
+
DUMMY = "dummy"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Deployment(Enum):
|
|
40
|
+
PLATFORM = "platform"
|
|
41
|
+
COPILOT = "copilot"
|
|
42
|
+
DUAL_DEPLOY_PLATFORM = "dual-deploy-platform-traffic"
|
|
43
|
+
DUAL_DEPLOY_COPILOT = "dual-deploy-copilot-traffic"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class OperationState:
|
|
48
|
+
created_rules: List[object] = field(default_factory=list)
|
|
49
|
+
deleted_rules: List[object] = field(default_factory=list)
|
|
50
|
+
updated_rules: List[object] = field(default_factory=list)
|
|
51
|
+
listener_arn: str = ""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class UpdateALBRules:
|
|
55
|
+
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
session,
|
|
59
|
+
config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
|
|
60
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
61
|
+
load_application=load_application,
|
|
62
|
+
load_balancer_p: LoadBalancerProvider = LoadBalancerProvider,
|
|
63
|
+
):
|
|
64
|
+
self.config_provider = config_provider
|
|
65
|
+
self.io = io
|
|
66
|
+
self.load_application = load_application
|
|
67
|
+
self.load_balancer: LoadBalancerProvider = load_balancer_p(session, io=self.io)
|
|
68
|
+
|
|
69
|
+
def update_alb_rules(
|
|
70
|
+
self,
|
|
71
|
+
environment: str,
|
|
72
|
+
):
|
|
73
|
+
"""
|
|
74
|
+
Change ALB rules for a given environment.
|
|
75
|
+
|
|
76
|
+
Attempt to rollback the rules created/deleted if a failure occurs.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
operation_state = OperationState()
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
self._execute_rule_updates(environment, operation_state)
|
|
83
|
+
if operation_state.created_rules:
|
|
84
|
+
self.io.info(f"Created rules: {len(operation_state.created_rules)}")
|
|
85
|
+
self._output_rule_changes(operation_state.created_rules)
|
|
86
|
+
if operation_state.deleted_rules:
|
|
87
|
+
self.io.info(f"Deleted rules: {len(operation_state.deleted_rules)}")
|
|
88
|
+
self._output_rule_changes(operation_state.deleted_rules)
|
|
89
|
+
if not operation_state.created_rules and not operation_state.deleted_rules:
|
|
90
|
+
self.io.info("No rule updates required")
|
|
91
|
+
except Exception as e:
|
|
92
|
+
if operation_state.created_rules or operation_state.deleted_rules:
|
|
93
|
+
self.io.error(f"Error during rule update: {str(e)}")
|
|
94
|
+
self.io.info("Attempting to rollback changes ...")
|
|
95
|
+
try:
|
|
96
|
+
self._rollback_changes(operation_state)
|
|
97
|
+
raise PlatformException(f"Rule update failed and rolled back")
|
|
98
|
+
except RollbackException as rollback_error:
|
|
99
|
+
raise PlatformException(f"Rollback failed: \n{str(rollback_error)}")
|
|
100
|
+
else:
|
|
101
|
+
raise
|
|
102
|
+
|
|
103
|
+
def _execute_rule_updates(self, environment: str, operation_state: OperationState):
|
|
104
|
+
platform_config = self.config_provider.get_enriched_config()
|
|
105
|
+
|
|
106
|
+
application_name = platform_config.get("application", "")
|
|
107
|
+
application = self.load_application(app=application_name)
|
|
108
|
+
|
|
109
|
+
service_deployment_mode = (
|
|
110
|
+
platform_config.get("environments")
|
|
111
|
+
.get(environment, {})
|
|
112
|
+
.get("service-deployment-mode", Deployment.COPILOT.value)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
self.io.info(f"Deployment Mode: {service_deployment_mode}")
|
|
116
|
+
|
|
117
|
+
listener_arn = self.load_balancer.get_https_listener_for_application(
|
|
118
|
+
application.name, environment
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
operation_state.listener_arn = listener_arn
|
|
122
|
+
|
|
123
|
+
self.io.debug(f"Listener ARN: {listener_arn}")
|
|
124
|
+
|
|
125
|
+
rules = self.load_balancer.get_rules_with_tags_by_listener_arn(listener_arn)
|
|
126
|
+
|
|
127
|
+
mapped_rules = {
|
|
128
|
+
rule_type: [rule for rule in rules if self._filter_rule_type(rule) == rule_type]
|
|
129
|
+
for rule_type in set(self._filter_rule_type(rule) for rule in rules)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if mapped_rules.get(RuleType.MANUAL.value, []):
|
|
133
|
+
rule_arns = [rule["RuleArn"] for rule in mapped_rules[RuleType.MANUAL.value]]
|
|
134
|
+
message = f"""The following rules have been created manually please review and if required set
|
|
135
|
+
the rules priority to the copilot range after priority: {COPILOT_RULE_PRIORITY}.\n
|
|
136
|
+
Rules: {rule_arns}"""
|
|
137
|
+
raise PlatformException(message)
|
|
138
|
+
|
|
139
|
+
if (
|
|
140
|
+
service_deployment_mode == Deployment.PLATFORM.value
|
|
141
|
+
or service_deployment_mode == Deployment.DUAL_DEPLOY_PLATFORM.value
|
|
142
|
+
):
|
|
143
|
+
service_models = ServiceManager().get_service_models(application, environment)
|
|
144
|
+
grouped = defaultdict(list)
|
|
145
|
+
|
|
146
|
+
service_mapped_tgs = self._get_tg_arns_for_platform_services(
|
|
147
|
+
application_name, environment
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
for service in service_models:
|
|
151
|
+
if service.type not in HTTP_SERVICE_TYPES or service.name not in service_mapped_tgs:
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
additional_rules = getattr(service.http, "additional_rules", None)
|
|
155
|
+
if additional_rules:
|
|
156
|
+
for rule in additional_rules:
|
|
157
|
+
grouped[(service.name, rule.path)].extend(rule.alias)
|
|
158
|
+
|
|
159
|
+
grouped[(service.name, service.http.path)].extend(service.http.alias)
|
|
160
|
+
|
|
161
|
+
rules = []
|
|
162
|
+
for (name, path), aliases in grouped.items():
|
|
163
|
+
path_pattern = ["/*"] if path == "/" else [path, f"{path}/*"]
|
|
164
|
+
condition_length = len(aliases) + len(path_pattern)
|
|
165
|
+
|
|
166
|
+
# AWS allows a maximum of 5 condition values per rule, this includes host and path values
|
|
167
|
+
max_conditions = 5
|
|
168
|
+
if condition_length > max_conditions:
|
|
169
|
+
i = 0
|
|
170
|
+
while i < condition_length:
|
|
171
|
+
remaining_slots = max_conditions - len(path_pattern)
|
|
172
|
+
alias_split = aliases[i : i + remaining_slots] if aliases else []
|
|
173
|
+
if not alias_split:
|
|
174
|
+
break
|
|
175
|
+
|
|
176
|
+
rules.append(
|
|
177
|
+
{
|
|
178
|
+
"service": name,
|
|
179
|
+
"path": path,
|
|
180
|
+
"path_pattern": path_pattern,
|
|
181
|
+
"aliases": sorted(set(alias_split)),
|
|
182
|
+
}
|
|
183
|
+
)
|
|
184
|
+
i += remaining_slots
|
|
185
|
+
else:
|
|
186
|
+
rules.append(
|
|
187
|
+
{
|
|
188
|
+
"service": name,
|
|
189
|
+
"path": path,
|
|
190
|
+
"path_pattern": path_pattern,
|
|
191
|
+
"aliases": sorted(set(aliases)),
|
|
192
|
+
}
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
rules.sort(
|
|
196
|
+
key=lambda r: (len([s for s in r["path"].split("/") if s]), r["aliases"]),
|
|
197
|
+
reverse=True,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
platform_rules = mapped_rules.get(RuleType.PLATFORM.value, [])
|
|
201
|
+
|
|
202
|
+
rule_priority = PLATFORM_RULE_STARTING_PRIORITY
|
|
203
|
+
for rule in rules:
|
|
204
|
+
actions = [
|
|
205
|
+
{"Type": "forward", "TargetGroupArn": service_mapped_tgs[rule["service"]]}
|
|
206
|
+
]
|
|
207
|
+
conditions = [
|
|
208
|
+
{"Field": "host-header", "HostHeaderConfig": {"Values": rule["aliases"]}},
|
|
209
|
+
{
|
|
210
|
+
"Field": "path-pattern",
|
|
211
|
+
"PathPatternConfig": {"Values": rule["path_pattern"]},
|
|
212
|
+
},
|
|
213
|
+
]
|
|
214
|
+
tags = [
|
|
215
|
+
{"Key": "application", "Value": application_name},
|
|
216
|
+
{"Key": "environment", "Value": environment},
|
|
217
|
+
{"Key": "service", "Value": rule["service"]},
|
|
218
|
+
{"Key": "reason", "Value": "service"},
|
|
219
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
220
|
+
]
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
rule = self.load_balancer.create_rule(
|
|
224
|
+
listener_arn,
|
|
225
|
+
actions,
|
|
226
|
+
conditions,
|
|
227
|
+
rule_priority,
|
|
228
|
+
tags,
|
|
229
|
+
)["Rules"][0]
|
|
230
|
+
rule_arn = rule["RuleArn"]
|
|
231
|
+
|
|
232
|
+
if rule_arn in [rule["RuleArn"] for rule in platform_rules]:
|
|
233
|
+
operation_state.updated_rules.append(rule)
|
|
234
|
+
else:
|
|
235
|
+
operation_state.created_rules.append(rule)
|
|
236
|
+
|
|
237
|
+
except ClientError as e:
|
|
238
|
+
if e.response["Error"]["Code"] == "PriorityInUse":
|
|
239
|
+
existing_rule = [
|
|
240
|
+
r for r in platform_rules if r["Priority"] == str(rule_priority)
|
|
241
|
+
]
|
|
242
|
+
if not existing_rule:
|
|
243
|
+
raise PlatformException(
|
|
244
|
+
f"Priority {rule_priority} in use but no matching platform rule found."
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
self._delete_rules(existing_rule, operation_state)
|
|
248
|
+
|
|
249
|
+
rule = self.load_balancer.create_rule(
|
|
250
|
+
listener_arn,
|
|
251
|
+
actions,
|
|
252
|
+
conditions,
|
|
253
|
+
rule_priority,
|
|
254
|
+
tags,
|
|
255
|
+
)["Rules"][0]
|
|
256
|
+
|
|
257
|
+
operation_state.created_rules.append(rule)
|
|
258
|
+
else:
|
|
259
|
+
raise
|
|
260
|
+
|
|
261
|
+
rule_priority += RULE_PRIORITY_INCREMENT
|
|
262
|
+
|
|
263
|
+
# Remove dangling rules
|
|
264
|
+
managed_rules = [
|
|
265
|
+
rule["RuleArn"]
|
|
266
|
+
for rules in [
|
|
267
|
+
operation_state.created_rules,
|
|
268
|
+
operation_state.updated_rules,
|
|
269
|
+
operation_state.deleted_rules,
|
|
270
|
+
]
|
|
271
|
+
for rule in rules
|
|
272
|
+
]
|
|
273
|
+
|
|
274
|
+
for rule in platform_rules:
|
|
275
|
+
if rule["RuleArn"] not in managed_rules:
|
|
276
|
+
self._delete_rules([rule], operation_state)
|
|
277
|
+
|
|
278
|
+
# Remove dummy rules
|
|
279
|
+
if service_deployment_mode == Deployment.PLATFORM.value:
|
|
280
|
+
self._delete_rules(mapped_rules.get(RuleType.DUMMY.value, []), operation_state)
|
|
281
|
+
|
|
282
|
+
if (
|
|
283
|
+
service_deployment_mode == Deployment.COPILOT.value
|
|
284
|
+
or service_deployment_mode == Deployment.DUAL_DEPLOY_COPILOT.value
|
|
285
|
+
):
|
|
286
|
+
rules_to_delete = mapped_rules.get(RuleType.PLATFORM.value, [])
|
|
287
|
+
if not rules_to_delete:
|
|
288
|
+
return
|
|
289
|
+
|
|
290
|
+
self.io.warn("Platform rules will be deleted")
|
|
291
|
+
self._output_rule_changes(rules_to_delete)
|
|
292
|
+
if self.io.confirm(
|
|
293
|
+
f"This command is destructive and will remove load balancer listener rules created by the platform, you may lose access to your services. Are you sure you want to continue?"
|
|
294
|
+
):
|
|
295
|
+
self._delete_rules(rules_to_delete, operation_state)
|
|
296
|
+
|
|
297
|
+
def _delete_rules(self, rules: List[dict], operation_state: OperationState):
|
|
298
|
+
for rule in rules:
|
|
299
|
+
rule_arn = rule["RuleArn"]
|
|
300
|
+
try:
|
|
301
|
+
self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
|
|
302
|
+
operation_state.deleted_rules.append(rule)
|
|
303
|
+
self.io.debug(f"Deleted existing rule: {rule_arn}")
|
|
304
|
+
except Exception as e:
|
|
305
|
+
self.io.error(f"Failed to delete existing rule {rule_arn}: {str(e)}")
|
|
306
|
+
raise
|
|
307
|
+
|
|
308
|
+
def _filter_rule_type(self, rule: dict) -> str:
|
|
309
|
+
if rule["Tags"]:
|
|
310
|
+
if rule["Tags"].get("managed-by", "") == MANAGED_BY_PLATFORM:
|
|
311
|
+
return RuleType.PLATFORM.value
|
|
312
|
+
if rule["Tags"].get("reason", None) == MAINTENANCE_PAGE_REASON:
|
|
313
|
+
return RuleType.MAINTENANCE.value
|
|
314
|
+
if rule["Tags"].get("reason", None) == DUMMY_RULE_REASON:
|
|
315
|
+
return RuleType.DUMMY.value
|
|
316
|
+
|
|
317
|
+
if rule["Priority"] == "default":
|
|
318
|
+
return RuleType.DEFAULT.value
|
|
319
|
+
if int(rule["Priority"]) >= COPILOT_RULE_PRIORITY:
|
|
320
|
+
return RuleType.COPILOT.value
|
|
321
|
+
|
|
322
|
+
return RuleType.MANUAL.value
|
|
323
|
+
|
|
324
|
+
def _get_tg_arns_for_platform_services(
|
|
325
|
+
self, application: str, environment: str, managed_by: str = MANAGED_BY_SERVICE_TERRAFORM
|
|
326
|
+
) -> dict:
|
|
327
|
+
tgs = self.load_balancer.get_target_groups_with_tags([])
|
|
328
|
+
service_mapped_tgs = {}
|
|
329
|
+
for tg in tgs:
|
|
330
|
+
if (
|
|
331
|
+
tg["Tags"].get("application", "") == application
|
|
332
|
+
and tg["Tags"].get("environment", "") == environment
|
|
333
|
+
and tg["Tags"].get("managed-by", "") == managed_by
|
|
334
|
+
):
|
|
335
|
+
if tg["Tags"].get("service", ""):
|
|
336
|
+
service_mapped_tgs[tg["Tags"].get("service")] = tg["TargetGroupArn"]
|
|
337
|
+
else:
|
|
338
|
+
tg_name = tg["TargetGroupName"]
|
|
339
|
+
self.io.warn(f"Target group {tg_name} has no 'service' tag")
|
|
340
|
+
return service_mapped_tgs
|
|
341
|
+
|
|
342
|
+
def _rollback_changes(self, operation_state: OperationState):
|
|
343
|
+
rollback_errors = []
|
|
344
|
+
delete_rollbacks = []
|
|
345
|
+
create_rollbacks = []
|
|
346
|
+
for rule in operation_state.created_rules:
|
|
347
|
+
rule_arn = rule["RuleArn"]
|
|
348
|
+
try:
|
|
349
|
+
self.io.debug(f"Rolling back: Deleting created rule {rule_arn}")
|
|
350
|
+
self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
|
|
351
|
+
delete_rollbacks.append(rule_arn)
|
|
352
|
+
except Exception as e:
|
|
353
|
+
error_msg = f"Failed to delete rule {rule_arn} during rollback: {str(e)}"
|
|
354
|
+
rollback_errors.append(error_msg)
|
|
355
|
+
|
|
356
|
+
for rule_snapshot in operation_state.deleted_rules:
|
|
357
|
+
rule_arn = rule_snapshot["RuleArn"]
|
|
358
|
+
try:
|
|
359
|
+
self.io.debug(f"Rolling back: Recreating deleted rule {rule_arn}")
|
|
360
|
+
create_rollbacks.append(
|
|
361
|
+
self.load_balancer.create_rule(
|
|
362
|
+
operation_state.listener_arn,
|
|
363
|
+
actions=rule_snapshot["Actions"],
|
|
364
|
+
conditions=[
|
|
365
|
+
{"Field": key, "Values": value}
|
|
366
|
+
for key, value in rule_snapshot["Conditions"].items()
|
|
367
|
+
],
|
|
368
|
+
priority=int(rule_snapshot["Priority"]),
|
|
369
|
+
tags=[
|
|
370
|
+
{"Key": key, "Value": value}
|
|
371
|
+
for key, value in rule_snapshot["Tags"].items()
|
|
372
|
+
],
|
|
373
|
+
)["Rules"][0]["RuleArn"]
|
|
374
|
+
)
|
|
375
|
+
except Exception as e:
|
|
376
|
+
error_msg = f"Failed to recreate rule {rule_arn} during rollback: {str(e)}"
|
|
377
|
+
rollback_errors.append(error_msg)
|
|
378
|
+
|
|
379
|
+
if rollback_errors:
|
|
380
|
+
self.io.warn("Some rollback operations failed. Manual intervention may be required.")
|
|
381
|
+
errors = "\n".join(rollback_errors)
|
|
382
|
+
raise RollbackException(f"Rollback partially failed: {errors}")
|
|
383
|
+
else:
|
|
384
|
+
self.io.info("Rollback completed successfully")
|
|
385
|
+
self.io.info(
|
|
386
|
+
f"Rolled back rules by creating: {create_rollbacks} \n and deleting {delete_rollbacks}"
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
def _output_rule_changes(self, rules):
|
|
390
|
+
for rule in rules:
|
|
391
|
+
hosts = []
|
|
392
|
+
paths = []
|
|
393
|
+
|
|
394
|
+
conditions = rule.get("Conditions", [])
|
|
395
|
+
if isinstance(conditions, list):
|
|
396
|
+
for condition in conditions:
|
|
397
|
+
if condition["Field"] == "host-header":
|
|
398
|
+
hosts.extend(condition["HostHeaderConfig"]["Values"])
|
|
399
|
+
elif condition["Field"] == "path-pattern":
|
|
400
|
+
paths.extend(condition["PathPatternConfig"]["Values"])
|
|
401
|
+
elif isinstance(conditions, dict):
|
|
402
|
+
hosts.extend(conditions.get("host-header", ""))
|
|
403
|
+
paths.extend(conditions.get("path-pattern", ""))
|
|
404
|
+
|
|
405
|
+
rule_arn = rule["RuleArn"]
|
|
406
|
+
priority = rule["Priority"]
|
|
407
|
+
hosts = ",".join(hosts)
|
|
408
|
+
paths = ",".join(paths)
|
|
409
|
+
self.io.info(f"ARN: {rule_arn}")
|
|
410
|
+
self.io.info(f"Priority: {priority}")
|
|
411
|
+
self.io.info(f"Hosts: {hosts}")
|
|
412
|
+
self.io.info(f"Paths: {paths}\n")
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from dbt_platform_helper.constants import CODEBASE_PIPELINE_MODULE_PATH
|
|
4
|
+
from dbt_platform_helper.constants import ENVIRONMENT_PIPELINE_MODULE_PATH
|
|
5
|
+
from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
|
|
6
|
+
from dbt_platform_helper.constants import (
|
|
7
|
+
TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
8
|
+
)
|
|
9
|
+
from dbt_platform_helper.constants import (
|
|
10
|
+
TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
11
|
+
)
|
|
12
|
+
from dbt_platform_helper.constants import (
|
|
13
|
+
TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
14
|
+
)
|
|
15
|
+
from dbt_platform_helper.entities.semantic_version import (
|
|
16
|
+
IncompatibleMajorVersionException,
|
|
17
|
+
)
|
|
18
|
+
from dbt_platform_helper.entities.semantic_version import (
|
|
19
|
+
IncompatibleMinorVersionException,
|
|
20
|
+
)
|
|
21
|
+
from dbt_platform_helper.entities.semantic_version import SemanticVersion
|
|
22
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
23
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
24
|
+
from dbt_platform_helper.providers.environment_variable import (
|
|
25
|
+
EnvironmentVariableProvider,
|
|
26
|
+
)
|
|
27
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
28
|
+
from dbt_platform_helper.providers.version import AWSCLIInstalledVersionProvider
|
|
29
|
+
from dbt_platform_helper.providers.version import CopilotInstalledVersionProvider
|
|
30
|
+
from dbt_platform_helper.providers.version import GithubLatestVersionProvider
|
|
31
|
+
from dbt_platform_helper.providers.version import InstalledVersionProvider
|
|
32
|
+
from dbt_platform_helper.providers.version import PyPiLatestVersionProvider
|
|
33
|
+
from dbt_platform_helper.providers.version import VersionProvider
|
|
34
|
+
from dbt_platform_helper.providers.version_status import VersionStatus
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def running_as_installed_package():
|
|
38
|
+
return "site-packages" in __file__
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def allow_override_of_versioning_checks_fn():
|
|
42
|
+
return not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class PlatformHelperVersionNotFoundException(PlatformException):
|
|
46
|
+
def __init__(self, message=None):
|
|
47
|
+
super().__init__(message or "Platform helper version could not be resolved.")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class PlatformHelperVersioning:
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
54
|
+
config_provider: ConfigProvider = ConfigProvider(),
|
|
55
|
+
environment_variable_provider: EnvironmentVariableProvider = EnvironmentVariableProvider(),
|
|
56
|
+
latest_version_provider: VersionProvider = PyPiLatestVersionProvider,
|
|
57
|
+
installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
|
|
58
|
+
allow_override_of_versioning_checks: bool = None,
|
|
59
|
+
platform_helper_version_override: str = None,
|
|
60
|
+
):
|
|
61
|
+
self.io = io
|
|
62
|
+
self.config_provider = config_provider
|
|
63
|
+
self.latest_version_provider = latest_version_provider
|
|
64
|
+
self.installed_version_provider = installed_version_provider
|
|
65
|
+
self.allow_override_of_versioning_checks = (
|
|
66
|
+
allow_override_of_versioning_checks
|
|
67
|
+
if allow_override_of_versioning_checks is not None
|
|
68
|
+
else allow_override_of_versioning_checks_fn()
|
|
69
|
+
)
|
|
70
|
+
self.environment_variable_provider = environment_variable_provider
|
|
71
|
+
self.platform_helper_version_override = platform_helper_version_override
|
|
72
|
+
|
|
73
|
+
def is_auto(self):
|
|
74
|
+
platform_config = self.config_provider.load_unvalidated_config_file()
|
|
75
|
+
default_version = platform_config.get("default_versions", {}).get("platform-helper")
|
|
76
|
+
return default_version == "auto"
|
|
77
|
+
|
|
78
|
+
def get_required_version(self) -> str:
|
|
79
|
+
if self.is_auto():
|
|
80
|
+
return str(self.get_version_status().latest)
|
|
81
|
+
else:
|
|
82
|
+
return self.get_default_version()
|
|
83
|
+
|
|
84
|
+
def _check_environment_is_configured_for_auto_versioning_within_a_pipeline(self):
|
|
85
|
+
platform_helper_version_is_set_in_environment = self.environment_variable_provider.get(
|
|
86
|
+
PLATFORM_HELPER_VERSION_OVERRIDE_KEY
|
|
87
|
+
) or self.environment_variable_provider.get("PLATFORM_HELPER_VERSION")
|
|
88
|
+
modules_override_is_set_in_environment = (
|
|
89
|
+
self.environment_variable_provider.get(
|
|
90
|
+
TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR
|
|
91
|
+
)
|
|
92
|
+
or self.environment_variable_provider.get(
|
|
93
|
+
TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
|
|
94
|
+
)
|
|
95
|
+
or self.environment_variable_provider.get(
|
|
96
|
+
TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
if platform_helper_version_is_set_in_environment and modules_override_is_set_in_environment:
|
|
100
|
+
return
|
|
101
|
+
else:
|
|
102
|
+
message = "You are on managed upgrades. Generate commands should only be running inside a pipeline environment."
|
|
103
|
+
if self.allow_override_of_versioning_checks:
|
|
104
|
+
self.io.warn(message)
|
|
105
|
+
self.io.info("Bypassing versioning enforcement")
|
|
106
|
+
else:
|
|
107
|
+
self.io.abort_with_error(message)
|
|
108
|
+
|
|
109
|
+
def check_platform_helper_version_mismatch(self):
|
|
110
|
+
if self.is_auto():
|
|
111
|
+
self._check_environment_is_configured_for_auto_versioning_within_a_pipeline()
|
|
112
|
+
|
|
113
|
+
version_status = self.get_version_status()
|
|
114
|
+
required_version = self.get_required_version()
|
|
115
|
+
|
|
116
|
+
if SemanticVersion.is_semantic_version(required_version):
|
|
117
|
+
required_version_semver = SemanticVersion.from_string(required_version)
|
|
118
|
+
|
|
119
|
+
if not version_status.installed == required_version_semver:
|
|
120
|
+
message = (
|
|
121
|
+
f"WARNING: You are running platform-helper v{version_status.installed} against "
|
|
122
|
+
f"v{required_version_semver} required by the project. Running anything besides the version required by the project may result in unpredictable and destructive changes."
|
|
123
|
+
)
|
|
124
|
+
self.io.warn(message)
|
|
125
|
+
|
|
126
|
+
def check_if_needs_update(self):
|
|
127
|
+
if self.allow_override_of_versioning_checks:
|
|
128
|
+
return
|
|
129
|
+
|
|
130
|
+
version_status = self.get_version_status()
|
|
131
|
+
|
|
132
|
+
message = (
|
|
133
|
+
f"You are running platform-helper v{version_status.installed}, upgrade to "
|
|
134
|
+
f"v{version_status.latest} by running run `pip install "
|
|
135
|
+
"--upgrade dbt-platform-helper`."
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
try:
|
|
139
|
+
version_status.installed.validate_compatibility_with(version_status.latest)
|
|
140
|
+
except IncompatibleMajorVersionException:
|
|
141
|
+
self.io.error(message)
|
|
142
|
+
except IncompatibleMinorVersionException:
|
|
143
|
+
self.io.warn(message)
|
|
144
|
+
|
|
145
|
+
def get_version_status(self) -> VersionStatus:
|
|
146
|
+
locally_installed_version = self.installed_version_provider.get_semantic_version(
|
|
147
|
+
"dbt-platform-helper"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
latest_release = self.latest_version_provider.get_semantic_version("dbt-platform-helper")
|
|
151
|
+
|
|
152
|
+
return VersionStatus(installed=locally_installed_version, latest=latest_release)
|
|
153
|
+
|
|
154
|
+
def get_default_version(self):
|
|
155
|
+
return (
|
|
156
|
+
self.config_provider.load_unvalidated_config_file()
|
|
157
|
+
.get("default_versions", {})
|
|
158
|
+
.get("platform-helper")
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
def get_template_version(self):
|
|
162
|
+
if self.is_auto():
|
|
163
|
+
return self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
|
|
164
|
+
if self.platform_helper_version_override:
|
|
165
|
+
return self.platform_helper_version_override
|
|
166
|
+
platform_helper_env_override = self.environment_variable_provider.get(
|
|
167
|
+
PLATFORM_HELPER_VERSION_OVERRIDE_KEY
|
|
168
|
+
)
|
|
169
|
+
if platform_helper_env_override:
|
|
170
|
+
return platform_helper_env_override
|
|
171
|
+
|
|
172
|
+
return self.get_default_version()
|
|
173
|
+
|
|
174
|
+
def get_pinned_version(self):
|
|
175
|
+
if self.is_auto():
|
|
176
|
+
return self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
|
|
177
|
+
|
|
178
|
+
return None
|
|
179
|
+
|
|
180
|
+
def _get_pipeline_modules_source(self, pipeline_module_path: str, override_env_var_key: str):
|
|
181
|
+
pipeline_module_override = self.environment_variable_provider.get(override_env_var_key)
|
|
182
|
+
|
|
183
|
+
if pipeline_module_override:
|
|
184
|
+
return pipeline_module_override
|
|
185
|
+
|
|
186
|
+
if self.platform_helper_version_override:
|
|
187
|
+
return f"{pipeline_module_path}{self.platform_helper_version_override}"
|
|
188
|
+
|
|
189
|
+
platform_helper_env_override = self.environment_variable_provider.get(
|
|
190
|
+
PLATFORM_HELPER_VERSION_OVERRIDE_KEY
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
if platform_helper_env_override:
|
|
194
|
+
return f"{pipeline_module_path}{platform_helper_env_override}"
|
|
195
|
+
|
|
196
|
+
return f"{pipeline_module_path}{self.get_required_version()}"
|
|
197
|
+
|
|
198
|
+
def get_environment_pipeline_modules_source(self):
|
|
199
|
+
return self._get_pipeline_modules_source(
|
|
200
|
+
ENVIRONMENT_PIPELINE_MODULE_PATH,
|
|
201
|
+
TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def get_codebase_pipeline_modules_source(self):
|
|
205
|
+
return self._get_pipeline_modules_source(
|
|
206
|
+
CODEBASE_PIPELINE_MODULE_PATH,
|
|
207
|
+
TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def get_extensions_module_source(self):
|
|
211
|
+
return self.environment_variable_provider.get(
|
|
212
|
+
TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class AWSVersioning:
|
|
217
|
+
def __init__(
|
|
218
|
+
self,
|
|
219
|
+
latest_version_provider: VersionProvider = None,
|
|
220
|
+
installed_version_provider: VersionProvider = None,
|
|
221
|
+
):
|
|
222
|
+
self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
|
|
223
|
+
self.installed_version_provider = (
|
|
224
|
+
installed_version_provider or AWSCLIInstalledVersionProvider
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
def get_version_status(self) -> VersionStatus:
|
|
228
|
+
return VersionStatus(
|
|
229
|
+
self.installed_version_provider.get_semantic_version(),
|
|
230
|
+
self.latest_version_provider.get_semantic_version("aws/aws-cli", True),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class CopilotVersioning:
|
|
235
|
+
def __init__(
|
|
236
|
+
self,
|
|
237
|
+
latest_version_provider: VersionProvider = None,
|
|
238
|
+
installed_version_provider: VersionProvider = None,
|
|
239
|
+
):
|
|
240
|
+
self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
|
|
241
|
+
self.installed_version_provider = (
|
|
242
|
+
installed_version_provider or CopilotInstalledVersionProvider
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
def get_version_status(self) -> VersionStatus:
|
|
246
|
+
return VersionStatus(
|
|
247
|
+
self.installed_version_provider.get_semantic_version(),
|
|
248
|
+
self.latest_version_provider.get_semantic_version("aws/copilot-cli"),
|
|
249
|
+
)
|