dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. dbt_platform_helper/COMMANDS.md +107 -27
  2. dbt_platform_helper/commands/application.py +5 -6
  3. dbt_platform_helper/commands/codebase.py +31 -10
  4. dbt_platform_helper/commands/conduit.py +3 -5
  5. dbt_platform_helper/commands/config.py +20 -311
  6. dbt_platform_helper/commands/copilot.py +18 -391
  7. dbt_platform_helper/commands/database.py +17 -9
  8. dbt_platform_helper/commands/environment.py +20 -14
  9. dbt_platform_helper/commands/generate.py +0 -3
  10. dbt_platform_helper/commands/internal.py +140 -0
  11. dbt_platform_helper/commands/notify.py +58 -78
  12. dbt_platform_helper/commands/pipeline.py +23 -19
  13. dbt_platform_helper/commands/secrets.py +39 -93
  14. dbt_platform_helper/commands/version.py +7 -12
  15. dbt_platform_helper/constants.py +52 -7
  16. dbt_platform_helper/domain/codebase.py +89 -39
  17. dbt_platform_helper/domain/conduit.py +335 -76
  18. dbt_platform_helper/domain/config.py +381 -0
  19. dbt_platform_helper/domain/copilot.py +398 -0
  20. dbt_platform_helper/domain/copilot_environment.py +8 -8
  21. dbt_platform_helper/domain/database_copy.py +2 -2
  22. dbt_platform_helper/domain/maintenance_page.py +254 -430
  23. dbt_platform_helper/domain/notify.py +64 -0
  24. dbt_platform_helper/domain/pipelines.py +43 -35
  25. dbt_platform_helper/domain/plans.py +41 -0
  26. dbt_platform_helper/domain/secrets.py +279 -0
  27. dbt_platform_helper/domain/service.py +570 -0
  28. dbt_platform_helper/domain/terraform_environment.py +14 -13
  29. dbt_platform_helper/domain/update_alb_rules.py +412 -0
  30. dbt_platform_helper/domain/versioning.py +249 -0
  31. dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
  32. dbt_platform_helper/entities/semantic_version.py +83 -0
  33. dbt_platform_helper/entities/service.py +339 -0
  34. dbt_platform_helper/platform_exception.py +4 -0
  35. dbt_platform_helper/providers/autoscaling.py +24 -0
  36. dbt_platform_helper/providers/aws/__init__.py +0 -0
  37. dbt_platform_helper/providers/aws/exceptions.py +70 -0
  38. dbt_platform_helper/providers/aws/interfaces.py +13 -0
  39. dbt_platform_helper/providers/aws/opensearch.py +23 -0
  40. dbt_platform_helper/providers/aws/redis.py +21 -0
  41. dbt_platform_helper/providers/aws/sso_auth.py +75 -0
  42. dbt_platform_helper/providers/cache.py +40 -4
  43. dbt_platform_helper/providers/cloudformation.py +1 -1
  44. dbt_platform_helper/providers/config.py +137 -19
  45. dbt_platform_helper/providers/config_validator.py +112 -51
  46. dbt_platform_helper/providers/copilot.py +24 -16
  47. dbt_platform_helper/providers/ecr.py +89 -7
  48. dbt_platform_helper/providers/ecs.py +228 -36
  49. dbt_platform_helper/providers/environment_variable.py +24 -0
  50. dbt_platform_helper/providers/files.py +1 -1
  51. dbt_platform_helper/providers/io.py +36 -4
  52. dbt_platform_helper/providers/kms.py +22 -0
  53. dbt_platform_helper/providers/load_balancers.py +402 -42
  54. dbt_platform_helper/providers/logs.py +72 -0
  55. dbt_platform_helper/providers/parameter_store.py +134 -0
  56. dbt_platform_helper/providers/s3.py +21 -0
  57. dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
  58. dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
  59. dbt_platform_helper/providers/schema_migrator.py +77 -0
  60. dbt_platform_helper/providers/secrets.py +5 -5
  61. dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
  62. dbt_platform_helper/providers/terraform_manifest.py +121 -19
  63. dbt_platform_helper/providers/version.py +106 -23
  64. dbt_platform_helper/providers/version_status.py +27 -0
  65. dbt_platform_helper/providers/vpc.py +36 -5
  66. dbt_platform_helper/providers/yaml_file.py +58 -2
  67. dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
  68. dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
  69. dbt_platform_helper/utilities/decorators.py +103 -0
  70. dbt_platform_helper/utils/application.py +119 -22
  71. dbt_platform_helper/utils/aws.py +39 -150
  72. dbt_platform_helper/utils/deep_merge.py +10 -0
  73. dbt_platform_helper/utils/git.py +1 -14
  74. dbt_platform_helper/utils/validation.py +1 -1
  75. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
  76. dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
  77. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
  78. platform_helper.py +3 -1
  79. terraform/elasticache-redis/plans.yml +85 -0
  80. terraform/opensearch/plans.yml +71 -0
  81. terraform/postgres/plans.yml +128 -0
  82. dbt_platform_helper/addon-plans.yml +0 -224
  83. dbt_platform_helper/providers/aws.py +0 -37
  84. dbt_platform_helper/providers/opensearch.py +0 -36
  85. dbt_platform_helper/providers/redis.py +0 -34
  86. dbt_platform_helper/providers/semantic_version.py +0 -126
  87. dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
  88. dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
  89. dbt_platform_helper/utils/cloudfoundry.py +0 -14
  90. dbt_platform_helper/utils/files.py +0 -53
  91. dbt_platform_helper/utils/manifests.py +0 -18
  92. dbt_platform_helper/utils/versioning.py +0 -238
  93. dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
  94. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
  95. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,412 @@
1
+ from collections import defaultdict
2
+ from dataclasses import dataclass
3
+ from dataclasses import field
4
+ from enum import Enum
5
+ from typing import List
6
+
7
+ from botocore.exceptions import ClientError
8
+
9
+ from dbt_platform_helper.constants import COPILOT_RULE_PRIORITY
10
+ from dbt_platform_helper.constants import DUMMY_RULE_REASON
11
+ from dbt_platform_helper.constants import HTTP_SERVICE_TYPES
12
+ from dbt_platform_helper.constants import MAINTENANCE_PAGE_REASON
13
+ from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
14
+ from dbt_platform_helper.constants import MANAGED_BY_SERVICE_TERRAFORM
15
+ from dbt_platform_helper.constants import PLATFORM_RULE_STARTING_PRIORITY
16
+ from dbt_platform_helper.constants import RULE_PRIORITY_INCREMENT
17
+ from dbt_platform_helper.domain.service import ServiceManager
18
+ from dbt_platform_helper.platform_exception import PlatformException
19
+ from dbt_platform_helper.providers.config import ConfigProvider
20
+ from dbt_platform_helper.providers.config_validator import ConfigValidator
21
+ from dbt_platform_helper.providers.io import ClickIOProvider
22
+ from dbt_platform_helper.providers.load_balancers import LoadBalancerProvider
23
+ from dbt_platform_helper.utils.application import load_application
24
+
25
+
26
+ class RollbackException(PlatformException):
27
+ pass
28
+
29
+
30
+ class RuleType(Enum):
31
+ PLATFORM = "platform"
32
+ MAINTENANCE = "maintenance"
33
+ DEFAULT = "default"
34
+ COPILOT = "copilot"
35
+ MANUAL = "manual"
36
+ DUMMY = "dummy"
37
+
38
+
39
+ class Deployment(Enum):
40
+ PLATFORM = "platform"
41
+ COPILOT = "copilot"
42
+ DUAL_DEPLOY_PLATFORM = "dual-deploy-platform-traffic"
43
+ DUAL_DEPLOY_COPILOT = "dual-deploy-copilot-traffic"
44
+
45
+
46
+ @dataclass
47
+ class OperationState:
48
+ created_rules: List[object] = field(default_factory=list)
49
+ deleted_rules: List[object] = field(default_factory=list)
50
+ updated_rules: List[object] = field(default_factory=list)
51
+ listener_arn: str = ""
52
+
53
+
54
+ class UpdateALBRules:
55
+
56
+ def __init__(
57
+ self,
58
+ session,
59
+ config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
60
+ io: ClickIOProvider = ClickIOProvider(),
61
+ load_application=load_application,
62
+ load_balancer_p: LoadBalancerProvider = LoadBalancerProvider,
63
+ ):
64
+ self.config_provider = config_provider
65
+ self.io = io
66
+ self.load_application = load_application
67
+ self.load_balancer: LoadBalancerProvider = load_balancer_p(session, io=self.io)
68
+
69
+ def update_alb_rules(
70
+ self,
71
+ environment: str,
72
+ ):
73
+ """
74
+ Change ALB rules for a given environment.
75
+
76
+ Attempt to rollback the rules created/deleted if a failure occurs.
77
+ """
78
+
79
+ operation_state = OperationState()
80
+
81
+ try:
82
+ self._execute_rule_updates(environment, operation_state)
83
+ if operation_state.created_rules:
84
+ self.io.info(f"Created rules: {len(operation_state.created_rules)}")
85
+ self._output_rule_changes(operation_state.created_rules)
86
+ if operation_state.deleted_rules:
87
+ self.io.info(f"Deleted rules: {len(operation_state.deleted_rules)}")
88
+ self._output_rule_changes(operation_state.deleted_rules)
89
+ if not operation_state.created_rules and not operation_state.deleted_rules:
90
+ self.io.info("No rule updates required")
91
+ except Exception as e:
92
+ if operation_state.created_rules or operation_state.deleted_rules:
93
+ self.io.error(f"Error during rule update: {str(e)}")
94
+ self.io.info("Attempting to rollback changes ...")
95
+ try:
96
+ self._rollback_changes(operation_state)
97
+ raise PlatformException(f"Rule update failed and rolled back")
98
+ except RollbackException as rollback_error:
99
+ raise PlatformException(f"Rollback failed: \n{str(rollback_error)}")
100
+ else:
101
+ raise
102
+
103
+ def _execute_rule_updates(self, environment: str, operation_state: OperationState):
104
+ platform_config = self.config_provider.get_enriched_config()
105
+
106
+ application_name = platform_config.get("application", "")
107
+ application = self.load_application(app=application_name)
108
+
109
+ service_deployment_mode = (
110
+ platform_config.get("environments")
111
+ .get(environment, {})
112
+ .get("service-deployment-mode", Deployment.COPILOT.value)
113
+ )
114
+
115
+ self.io.info(f"Deployment Mode: {service_deployment_mode}")
116
+
117
+ listener_arn = self.load_balancer.get_https_listener_for_application(
118
+ application.name, environment
119
+ )
120
+
121
+ operation_state.listener_arn = listener_arn
122
+
123
+ self.io.debug(f"Listener ARN: {listener_arn}")
124
+
125
+ rules = self.load_balancer.get_rules_with_tags_by_listener_arn(listener_arn)
126
+
127
+ mapped_rules = {
128
+ rule_type: [rule for rule in rules if self._filter_rule_type(rule) == rule_type]
129
+ for rule_type in set(self._filter_rule_type(rule) for rule in rules)
130
+ }
131
+
132
+ if mapped_rules.get(RuleType.MANUAL.value, []):
133
+ rule_arns = [rule["RuleArn"] for rule in mapped_rules[RuleType.MANUAL.value]]
134
+ message = f"""The following rules have been created manually please review and if required set
135
+ the rules priority to the copilot range after priority: {COPILOT_RULE_PRIORITY}.\n
136
+ Rules: {rule_arns}"""
137
+ raise PlatformException(message)
138
+
139
+ if (
140
+ service_deployment_mode == Deployment.PLATFORM.value
141
+ or service_deployment_mode == Deployment.DUAL_DEPLOY_PLATFORM.value
142
+ ):
143
+ service_models = ServiceManager().get_service_models(application, environment)
144
+ grouped = defaultdict(list)
145
+
146
+ service_mapped_tgs = self._get_tg_arns_for_platform_services(
147
+ application_name, environment
148
+ )
149
+
150
+ for service in service_models:
151
+ if service.type not in HTTP_SERVICE_TYPES or service.name not in service_mapped_tgs:
152
+ continue
153
+
154
+ additional_rules = getattr(service.http, "additional_rules", None)
155
+ if additional_rules:
156
+ for rule in additional_rules:
157
+ grouped[(service.name, rule.path)].extend(rule.alias)
158
+
159
+ grouped[(service.name, service.http.path)].extend(service.http.alias)
160
+
161
+ rules = []
162
+ for (name, path), aliases in grouped.items():
163
+ path_pattern = ["/*"] if path == "/" else [path, f"{path}/*"]
164
+ condition_length = len(aliases) + len(path_pattern)
165
+
166
+ # AWS allows a maximum of 5 condition values per rule, this includes host and path values
167
+ max_conditions = 5
168
+ if condition_length > max_conditions:
169
+ i = 0
170
+ while i < condition_length:
171
+ remaining_slots = max_conditions - len(path_pattern)
172
+ alias_split = aliases[i : i + remaining_slots] if aliases else []
173
+ if not alias_split:
174
+ break
175
+
176
+ rules.append(
177
+ {
178
+ "service": name,
179
+ "path": path,
180
+ "path_pattern": path_pattern,
181
+ "aliases": sorted(set(alias_split)),
182
+ }
183
+ )
184
+ i += remaining_slots
185
+ else:
186
+ rules.append(
187
+ {
188
+ "service": name,
189
+ "path": path,
190
+ "path_pattern": path_pattern,
191
+ "aliases": sorted(set(aliases)),
192
+ }
193
+ )
194
+
195
+ rules.sort(
196
+ key=lambda r: (len([s for s in r["path"].split("/") if s]), r["aliases"]),
197
+ reverse=True,
198
+ )
199
+
200
+ platform_rules = mapped_rules.get(RuleType.PLATFORM.value, [])
201
+
202
+ rule_priority = PLATFORM_RULE_STARTING_PRIORITY
203
+ for rule in rules:
204
+ actions = [
205
+ {"Type": "forward", "TargetGroupArn": service_mapped_tgs[rule["service"]]}
206
+ ]
207
+ conditions = [
208
+ {"Field": "host-header", "HostHeaderConfig": {"Values": rule["aliases"]}},
209
+ {
210
+ "Field": "path-pattern",
211
+ "PathPatternConfig": {"Values": rule["path_pattern"]},
212
+ },
213
+ ]
214
+ tags = [
215
+ {"Key": "application", "Value": application_name},
216
+ {"Key": "environment", "Value": environment},
217
+ {"Key": "service", "Value": rule["service"]},
218
+ {"Key": "reason", "Value": "service"},
219
+ {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
220
+ ]
221
+
222
+ try:
223
+ rule = self.load_balancer.create_rule(
224
+ listener_arn,
225
+ actions,
226
+ conditions,
227
+ rule_priority,
228
+ tags,
229
+ )["Rules"][0]
230
+ rule_arn = rule["RuleArn"]
231
+
232
+ if rule_arn in [rule["RuleArn"] for rule in platform_rules]:
233
+ operation_state.updated_rules.append(rule)
234
+ else:
235
+ operation_state.created_rules.append(rule)
236
+
237
+ except ClientError as e:
238
+ if e.response["Error"]["Code"] == "PriorityInUse":
239
+ existing_rule = [
240
+ r for r in platform_rules if r["Priority"] == str(rule_priority)
241
+ ]
242
+ if not existing_rule:
243
+ raise PlatformException(
244
+ f"Priority {rule_priority} in use but no matching platform rule found."
245
+ )
246
+
247
+ self._delete_rules(existing_rule, operation_state)
248
+
249
+ rule = self.load_balancer.create_rule(
250
+ listener_arn,
251
+ actions,
252
+ conditions,
253
+ rule_priority,
254
+ tags,
255
+ )["Rules"][0]
256
+
257
+ operation_state.created_rules.append(rule)
258
+ else:
259
+ raise
260
+
261
+ rule_priority += RULE_PRIORITY_INCREMENT
262
+
263
+ # Remove dangling rules
264
+ managed_rules = [
265
+ rule["RuleArn"]
266
+ for rules in [
267
+ operation_state.created_rules,
268
+ operation_state.updated_rules,
269
+ operation_state.deleted_rules,
270
+ ]
271
+ for rule in rules
272
+ ]
273
+
274
+ for rule in platform_rules:
275
+ if rule["RuleArn"] not in managed_rules:
276
+ self._delete_rules([rule], operation_state)
277
+
278
+ # Remove dummy rules
279
+ if service_deployment_mode == Deployment.PLATFORM.value:
280
+ self._delete_rules(mapped_rules.get(RuleType.DUMMY.value, []), operation_state)
281
+
282
+ if (
283
+ service_deployment_mode == Deployment.COPILOT.value
284
+ or service_deployment_mode == Deployment.DUAL_DEPLOY_COPILOT.value
285
+ ):
286
+ rules_to_delete = mapped_rules.get(RuleType.PLATFORM.value, [])
287
+ if not rules_to_delete:
288
+ return
289
+
290
+ self.io.warn("Platform rules will be deleted")
291
+ self._output_rule_changes(rules_to_delete)
292
+ if self.io.confirm(
293
+ f"This command is destructive and will remove load balancer listener rules created by the platform, you may lose access to your services. Are you sure you want to continue?"
294
+ ):
295
+ self._delete_rules(rules_to_delete, operation_state)
296
+
297
+ def _delete_rules(self, rules: List[dict], operation_state: OperationState):
298
+ for rule in rules:
299
+ rule_arn = rule["RuleArn"]
300
+ try:
301
+ self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
302
+ operation_state.deleted_rules.append(rule)
303
+ self.io.debug(f"Deleted existing rule: {rule_arn}")
304
+ except Exception as e:
305
+ self.io.error(f"Failed to delete existing rule {rule_arn}: {str(e)}")
306
+ raise
307
+
308
+ def _filter_rule_type(self, rule: dict) -> str:
309
+ if rule["Tags"]:
310
+ if rule["Tags"].get("managed-by", "") == MANAGED_BY_PLATFORM:
311
+ return RuleType.PLATFORM.value
312
+ if rule["Tags"].get("reason", None) == MAINTENANCE_PAGE_REASON:
313
+ return RuleType.MAINTENANCE.value
314
+ if rule["Tags"].get("reason", None) == DUMMY_RULE_REASON:
315
+ return RuleType.DUMMY.value
316
+
317
+ if rule["Priority"] == "default":
318
+ return RuleType.DEFAULT.value
319
+ if int(rule["Priority"]) >= COPILOT_RULE_PRIORITY:
320
+ return RuleType.COPILOT.value
321
+
322
+ return RuleType.MANUAL.value
323
+
324
+ def _get_tg_arns_for_platform_services(
325
+ self, application: str, environment: str, managed_by: str = MANAGED_BY_SERVICE_TERRAFORM
326
+ ) -> dict:
327
+ tgs = self.load_balancer.get_target_groups_with_tags([])
328
+ service_mapped_tgs = {}
329
+ for tg in tgs:
330
+ if (
331
+ tg["Tags"].get("application", "") == application
332
+ and tg["Tags"].get("environment", "") == environment
333
+ and tg["Tags"].get("managed-by", "") == managed_by
334
+ ):
335
+ if tg["Tags"].get("service", ""):
336
+ service_mapped_tgs[tg["Tags"].get("service")] = tg["TargetGroupArn"]
337
+ else:
338
+ tg_name = tg["TargetGroupName"]
339
+ self.io.warn(f"Target group {tg_name} has no 'service' tag")
340
+ return service_mapped_tgs
341
+
342
+ def _rollback_changes(self, operation_state: OperationState):
343
+ rollback_errors = []
344
+ delete_rollbacks = []
345
+ create_rollbacks = []
346
+ for rule in operation_state.created_rules:
347
+ rule_arn = rule["RuleArn"]
348
+ try:
349
+ self.io.debug(f"Rolling back: Deleting created rule {rule_arn}")
350
+ self.load_balancer.delete_listener_rule_by_resource_arn(rule_arn)
351
+ delete_rollbacks.append(rule_arn)
352
+ except Exception as e:
353
+ error_msg = f"Failed to delete rule {rule_arn} during rollback: {str(e)}"
354
+ rollback_errors.append(error_msg)
355
+
356
+ for rule_snapshot in operation_state.deleted_rules:
357
+ rule_arn = rule_snapshot["RuleArn"]
358
+ try:
359
+ self.io.debug(f"Rolling back: Recreating deleted rule {rule_arn}")
360
+ create_rollbacks.append(
361
+ self.load_balancer.create_rule(
362
+ operation_state.listener_arn,
363
+ actions=rule_snapshot["Actions"],
364
+ conditions=[
365
+ {"Field": key, "Values": value}
366
+ for key, value in rule_snapshot["Conditions"].items()
367
+ ],
368
+ priority=int(rule_snapshot["Priority"]),
369
+ tags=[
370
+ {"Key": key, "Value": value}
371
+ for key, value in rule_snapshot["Tags"].items()
372
+ ],
373
+ )["Rules"][0]["RuleArn"]
374
+ )
375
+ except Exception as e:
376
+ error_msg = f"Failed to recreate rule {rule_arn} during rollback: {str(e)}"
377
+ rollback_errors.append(error_msg)
378
+
379
+ if rollback_errors:
380
+ self.io.warn("Some rollback operations failed. Manual intervention may be required.")
381
+ errors = "\n".join(rollback_errors)
382
+ raise RollbackException(f"Rollback partially failed: {errors}")
383
+ else:
384
+ self.io.info("Rollback completed successfully")
385
+ self.io.info(
386
+ f"Rolled back rules by creating: {create_rollbacks} \n and deleting {delete_rollbacks}"
387
+ )
388
+
389
+ def _output_rule_changes(self, rules):
390
+ for rule in rules:
391
+ hosts = []
392
+ paths = []
393
+
394
+ conditions = rule.get("Conditions", [])
395
+ if isinstance(conditions, list):
396
+ for condition in conditions:
397
+ if condition["Field"] == "host-header":
398
+ hosts.extend(condition["HostHeaderConfig"]["Values"])
399
+ elif condition["Field"] == "path-pattern":
400
+ paths.extend(condition["PathPatternConfig"]["Values"])
401
+ elif isinstance(conditions, dict):
402
+ hosts.extend(conditions.get("host-header", ""))
403
+ paths.extend(conditions.get("path-pattern", ""))
404
+
405
+ rule_arn = rule["RuleArn"]
406
+ priority = rule["Priority"]
407
+ hosts = ",".join(hosts)
408
+ paths = ",".join(paths)
409
+ self.io.info(f"ARN: {rule_arn}")
410
+ self.io.info(f"Priority: {priority}")
411
+ self.io.info(f"Hosts: {hosts}")
412
+ self.io.info(f"Paths: {paths}\n")
@@ -0,0 +1,249 @@
1
+ import os
2
+
3
+ from dbt_platform_helper.constants import CODEBASE_PIPELINE_MODULE_PATH
4
+ from dbt_platform_helper.constants import ENVIRONMENT_PIPELINE_MODULE_PATH
5
+ from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
6
+ from dbt_platform_helper.constants import (
7
+ TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
8
+ )
9
+ from dbt_platform_helper.constants import (
10
+ TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
11
+ )
12
+ from dbt_platform_helper.constants import (
13
+ TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR,
14
+ )
15
+ from dbt_platform_helper.entities.semantic_version import (
16
+ IncompatibleMajorVersionException,
17
+ )
18
+ from dbt_platform_helper.entities.semantic_version import (
19
+ IncompatibleMinorVersionException,
20
+ )
21
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
22
+ from dbt_platform_helper.platform_exception import PlatformException
23
+ from dbt_platform_helper.providers.config import ConfigProvider
24
+ from dbt_platform_helper.providers.environment_variable import (
25
+ EnvironmentVariableProvider,
26
+ )
27
+ from dbt_platform_helper.providers.io import ClickIOProvider
28
+ from dbt_platform_helper.providers.version import AWSCLIInstalledVersionProvider
29
+ from dbt_platform_helper.providers.version import CopilotInstalledVersionProvider
30
+ from dbt_platform_helper.providers.version import GithubLatestVersionProvider
31
+ from dbt_platform_helper.providers.version import InstalledVersionProvider
32
+ from dbt_platform_helper.providers.version import PyPiLatestVersionProvider
33
+ from dbt_platform_helper.providers.version import VersionProvider
34
+ from dbt_platform_helper.providers.version_status import VersionStatus
35
+
36
+
37
+ def running_as_installed_package():
38
+ return "site-packages" in __file__
39
+
40
+
41
+ def allow_override_of_versioning_checks_fn():
42
+ return not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ
43
+
44
+
45
+ class PlatformHelperVersionNotFoundException(PlatformException):
46
+ def __init__(self, message=None):
47
+ super().__init__(message or "Platform helper version could not be resolved.")
48
+
49
+
50
+ class PlatformHelperVersioning:
51
+ def __init__(
52
+ self,
53
+ io: ClickIOProvider = ClickIOProvider(),
54
+ config_provider: ConfigProvider = ConfigProvider(),
55
+ environment_variable_provider: EnvironmentVariableProvider = EnvironmentVariableProvider(),
56
+ latest_version_provider: VersionProvider = PyPiLatestVersionProvider,
57
+ installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
58
+ allow_override_of_versioning_checks: bool = None,
59
+ platform_helper_version_override: str = None,
60
+ ):
61
+ self.io = io
62
+ self.config_provider = config_provider
63
+ self.latest_version_provider = latest_version_provider
64
+ self.installed_version_provider = installed_version_provider
65
+ self.allow_override_of_versioning_checks = (
66
+ allow_override_of_versioning_checks
67
+ if allow_override_of_versioning_checks is not None
68
+ else allow_override_of_versioning_checks_fn()
69
+ )
70
+ self.environment_variable_provider = environment_variable_provider
71
+ self.platform_helper_version_override = platform_helper_version_override
72
+
73
+ def is_auto(self):
74
+ platform_config = self.config_provider.load_unvalidated_config_file()
75
+ default_version = platform_config.get("default_versions", {}).get("platform-helper")
76
+ return default_version == "auto"
77
+
78
+ def get_required_version(self) -> str:
79
+ if self.is_auto():
80
+ return str(self.get_version_status().latest)
81
+ else:
82
+ return self.get_default_version()
83
+
84
+ def _check_environment_is_configured_for_auto_versioning_within_a_pipeline(self):
85
+ platform_helper_version_is_set_in_environment = self.environment_variable_provider.get(
86
+ PLATFORM_HELPER_VERSION_OVERRIDE_KEY
87
+ ) or self.environment_variable_provider.get("PLATFORM_HELPER_VERSION")
88
+ modules_override_is_set_in_environment = (
89
+ self.environment_variable_provider.get(
90
+ TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR
91
+ )
92
+ or self.environment_variable_provider.get(
93
+ TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
94
+ )
95
+ or self.environment_variable_provider.get(
96
+ TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
97
+ )
98
+ )
99
+ if platform_helper_version_is_set_in_environment and modules_override_is_set_in_environment:
100
+ return
101
+ else:
102
+ message = "You are on managed upgrades. Generate commands should only be running inside a pipeline environment."
103
+ if self.allow_override_of_versioning_checks:
104
+ self.io.warn(message)
105
+ self.io.info("Bypassing versioning enforcement")
106
+ else:
107
+ self.io.abort_with_error(message)
108
+
109
+ def check_platform_helper_version_mismatch(self):
110
+ if self.is_auto():
111
+ self._check_environment_is_configured_for_auto_versioning_within_a_pipeline()
112
+
113
+ version_status = self.get_version_status()
114
+ required_version = self.get_required_version()
115
+
116
+ if SemanticVersion.is_semantic_version(required_version):
117
+ required_version_semver = SemanticVersion.from_string(required_version)
118
+
119
+ if not version_status.installed == required_version_semver:
120
+ message = (
121
+ f"WARNING: You are running platform-helper v{version_status.installed} against "
122
+ f"v{required_version_semver} required by the project. Running anything besides the version required by the project may result in unpredictable and destructive changes."
123
+ )
124
+ self.io.warn(message)
125
+
126
+ def check_if_needs_update(self):
127
+ if self.allow_override_of_versioning_checks:
128
+ return
129
+
130
+ version_status = self.get_version_status()
131
+
132
+ message = (
133
+ f"You are running platform-helper v{version_status.installed}, upgrade to "
134
+ f"v{version_status.latest} by running run `pip install "
135
+ "--upgrade dbt-platform-helper`."
136
+ )
137
+
138
+ try:
139
+ version_status.installed.validate_compatibility_with(version_status.latest)
140
+ except IncompatibleMajorVersionException:
141
+ self.io.error(message)
142
+ except IncompatibleMinorVersionException:
143
+ self.io.warn(message)
144
+
145
+ def get_version_status(self) -> VersionStatus:
146
+ locally_installed_version = self.installed_version_provider.get_semantic_version(
147
+ "dbt-platform-helper"
148
+ )
149
+
150
+ latest_release = self.latest_version_provider.get_semantic_version("dbt-platform-helper")
151
+
152
+ return VersionStatus(installed=locally_installed_version, latest=latest_release)
153
+
154
+ def get_default_version(self):
155
+ return (
156
+ self.config_provider.load_unvalidated_config_file()
157
+ .get("default_versions", {})
158
+ .get("platform-helper")
159
+ )
160
+
161
+ def get_template_version(self):
162
+ if self.is_auto():
163
+ return self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
164
+ if self.platform_helper_version_override:
165
+ return self.platform_helper_version_override
166
+ platform_helper_env_override = self.environment_variable_provider.get(
167
+ PLATFORM_HELPER_VERSION_OVERRIDE_KEY
168
+ )
169
+ if platform_helper_env_override:
170
+ return platform_helper_env_override
171
+
172
+ return self.get_default_version()
173
+
174
+ def get_pinned_version(self):
175
+ if self.is_auto():
176
+ return self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
177
+
178
+ return None
179
+
180
+ def _get_pipeline_modules_source(self, pipeline_module_path: str, override_env_var_key: str):
181
+ pipeline_module_override = self.environment_variable_provider.get(override_env_var_key)
182
+
183
+ if pipeline_module_override:
184
+ return pipeline_module_override
185
+
186
+ if self.platform_helper_version_override:
187
+ return f"{pipeline_module_path}{self.platform_helper_version_override}"
188
+
189
+ platform_helper_env_override = self.environment_variable_provider.get(
190
+ PLATFORM_HELPER_VERSION_OVERRIDE_KEY
191
+ )
192
+
193
+ if platform_helper_env_override:
194
+ return f"{pipeline_module_path}{platform_helper_env_override}"
195
+
196
+ return f"{pipeline_module_path}{self.get_required_version()}"
197
+
198
+ def get_environment_pipeline_modules_source(self):
199
+ return self._get_pipeline_modules_source(
200
+ ENVIRONMENT_PIPELINE_MODULE_PATH,
201
+ TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
202
+ )
203
+
204
+ def get_codebase_pipeline_modules_source(self):
205
+ return self._get_pipeline_modules_source(
206
+ CODEBASE_PIPELINE_MODULE_PATH,
207
+ TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
208
+ )
209
+
210
+ def get_extensions_module_source(self):
211
+ return self.environment_variable_provider.get(
212
+ TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR
213
+ )
214
+
215
+
216
+ class AWSVersioning:
217
+ def __init__(
218
+ self,
219
+ latest_version_provider: VersionProvider = None,
220
+ installed_version_provider: VersionProvider = None,
221
+ ):
222
+ self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
223
+ self.installed_version_provider = (
224
+ installed_version_provider or AWSCLIInstalledVersionProvider
225
+ )
226
+
227
+ def get_version_status(self) -> VersionStatus:
228
+ return VersionStatus(
229
+ self.installed_version_provider.get_semantic_version(),
230
+ self.latest_version_provider.get_semantic_version("aws/aws-cli", True),
231
+ )
232
+
233
+
234
+ class CopilotVersioning:
235
+ def __init__(
236
+ self,
237
+ latest_version_provider: VersionProvider = None,
238
+ installed_version_provider: VersionProvider = None,
239
+ ):
240
+ self.latest_version_provider = latest_version_provider or GithubLatestVersionProvider
241
+ self.installed_version_provider = (
242
+ installed_version_provider or CopilotInstalledVersionProvider
243
+ )
244
+
245
+ def get_version_status(self) -> VersionStatus:
246
+ return VersionStatus(
247
+ self.installed_version_provider.get_semantic_version(),
248
+ self.latest_version_provider.get_semantic_version("aws/copilot-cli"),
249
+ )