dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. dbt_platform_helper/COMMANDS.md +107 -27
  2. dbt_platform_helper/commands/application.py +5 -6
  3. dbt_platform_helper/commands/codebase.py +31 -10
  4. dbt_platform_helper/commands/conduit.py +3 -5
  5. dbt_platform_helper/commands/config.py +20 -311
  6. dbt_platform_helper/commands/copilot.py +18 -391
  7. dbt_platform_helper/commands/database.py +17 -9
  8. dbt_platform_helper/commands/environment.py +20 -14
  9. dbt_platform_helper/commands/generate.py +0 -3
  10. dbt_platform_helper/commands/internal.py +140 -0
  11. dbt_platform_helper/commands/notify.py +58 -78
  12. dbt_platform_helper/commands/pipeline.py +23 -19
  13. dbt_platform_helper/commands/secrets.py +39 -93
  14. dbt_platform_helper/commands/version.py +7 -12
  15. dbt_platform_helper/constants.py +52 -7
  16. dbt_platform_helper/domain/codebase.py +89 -39
  17. dbt_platform_helper/domain/conduit.py +335 -76
  18. dbt_platform_helper/domain/config.py +381 -0
  19. dbt_platform_helper/domain/copilot.py +398 -0
  20. dbt_platform_helper/domain/copilot_environment.py +8 -8
  21. dbt_platform_helper/domain/database_copy.py +2 -2
  22. dbt_platform_helper/domain/maintenance_page.py +254 -430
  23. dbt_platform_helper/domain/notify.py +64 -0
  24. dbt_platform_helper/domain/pipelines.py +43 -35
  25. dbt_platform_helper/domain/plans.py +41 -0
  26. dbt_platform_helper/domain/secrets.py +279 -0
  27. dbt_platform_helper/domain/service.py +570 -0
  28. dbt_platform_helper/domain/terraform_environment.py +14 -13
  29. dbt_platform_helper/domain/update_alb_rules.py +412 -0
  30. dbt_platform_helper/domain/versioning.py +249 -0
  31. dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
  32. dbt_platform_helper/entities/semantic_version.py +83 -0
  33. dbt_platform_helper/entities/service.py +339 -0
  34. dbt_platform_helper/platform_exception.py +4 -0
  35. dbt_platform_helper/providers/autoscaling.py +24 -0
  36. dbt_platform_helper/providers/aws/__init__.py +0 -0
  37. dbt_platform_helper/providers/aws/exceptions.py +70 -0
  38. dbt_platform_helper/providers/aws/interfaces.py +13 -0
  39. dbt_platform_helper/providers/aws/opensearch.py +23 -0
  40. dbt_platform_helper/providers/aws/redis.py +21 -0
  41. dbt_platform_helper/providers/aws/sso_auth.py +75 -0
  42. dbt_platform_helper/providers/cache.py +40 -4
  43. dbt_platform_helper/providers/cloudformation.py +1 -1
  44. dbt_platform_helper/providers/config.py +137 -19
  45. dbt_platform_helper/providers/config_validator.py +112 -51
  46. dbt_platform_helper/providers/copilot.py +24 -16
  47. dbt_platform_helper/providers/ecr.py +89 -7
  48. dbt_platform_helper/providers/ecs.py +228 -36
  49. dbt_platform_helper/providers/environment_variable.py +24 -0
  50. dbt_platform_helper/providers/files.py +1 -1
  51. dbt_platform_helper/providers/io.py +36 -4
  52. dbt_platform_helper/providers/kms.py +22 -0
  53. dbt_platform_helper/providers/load_balancers.py +402 -42
  54. dbt_platform_helper/providers/logs.py +72 -0
  55. dbt_platform_helper/providers/parameter_store.py +134 -0
  56. dbt_platform_helper/providers/s3.py +21 -0
  57. dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
  58. dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
  59. dbt_platform_helper/providers/schema_migrator.py +77 -0
  60. dbt_platform_helper/providers/secrets.py +5 -5
  61. dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
  62. dbt_platform_helper/providers/terraform_manifest.py +121 -19
  63. dbt_platform_helper/providers/version.py +106 -23
  64. dbt_platform_helper/providers/version_status.py +27 -0
  65. dbt_platform_helper/providers/vpc.py +36 -5
  66. dbt_platform_helper/providers/yaml_file.py +58 -2
  67. dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
  68. dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
  69. dbt_platform_helper/utilities/decorators.py +103 -0
  70. dbt_platform_helper/utils/application.py +119 -22
  71. dbt_platform_helper/utils/aws.py +39 -150
  72. dbt_platform_helper/utils/deep_merge.py +10 -0
  73. dbt_platform_helper/utils/git.py +1 -14
  74. dbt_platform_helper/utils/validation.py +1 -1
  75. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
  76. dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
  77. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
  78. platform_helper.py +3 -1
  79. terraform/elasticache-redis/plans.yml +85 -0
  80. terraform/opensearch/plans.yml +71 -0
  81. terraform/postgres/plans.yml +128 -0
  82. dbt_platform_helper/addon-plans.yml +0 -224
  83. dbt_platform_helper/providers/aws.py +0 -37
  84. dbt_platform_helper/providers/opensearch.py +0 -36
  85. dbt_platform_helper/providers/redis.py +0 -34
  86. dbt_platform_helper/providers/semantic_version.py +0 -126
  87. dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
  88. dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
  89. dbt_platform_helper/utils/cloudfoundry.py +0 -14
  90. dbt_platform_helper/utils/files.py +0 -53
  91. dbt_platform_helper/utils/manifests.py +0 -18
  92. dbt_platform_helper/utils/versioning.py +0 -238
  93. dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
  94. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
  95. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,570 @@
1
+ import json
2
+ import os
3
+ import time
4
+ import urllib.parse
5
+ from collections import OrderedDict
6
+ from copy import deepcopy
7
+ from datetime import datetime
8
+ from datetime import timezone
9
+ from importlib.metadata import version
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from dbt_platform_helper.constants import PLATFORM_HELPER_PACKAGE_NAME
14
+ from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
15
+ from dbt_platform_helper.constants import SERVICE_CONFIG_FILE
16
+ from dbt_platform_helper.constants import SERVICE_DIRECTORY
17
+ from dbt_platform_helper.constants import (
18
+ TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR,
19
+ )
20
+ from dbt_platform_helper.constants import TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR
21
+ from dbt_platform_helper.domain.terraform_environment import (
22
+ EnvironmentNotFoundException,
23
+ )
24
+ from dbt_platform_helper.entities.service import ServiceConfig
25
+ from dbt_platform_helper.platform_exception import PlatformException
26
+ from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
27
+ from dbt_platform_helper.providers.config import ConfigProvider
28
+ from dbt_platform_helper.providers.config_validator import ConfigValidator
29
+ from dbt_platform_helper.providers.ecs import ECS
30
+ from dbt_platform_helper.providers.environment_variable import (
31
+ EnvironmentVariableProvider,
32
+ )
33
+ from dbt_platform_helper.providers.files import FileProvider
34
+ from dbt_platform_helper.providers.io import ClickIOProvider
35
+ from dbt_platform_helper.providers.logs import LogsProvider
36
+ from dbt_platform_helper.providers.s3 import S3Provider
37
+ from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
38
+ from dbt_platform_helper.providers.version import InstalledVersionProvider
39
+ from dbt_platform_helper.providers.yaml_file import YamlFileProvider
40
+ from dbt_platform_helper.utils.application import load_application
41
+ from dbt_platform_helper.utils.deep_merge import deep_merge
42
+
43
+ SERVICE_TYPES = ["Load Balanced Web Service", "Backend Service"]
44
+ DEPLOYMENT_TIMEOUT_SECONDS = 1200
45
+ POLL_INTERVAL_SECONDS = 5
46
+
47
+ # TODO add schema version to service config
48
+
49
+
50
+ class ServiceManager:
51
+ def __init__(
52
+ self,
53
+ config_provider=ConfigProvider(ConfigValidator()),
54
+ io: ClickIOProvider = ClickIOProvider(),
55
+ file_provider=YamlFileProvider,
56
+ manifest_provider: TerraformManifestProvider = None,
57
+ platform_helper_version_override: str = None,
58
+ load_application=load_application,
59
+ installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
60
+ ecs_provider: ECS = None,
61
+ s3_provider: S3Provider = None,
62
+ logs_provider: LogsProvider = None,
63
+ autoscaling_provider: AutoscalingProvider = None,
64
+ ):
65
+
66
+ self.file_provider = file_provider
67
+ self.config_provider = config_provider
68
+ self.io = io
69
+ self.manifest_provider = manifest_provider or TerraformManifestProvider()
70
+ self.platform_helper_version_override = (
71
+ platform_helper_version_override
72
+ or EnvironmentVariableProvider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
73
+ )
74
+ self.load_application = load_application
75
+ self.installed_version_provider = installed_version_provider
76
+ self.ecs_provider = ecs_provider
77
+ self.s3_provider = s3_provider
78
+ self.logs_provider = logs_provider
79
+ self.autoscaling_provider = autoscaling_provider
80
+
81
+ def generate(self, environment: str, services: list[str]):
82
+
83
+ config = self.config_provider.get_enriched_config()
84
+ application_name = config.get("application", "")
85
+ application = self.load_application(app=application_name)
86
+
87
+ if environment not in application.environments:
88
+ raise EnvironmentNotFoundException(
89
+ f"Cannot generate Terraform for environment '{environment}'. It does not exist in your configuration."
90
+ )
91
+
92
+ platform_helper_version_for_template: str = (
93
+ self.platform_helper_version_override
94
+ or config.get("default_versions", {}).get("platform-helper")
95
+ )
96
+
97
+ source_type = EnvironmentVariableProvider.get(TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR)
98
+
99
+ if source_type == "LOCAL":
100
+ module_source_override = ServiceConfig.local_terraform_source
101
+ elif source_type == "OVERRIDE":
102
+ module_source_override = EnvironmentVariableProvider.get(
103
+ TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR
104
+ )
105
+ else:
106
+ module_source_override = None
107
+
108
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
109
+
110
+ service_models = self.get_service_models(application, environment, services)
111
+
112
+ for service in service_models:
113
+
114
+ model_dump = service.model_dump(
115
+ exclude_none=True,
116
+ by_alias=True,
117
+ mode="json",
118
+ ) # Use by_alias=True so that the Cooldown field 'in_' is written as 'in' in the output
119
+
120
+ output_path = Path(
121
+ f"terraform/{SERVICE_DIRECTORY}/{environment}/{service.name}/{SERVICE_CONFIG_FILE}"
122
+ )
123
+ output_path.parent.mkdir(parents=True, exist_ok=True)
124
+
125
+ self.file_provider.write(
126
+ str(output_path),
127
+ model_dump,
128
+ f"# WARNING: This is an autogenerated file, not for manual editing.\n# Generated by platform-helper {version('dbt-platform-helper')} / {timestamp}.\n",
129
+ )
130
+
131
+ self.manifest_provider.generate_service_config(
132
+ service,
133
+ environment,
134
+ platform_helper_version_for_template,
135
+ config,
136
+ module_source_override,
137
+ )
138
+
139
+ def get_service_models(self, application, environment, services=None) -> list[ServiceConfig]:
140
+ if not services:
141
+ services = []
142
+ try:
143
+ for dir in Path("services").iterdir():
144
+ if dir.is_dir():
145
+ config_path = dir / SERVICE_CONFIG_FILE
146
+ if config_path.exists():
147
+ services.append(dir.name)
148
+ else:
149
+ self.io.warn(
150
+ f"Failed loading service name from {dir.name}.\n"
151
+ "Please ensure that your '/services' directory follows the correct structure (i.e. /services/<service_name>/service-config.yml) and the 'service-config.yml' contents are correct."
152
+ )
153
+ except Exception as e:
154
+ self.io.abort_with_error(f"Failed extracting services with exception, {e}")
155
+
156
+ service_models = []
157
+ for service in services:
158
+ file_content = self.file_provider.load(
159
+ f"{SERVICE_DIRECTORY}/{service}/{SERVICE_CONFIG_FILE}"
160
+ )
161
+
162
+ file_content = self.file_provider.find_and_replace(
163
+ config=file_content,
164
+ strings=[
165
+ "${PLATFORM_APPLICATION_NAME}",
166
+ "${PLATFORM_ENVIRONMENT_NAME}",
167
+ ],
168
+ replacements=[application.name, environment],
169
+ )
170
+
171
+ env_overrides = file_content.get("environments", {}).get(environment)
172
+ if env_overrides:
173
+ merged_config = deep_merge(file_content, env_overrides)
174
+ else:
175
+ merged_config = file_content
176
+ merged_config.pop("environments", None)
177
+
178
+ service_model = ServiceConfig(**merged_config)
179
+ service_models.append(service_model)
180
+
181
+ return service_models
182
+
183
+ def migrate_copilot_manifests(self) -> None:
184
+ service_directory = Path("services/")
185
+ service_directory.mkdir(parents=True, exist_ok=True)
186
+
187
+ for dirname, _, filenames in os.walk("copilot"):
188
+ if "manifest.yml" in filenames and "environments" not in dirname:
189
+ copilot_manifest = self.file_provider.load(f"{dirname}/manifest.yml")
190
+ service_manifest = OrderedDict(deepcopy(copilot_manifest))
191
+
192
+ if service_manifest["type"] not in SERVICE_TYPES:
193
+ continue
194
+
195
+ if "environments" in service_manifest:
196
+ for env in service_manifest["environments"]:
197
+ env_config = service_manifest["environments"][env]
198
+ if "http" in env_config:
199
+ if "alb" in env_config["http"]:
200
+ del env_config["http"]["alb"]
201
+ if isinstance(env_config["http"].get("alias", []), str):
202
+ env_config["http"]["alias"] = [env_config["http"]["alias"]]
203
+ if "healthcheck" in env_config["http"]:
204
+ if "interval" in env_config["http"]["healthcheck"]:
205
+ interval = env_config["http"]["healthcheck"]["interval"]
206
+ env_config["http"]["healthcheck"]["interval"] = int(
207
+ interval.rstrip("s")
208
+ )
209
+ if "timeout" in env_config["http"]["healthcheck"]:
210
+ timeout = env_config["http"]["healthcheck"]["timeout"]
211
+ env_config["http"]["healthcheck"]["timeout"] = int(
212
+ timeout.rstrip("s")
213
+ )
214
+ if "grace_period" in env_config["http"]["healthcheck"]:
215
+ grace_period = env_config["http"]["healthcheck"]["grace_period"]
216
+ env_config["http"]["healthcheck"]["grace_period"] = int(
217
+ grace_period.rstrip("s")
218
+ )
219
+ if "image" in env_config:
220
+ image_without_tag = env_config["image"]["location"].rsplit(":", 1)[0]
221
+ env_config["image"]["location"] = image_without_tag
222
+ if "healthcheck" in env_config["image"]:
223
+ if "interval" in env_config["image"]["healthcheck"]:
224
+ interval = env_config["image"]["healthcheck"]["interval"]
225
+ env_config["image"]["healthcheck"]["interval"] = int(
226
+ interval.rstrip("s")
227
+ )
228
+ if "timeout" in env_config["image"]["healthcheck"]:
229
+ timeout = env_config["image"]["healthcheck"]["timeout"]
230
+ env_config["image"]["healthcheck"]["timeout"] = int(
231
+ timeout.rstrip("s")
232
+ )
233
+ if "start_period" in env_config["image"]["healthcheck"]:
234
+ start_period = env_config["image"]["healthcheck"][
235
+ "start_period"
236
+ ]
237
+ env_config["image"]["healthcheck"]["start_period"] = int(
238
+ start_period.rstrip("s")
239
+ )
240
+ if "count" in env_config:
241
+ if not isinstance(env_config.get("count"), int):
242
+ if "cooldown" in env_config["count"]:
243
+ if "in" in env_config["count"]["cooldown"]:
244
+ scaling_in = env_config["count"]["cooldown"]["in"]
245
+ env_config["count"]["cooldown"]["in"] = int(
246
+ scaling_in.rstrip("s")
247
+ )
248
+ if "out" in env_config["count"]["cooldown"]:
249
+ scaling_out = env_config["count"]["cooldown"]["out"]
250
+ env_config["count"]["cooldown"]["out"] = int(
251
+ scaling_out.rstrip("s")
252
+ )
253
+ if "network" in env_config:
254
+ del env_config["network"]
255
+ if "observability" in env_config:
256
+ del env_config["observability"]
257
+
258
+ if "healthcheck" in service_manifest.get("http", {}):
259
+ if "interval" in service_manifest["http"]["healthcheck"]:
260
+ interval = service_manifest["http"]["healthcheck"]["interval"]
261
+ service_manifest["http"]["healthcheck"]["interval"] = int(
262
+ interval.rstrip("s")
263
+ )
264
+ if "timeout" in service_manifest["http"]["healthcheck"]:
265
+ timeout = service_manifest["http"]["healthcheck"]["timeout"]
266
+ service_manifest["http"]["healthcheck"]["timeout"] = int(
267
+ timeout.rstrip("s")
268
+ )
269
+ if "grace_period" in service_manifest["http"]["healthcheck"]:
270
+ grace_period = service_manifest["http"]["healthcheck"]["grace_period"]
271
+ service_manifest["http"]["healthcheck"]["grace_period"] = int(
272
+ grace_period.rstrip("s")
273
+ )
274
+
275
+ if "image" in service_manifest:
276
+ if "location" in service_manifest["image"]:
277
+ image_without_tag = service_manifest["image"]["location"].rsplit(":", 1)[0]
278
+ service_manifest["image"]["location"] = image_without_tag
279
+ if "healthcheck" in service_manifest["image"]:
280
+ if "interval" in service_manifest["image"]["healthcheck"]:
281
+ interval = service_manifest["image"]["healthcheck"]["interval"]
282
+ service_manifest["image"]["healthcheck"]["interval"] = int(
283
+ interval.rstrip("s")
284
+ )
285
+ if "timeout" in service_manifest["image"]["healthcheck"]:
286
+ timeout = service_manifest["image"]["healthcheck"]["timeout"]
287
+ service_manifest["image"]["healthcheck"]["timeout"] = int(
288
+ timeout.rstrip("s")
289
+ )
290
+ if "start_period" in service_manifest["image"]["healthcheck"]:
291
+ start_period = service_manifest["image"]["healthcheck"]["start_period"]
292
+ service_manifest["image"]["healthcheck"]["start_period"] = int(
293
+ start_period.rstrip("s")
294
+ )
295
+
296
+ if "count" in service_manifest:
297
+ if not isinstance(service_manifest.get("count"), int):
298
+ if "cooldown" in service_manifest.get("count"):
299
+ if "in" in service_manifest["count"]["cooldown"]:
300
+ scaling_in = service_manifest["count"]["cooldown"]["in"]
301
+ service_manifest["count"]["cooldown"]["in"] = int(
302
+ scaling_in.rstrip("s")
303
+ )
304
+ if "out" in service_manifest["count"]["cooldown"]:
305
+ scaling_out = service_manifest["count"]["cooldown"]["out"]
306
+ service_manifest["count"]["cooldown"]["out"] = int(
307
+ scaling_out.rstrip("s")
308
+ )
309
+
310
+ if "network" in service_manifest:
311
+ del service_manifest["network"]
312
+ if "observability" in service_manifest:
313
+ del service_manifest["observability"]
314
+
315
+ if "entrypoint" in service_manifest:
316
+ if isinstance(service_manifest["entrypoint"], str):
317
+ service_manifest["entrypoint"] = [service_manifest["entrypoint"]]
318
+
319
+ if "alias" in service_manifest.get("http", {}):
320
+ if isinstance(service_manifest["http"]["alias"], str):
321
+ service_manifest["http"]["alias"] = [service_manifest["http"]["alias"]]
322
+
323
+ service_manifest = self.file_provider.find_and_replace(
324
+ config=service_manifest,
325
+ strings=["${COPILOT_APPLICATION_NAME}", "${COPILOT_ENVIRONMENT_NAME}"],
326
+ replacements=["${PLATFORM_APPLICATION_NAME}", "${PLATFORM_ENVIRONMENT_NAME}"],
327
+ )
328
+
329
+ service_manifest = self.file_provider.remove_empty_keys(service_manifest)
330
+
331
+ if "sidecars" in service_manifest:
332
+ new_sidecars = {}
333
+ writable_directories = []
334
+
335
+ for sidecar_name, sidecar in service_manifest["sidecars"].items():
336
+ if "chown" not in sidecar.get("command", "") and "chmod" not in sidecar.get(
337
+ "command", ""
338
+ ):
339
+ new_sidecars[sidecar_name] = sidecar
340
+ if "chown" in sidecar.get("command", "") and "mount_points" in sidecar:
341
+ for mountpoint in sidecar["mount_points"]:
342
+ writable_directories.append(mountpoint["path"])
343
+
344
+ service_manifest["sidecars"] = new_sidecars
345
+ if "storage" in service_manifest:
346
+ service_manifest["storage"]["writable_directories"] = writable_directories
347
+
348
+ service_path = service_directory / service_manifest["name"]
349
+
350
+ self.io.info(
351
+ FileProvider.mkfile(
352
+ service_path,
353
+ "service-config.yml",
354
+ "",
355
+ overwrite=True,
356
+ )
357
+ )
358
+
359
+ current_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
360
+ message = f"# Generated by platform-helper {self.installed_version_provider.get_semantic_version(PLATFORM_HELPER_PACKAGE_NAME)} / {current_date}.\n\n"
361
+
362
+ self.file_provider.write(
363
+ f"{service_path}/service-config.yml", dict(service_manifest), message
364
+ )
365
+
366
+ def deploy(
367
+ self,
368
+ service: str,
369
+ environment: str,
370
+ application: str,
371
+ image_tag: str = None,
372
+ ):
373
+ """Register a new ECS task definition revision, update the ECS service
374
+ with it, monitor service, task and container logs, and wait until
375
+ deployment is complete."""
376
+
377
+ start_time = datetime.now(timezone.utc)
378
+ cluster_name = f"{application}-{environment}-cluster"
379
+ ecs_service_name = f"{application}-{environment}-{service}"
380
+
381
+ s3_response = self.s3_provider.get_object(
382
+ bucket_name=f"ecs-task-definitions-{application}-{environment}",
383
+ object_key=f"{application}/{environment}/{service}.json",
384
+ )
385
+
386
+ task_definition = json.loads(s3_response)
387
+
388
+ self.io.info(
389
+ f"Deploying image tag '{image_tag}' to service '{ecs_service_name}' in environment '{environment}'.\n"
390
+ )
391
+
392
+ task_def_arn = self.ecs_provider.register_task_definition(
393
+ service=service,
394
+ image_tag=image_tag,
395
+ task_definition=task_definition,
396
+ )
397
+
398
+ self._output_with_timestamp(
399
+ f"Task definition successfully registered with ARN '{task_def_arn}'."
400
+ )
401
+
402
+ autoscaling_response = self.autoscaling_provider.describe_autoscaling_target(
403
+ cluster_name=cluster_name, ecs_service_name=ecs_service_name
404
+ )
405
+ desired_count = autoscaling_response.get("MinCapacity", 1)
406
+
407
+ update_response = self.ecs_provider.update_service(
408
+ service=service,
409
+ task_def_arn=task_def_arn,
410
+ environment=environment,
411
+ application=application,
412
+ desired_count=desired_count,
413
+ )
414
+
415
+ self._output_with_timestamp(
416
+ f"Successfully updated ECS service '{update_response['serviceName']}'."
417
+ )
418
+
419
+ primary_deployment_id = self._get_primary_deployment_id(service_response=update_response)
420
+
421
+ self._output_with_timestamp(
422
+ f"New deployment with ID '{primary_deployment_id}' has been triggered."
423
+ )
424
+
425
+ seen_events = set()
426
+ deadline = time.monotonic() + DEPLOYMENT_TIMEOUT_SECONDS
427
+
428
+ while time.monotonic() < deadline:
429
+ service_response = self.ecs_provider.describe_service(
430
+ application=application, environment=environment, service=service
431
+ )
432
+ primary_deployment_id = self._get_primary_deployment_id(
433
+ service_response=service_response
434
+ )
435
+
436
+ self._monitor_service_events(
437
+ service_response=service_response, seen_events=seen_events, start_time=start_time
438
+ )
439
+
440
+ task_ids = self._wait_for_new_tasks(
441
+ cluster_name=cluster_name, deployment_id=primary_deployment_id
442
+ )
443
+ task_response = self.ecs_provider.describe_tasks(
444
+ cluster_name=f"{application}-{environment}-cluster",
445
+ task_ids=task_ids,
446
+ )
447
+
448
+ log_group = f"/platform/ecs/service/{application}/{environment}/{service}"
449
+ self._monitor_task_events(
450
+ task_response=task_response, seen_events=seen_events, log_group=log_group
451
+ )
452
+
453
+ state, reason = self.ecs_provider.get_service_deployment_state(
454
+ cluster_name=cluster_name,
455
+ service_name=ecs_service_name,
456
+ start_time=start_time.timestamp(),
457
+ )
458
+
459
+ if state == "SUCCESSFUL":
460
+ self._output_with_timestamp("Deployment complete.")
461
+ return
462
+ if state in ["STOPPED", "ROLLBACK_SUCCESSFUL", "ROLLBACK_FAILED"]:
463
+ raise PlatformException(f"Deployment failed: {reason or 'unknown reason'}")
464
+
465
+ time.sleep(POLL_INTERVAL_SECONDS)
466
+
467
+ raise PlatformException("Timed out waiting for service to stabilise.")
468
+
469
+ @staticmethod
470
+ def _get_primary_deployment_id(service_response: dict[str, Any]):
471
+ for dep in service_response["deployments"]:
472
+ if dep["status"] == "PRIMARY":
473
+ return dep["id"]
474
+ raise PlatformException(
475
+ f"Unable to find primary ECS deployment for service '{service_response['serviceName']}'."
476
+ )
477
+
478
+ def _monitor_service_events(
479
+ self, service_response: dict[str, Any], seen_events: set[str], start_time: datetime
480
+ ):
481
+ """Output ECS service events during deployment."""
482
+
483
+ for event in reversed(service_response.get("events", [])):
484
+ if event["id"] not in seen_events and event["createdAt"] > start_time:
485
+ seen_events.add(event["id"])
486
+ timestamp = event["createdAt"].strftime("%H:%M:%S")
487
+ message = event["message"]
488
+ self._output_with_timestamp(
489
+ message=message,
490
+ error=("error" in message or "failed" in message),
491
+ timestamp=timestamp,
492
+ )
493
+
494
+ def _monitor_task_events(
495
+ self, task_response: list[dict[str, Any]], seen_events: set[str], log_group: str
496
+ ):
497
+ """Output ECS task and container errors during deployment."""
498
+
499
+ for task in task_response:
500
+ for container in task["containers"]:
501
+ if container.get("exitCode", 0) != 1:
502
+ continue
503
+
504
+ task_id = task["taskArn"].split("/")[-1]
505
+ container_name = container["name"]
506
+ log_stream = f"platform/{container_name}/{task_id}"
507
+
508
+ if f"{task_id}-{container_name}" not in seen_events:
509
+ seen_events.add(f"{task_id}-{container_name}")
510
+ self._output_with_timestamp(
511
+ message=f"Container '{container_name}' stopped in task '{task_id}'.",
512
+ error=True,
513
+ )
514
+ log_url = f"https://eu-west-2.console.aws.amazon.com/cloudwatch/home?region=eu-west-2#logsV2:log-groups/log-group/{urllib.parse.quote_plus(log_group)}/log-events/{urllib.parse.quote_plus(log_stream)}"
515
+ self._output_with_timestamp(
516
+ message=f"View CloudWatch log: {log_url}", error=True
517
+ )
518
+
519
+ log_events = self.logs_provider.get_log_stream_events(
520
+ log_group=log_group,
521
+ log_stream=log_stream,
522
+ limit=20,
523
+ )
524
+
525
+ for event in log_events:
526
+ try:
527
+ message = json.loads(event["message"])
528
+ except json.decoder.JSONDecodeError:
529
+ message = event["message"]
530
+
531
+ if f"{task_id}-{message}" not in seen_events:
532
+ seen_events.add(f"{task_id}-{message}")
533
+ self._output_with_timestamp(message=message, error=True)
534
+
535
+ def _wait_for_new_tasks(self, cluster_name: str, deployment_id: str) -> list[str]:
536
+ """Return first ECS task ID started by the PRIMARY ECS deployment."""
537
+
538
+ timeout_seconds = 300
539
+ deadline = time.monotonic() + timeout_seconds
540
+
541
+ while time.monotonic() < deadline:
542
+ task_arns = self.ecs_provider.get_ecs_task_arns(
543
+ cluster=cluster_name,
544
+ started_by=deployment_id,
545
+ desired_status="RUNNING",
546
+ )
547
+
548
+ if task_arns:
549
+ break
550
+
551
+ time.sleep(POLL_INTERVAL_SECONDS)
552
+
553
+ if not task_arns:
554
+ raise PlatformException(
555
+ f"Timed out waiting for RUNNING ECS tasks to spin up after {timeout_seconds}s."
556
+ )
557
+
558
+ task_ids = []
559
+ for arn in task_arns:
560
+ task_ids.append(arn.rsplit("/", 1)[-1])
561
+ return task_ids
562
+
563
+ def _output_with_timestamp(self, message: str, error: bool = False, timestamp: datetime = None):
564
+ if not timestamp:
565
+ timestamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
566
+
567
+ if error:
568
+ self.io.deploy_error(f"[{timestamp}] {message}")
569
+ else:
570
+ self.io.info(f"[{timestamp}] {message}")
@@ -1,9 +1,7 @@
1
+ from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
1
2
  from dbt_platform_helper.platform_exception import PlatformException
2
3
  from dbt_platform_helper.providers.io import ClickIOProvider
3
4
  from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
4
- from dbt_platform_helper.utils.versioning import (
5
- get_required_terraform_platform_modules_version,
6
- )
7
5
 
8
6
 
9
7
  class TerraformEnvironmentException(PlatformException):
@@ -20,12 +18,20 @@ class TerraformEnvironment:
20
18
  config_provider,
21
19
  manifest_provider: TerraformManifestProvider = None,
22
20
  io: ClickIOProvider = ClickIOProvider(),
21
+ platform_helper_versioning: PlatformHelperVersioning = None,
23
22
  ):
24
23
  self.io = io
25
24
  self.config_provider = config_provider
26
25
  self.manifest_provider = manifest_provider or TerraformManifestProvider()
26
+ self.platform_helper_versioning = platform_helper_versioning
27
+
28
+ def generate(
29
+ self,
30
+ environment_name: str,
31
+ ):
32
+
33
+ self.platform_helper_versioning.check_platform_helper_version_mismatch()
27
34
 
28
- def generate(self, environment_name, terraform_platform_modules_version_override=None):
29
35
  config = self.config_provider.get_enriched_config()
30
36
 
31
37
  if environment_name not in config.get("environments").keys():
@@ -33,14 +39,9 @@ class TerraformEnvironment:
33
39
  f"cannot generate terraform for environment {environment_name}. It does not exist in your configuration"
34
40
  )
35
41
 
36
- platform_config_terraform_modules_default_version = config.get("default_versions", {}).get(
37
- "terraform-platform-modules", ""
38
- )
39
- terraform_platform_modules_version = get_required_terraform_platform_modules_version(
40
- terraform_platform_modules_version_override,
41
- platform_config_terraform_modules_default_version,
42
- )
43
-
44
42
  self.manifest_provider.generate_environment_config(
45
- config, environment_name, terraform_platform_modules_version
43
+ config,
44
+ environment_name,
45
+ self.platform_helper_versioning.get_template_version(),
46
+ self.platform_helper_versioning.get_extensions_module_source(),
46
47
  )