dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +107 -27
- dbt_platform_helper/commands/application.py +5 -6
- dbt_platform_helper/commands/codebase.py +31 -10
- dbt_platform_helper/commands/conduit.py +3 -5
- dbt_platform_helper/commands/config.py +20 -311
- dbt_platform_helper/commands/copilot.py +18 -391
- dbt_platform_helper/commands/database.py +17 -9
- dbt_platform_helper/commands/environment.py +20 -14
- dbt_platform_helper/commands/generate.py +0 -3
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/notify.py +58 -78
- dbt_platform_helper/commands/pipeline.py +23 -19
- dbt_platform_helper/commands/secrets.py +39 -93
- dbt_platform_helper/commands/version.py +7 -12
- dbt_platform_helper/constants.py +52 -7
- dbt_platform_helper/domain/codebase.py +89 -39
- dbt_platform_helper/domain/conduit.py +335 -76
- dbt_platform_helper/domain/config.py +381 -0
- dbt_platform_helper/domain/copilot.py +398 -0
- dbt_platform_helper/domain/copilot_environment.py +8 -8
- dbt_platform_helper/domain/database_copy.py +2 -2
- dbt_platform_helper/domain/maintenance_page.py +254 -430
- dbt_platform_helper/domain/notify.py +64 -0
- dbt_platform_helper/domain/pipelines.py +43 -35
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +14 -13
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +249 -0
- dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
- dbt_platform_helper/entities/semantic_version.py +83 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/__init__.py +0 -0
- dbt_platform_helper/providers/aws/exceptions.py +70 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/aws/sso_auth.py +75 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/cloudformation.py +1 -1
- dbt_platform_helper/providers/config.py +137 -19
- dbt_platform_helper/providers/config_validator.py +112 -51
- dbt_platform_helper/providers/copilot.py +24 -16
- dbt_platform_helper/providers/ecr.py +89 -7
- dbt_platform_helper/providers/ecs.py +228 -36
- dbt_platform_helper/providers/environment_variable.py +24 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +36 -4
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +402 -42
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +134 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
- dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
- dbt_platform_helper/providers/schema_migrator.py +77 -0
- dbt_platform_helper/providers/secrets.py +5 -5
- dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
- dbt_platform_helper/providers/terraform_manifest.py +121 -19
- dbt_platform_helper/providers/version.py +106 -23
- dbt_platform_helper/providers/version_status.py +27 -0
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +58 -2
- dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utilities/decorators.py +103 -0
- dbt_platform_helper/utils/application.py +119 -22
- dbt_platform_helper/utils/aws.py +39 -150
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -14
- dbt_platform_helper/utils/validation.py +1 -1
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
- dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +3 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- dbt_platform_helper/providers/aws.py +0 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/providers/semantic_version.py +0 -126
- dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
- dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
- dbt_platform_helper/utils/cloudfoundry.py +0 -14
- dbt_platform_helper/utils/files.py +0 -53
- dbt_platform_helper/utils/manifests.py +0 -18
- dbt_platform_helper/utils/versioning.py +0 -238
- dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
import urllib.parse
|
|
5
|
+
from collections import OrderedDict
|
|
6
|
+
from copy import deepcopy
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from datetime import timezone
|
|
9
|
+
from importlib.metadata import version
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from dbt_platform_helper.constants import PLATFORM_HELPER_PACKAGE_NAME
|
|
14
|
+
from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
|
|
15
|
+
from dbt_platform_helper.constants import SERVICE_CONFIG_FILE
|
|
16
|
+
from dbt_platform_helper.constants import SERVICE_DIRECTORY
|
|
17
|
+
from dbt_platform_helper.constants import (
|
|
18
|
+
TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR,
|
|
19
|
+
)
|
|
20
|
+
from dbt_platform_helper.constants import TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR
|
|
21
|
+
from dbt_platform_helper.domain.terraform_environment import (
|
|
22
|
+
EnvironmentNotFoundException,
|
|
23
|
+
)
|
|
24
|
+
from dbt_platform_helper.entities.service import ServiceConfig
|
|
25
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
26
|
+
from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
|
|
27
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
28
|
+
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
29
|
+
from dbt_platform_helper.providers.ecs import ECS
|
|
30
|
+
from dbt_platform_helper.providers.environment_variable import (
|
|
31
|
+
EnvironmentVariableProvider,
|
|
32
|
+
)
|
|
33
|
+
from dbt_platform_helper.providers.files import FileProvider
|
|
34
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
35
|
+
from dbt_platform_helper.providers.logs import LogsProvider
|
|
36
|
+
from dbt_platform_helper.providers.s3 import S3Provider
|
|
37
|
+
from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
|
|
38
|
+
from dbt_platform_helper.providers.version import InstalledVersionProvider
|
|
39
|
+
from dbt_platform_helper.providers.yaml_file import YamlFileProvider
|
|
40
|
+
from dbt_platform_helper.utils.application import load_application
|
|
41
|
+
from dbt_platform_helper.utils.deep_merge import deep_merge
|
|
42
|
+
|
|
43
|
+
SERVICE_TYPES = ["Load Balanced Web Service", "Backend Service"]
|
|
44
|
+
DEPLOYMENT_TIMEOUT_SECONDS = 1200
|
|
45
|
+
POLL_INTERVAL_SECONDS = 5
|
|
46
|
+
|
|
47
|
+
# TODO add schema version to service config
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ServiceManager:
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
config_provider=ConfigProvider(ConfigValidator()),
|
|
54
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
55
|
+
file_provider=YamlFileProvider,
|
|
56
|
+
manifest_provider: TerraformManifestProvider = None,
|
|
57
|
+
platform_helper_version_override: str = None,
|
|
58
|
+
load_application=load_application,
|
|
59
|
+
installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
|
|
60
|
+
ecs_provider: ECS = None,
|
|
61
|
+
s3_provider: S3Provider = None,
|
|
62
|
+
logs_provider: LogsProvider = None,
|
|
63
|
+
autoscaling_provider: AutoscalingProvider = None,
|
|
64
|
+
):
|
|
65
|
+
|
|
66
|
+
self.file_provider = file_provider
|
|
67
|
+
self.config_provider = config_provider
|
|
68
|
+
self.io = io
|
|
69
|
+
self.manifest_provider = manifest_provider or TerraformManifestProvider()
|
|
70
|
+
self.platform_helper_version_override = (
|
|
71
|
+
platform_helper_version_override
|
|
72
|
+
or EnvironmentVariableProvider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
|
|
73
|
+
)
|
|
74
|
+
self.load_application = load_application
|
|
75
|
+
self.installed_version_provider = installed_version_provider
|
|
76
|
+
self.ecs_provider = ecs_provider
|
|
77
|
+
self.s3_provider = s3_provider
|
|
78
|
+
self.logs_provider = logs_provider
|
|
79
|
+
self.autoscaling_provider = autoscaling_provider
|
|
80
|
+
|
|
81
|
+
def generate(self, environment: str, services: list[str]):
|
|
82
|
+
|
|
83
|
+
config = self.config_provider.get_enriched_config()
|
|
84
|
+
application_name = config.get("application", "")
|
|
85
|
+
application = self.load_application(app=application_name)
|
|
86
|
+
|
|
87
|
+
if environment not in application.environments:
|
|
88
|
+
raise EnvironmentNotFoundException(
|
|
89
|
+
f"Cannot generate Terraform for environment '{environment}'. It does not exist in your configuration."
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
platform_helper_version_for_template: str = (
|
|
93
|
+
self.platform_helper_version_override
|
|
94
|
+
or config.get("default_versions", {}).get("platform-helper")
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
source_type = EnvironmentVariableProvider.get(TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR)
|
|
98
|
+
|
|
99
|
+
if source_type == "LOCAL":
|
|
100
|
+
module_source_override = ServiceConfig.local_terraform_source
|
|
101
|
+
elif source_type == "OVERRIDE":
|
|
102
|
+
module_source_override = EnvironmentVariableProvider.get(
|
|
103
|
+
TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR
|
|
104
|
+
)
|
|
105
|
+
else:
|
|
106
|
+
module_source_override = None
|
|
107
|
+
|
|
108
|
+
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
109
|
+
|
|
110
|
+
service_models = self.get_service_models(application, environment, services)
|
|
111
|
+
|
|
112
|
+
for service in service_models:
|
|
113
|
+
|
|
114
|
+
model_dump = service.model_dump(
|
|
115
|
+
exclude_none=True,
|
|
116
|
+
by_alias=True,
|
|
117
|
+
mode="json",
|
|
118
|
+
) # Use by_alias=True so that the Cooldown field 'in_' is written as 'in' in the output
|
|
119
|
+
|
|
120
|
+
output_path = Path(
|
|
121
|
+
f"terraform/{SERVICE_DIRECTORY}/{environment}/{service.name}/{SERVICE_CONFIG_FILE}"
|
|
122
|
+
)
|
|
123
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
|
|
125
|
+
self.file_provider.write(
|
|
126
|
+
str(output_path),
|
|
127
|
+
model_dump,
|
|
128
|
+
f"# WARNING: This is an autogenerated file, not for manual editing.\n# Generated by platform-helper {version('dbt-platform-helper')} / {timestamp}.\n",
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
self.manifest_provider.generate_service_config(
|
|
132
|
+
service,
|
|
133
|
+
environment,
|
|
134
|
+
platform_helper_version_for_template,
|
|
135
|
+
config,
|
|
136
|
+
module_source_override,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def get_service_models(self, application, environment, services=None) -> list[ServiceConfig]:
|
|
140
|
+
if not services:
|
|
141
|
+
services = []
|
|
142
|
+
try:
|
|
143
|
+
for dir in Path("services").iterdir():
|
|
144
|
+
if dir.is_dir():
|
|
145
|
+
config_path = dir / SERVICE_CONFIG_FILE
|
|
146
|
+
if config_path.exists():
|
|
147
|
+
services.append(dir.name)
|
|
148
|
+
else:
|
|
149
|
+
self.io.warn(
|
|
150
|
+
f"Failed loading service name from {dir.name}.\n"
|
|
151
|
+
"Please ensure that your '/services' directory follows the correct structure (i.e. /services/<service_name>/service-config.yml) and the 'service-config.yml' contents are correct."
|
|
152
|
+
)
|
|
153
|
+
except Exception as e:
|
|
154
|
+
self.io.abort_with_error(f"Failed extracting services with exception, {e}")
|
|
155
|
+
|
|
156
|
+
service_models = []
|
|
157
|
+
for service in services:
|
|
158
|
+
file_content = self.file_provider.load(
|
|
159
|
+
f"{SERVICE_DIRECTORY}/{service}/{SERVICE_CONFIG_FILE}"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
file_content = self.file_provider.find_and_replace(
|
|
163
|
+
config=file_content,
|
|
164
|
+
strings=[
|
|
165
|
+
"${PLATFORM_APPLICATION_NAME}",
|
|
166
|
+
"${PLATFORM_ENVIRONMENT_NAME}",
|
|
167
|
+
],
|
|
168
|
+
replacements=[application.name, environment],
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
env_overrides = file_content.get("environments", {}).get(environment)
|
|
172
|
+
if env_overrides:
|
|
173
|
+
merged_config = deep_merge(file_content, env_overrides)
|
|
174
|
+
else:
|
|
175
|
+
merged_config = file_content
|
|
176
|
+
merged_config.pop("environments", None)
|
|
177
|
+
|
|
178
|
+
service_model = ServiceConfig(**merged_config)
|
|
179
|
+
service_models.append(service_model)
|
|
180
|
+
|
|
181
|
+
return service_models
|
|
182
|
+
|
|
183
|
+
def migrate_copilot_manifests(self) -> None:
|
|
184
|
+
service_directory = Path("services/")
|
|
185
|
+
service_directory.mkdir(parents=True, exist_ok=True)
|
|
186
|
+
|
|
187
|
+
for dirname, _, filenames in os.walk("copilot"):
|
|
188
|
+
if "manifest.yml" in filenames and "environments" not in dirname:
|
|
189
|
+
copilot_manifest = self.file_provider.load(f"{dirname}/manifest.yml")
|
|
190
|
+
service_manifest = OrderedDict(deepcopy(copilot_manifest))
|
|
191
|
+
|
|
192
|
+
if service_manifest["type"] not in SERVICE_TYPES:
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
if "environments" in service_manifest:
|
|
196
|
+
for env in service_manifest["environments"]:
|
|
197
|
+
env_config = service_manifest["environments"][env]
|
|
198
|
+
if "http" in env_config:
|
|
199
|
+
if "alb" in env_config["http"]:
|
|
200
|
+
del env_config["http"]["alb"]
|
|
201
|
+
if isinstance(env_config["http"].get("alias", []), str):
|
|
202
|
+
env_config["http"]["alias"] = [env_config["http"]["alias"]]
|
|
203
|
+
if "healthcheck" in env_config["http"]:
|
|
204
|
+
if "interval" in env_config["http"]["healthcheck"]:
|
|
205
|
+
interval = env_config["http"]["healthcheck"]["interval"]
|
|
206
|
+
env_config["http"]["healthcheck"]["interval"] = int(
|
|
207
|
+
interval.rstrip("s")
|
|
208
|
+
)
|
|
209
|
+
if "timeout" in env_config["http"]["healthcheck"]:
|
|
210
|
+
timeout = env_config["http"]["healthcheck"]["timeout"]
|
|
211
|
+
env_config["http"]["healthcheck"]["timeout"] = int(
|
|
212
|
+
timeout.rstrip("s")
|
|
213
|
+
)
|
|
214
|
+
if "grace_period" in env_config["http"]["healthcheck"]:
|
|
215
|
+
grace_period = env_config["http"]["healthcheck"]["grace_period"]
|
|
216
|
+
env_config["http"]["healthcheck"]["grace_period"] = int(
|
|
217
|
+
grace_period.rstrip("s")
|
|
218
|
+
)
|
|
219
|
+
if "image" in env_config:
|
|
220
|
+
image_without_tag = env_config["image"]["location"].rsplit(":", 1)[0]
|
|
221
|
+
env_config["image"]["location"] = image_without_tag
|
|
222
|
+
if "healthcheck" in env_config["image"]:
|
|
223
|
+
if "interval" in env_config["image"]["healthcheck"]:
|
|
224
|
+
interval = env_config["image"]["healthcheck"]["interval"]
|
|
225
|
+
env_config["image"]["healthcheck"]["interval"] = int(
|
|
226
|
+
interval.rstrip("s")
|
|
227
|
+
)
|
|
228
|
+
if "timeout" in env_config["image"]["healthcheck"]:
|
|
229
|
+
timeout = env_config["image"]["healthcheck"]["timeout"]
|
|
230
|
+
env_config["image"]["healthcheck"]["timeout"] = int(
|
|
231
|
+
timeout.rstrip("s")
|
|
232
|
+
)
|
|
233
|
+
if "start_period" in env_config["image"]["healthcheck"]:
|
|
234
|
+
start_period = env_config["image"]["healthcheck"][
|
|
235
|
+
"start_period"
|
|
236
|
+
]
|
|
237
|
+
env_config["image"]["healthcheck"]["start_period"] = int(
|
|
238
|
+
start_period.rstrip("s")
|
|
239
|
+
)
|
|
240
|
+
if "count" in env_config:
|
|
241
|
+
if not isinstance(env_config.get("count"), int):
|
|
242
|
+
if "cooldown" in env_config["count"]:
|
|
243
|
+
if "in" in env_config["count"]["cooldown"]:
|
|
244
|
+
scaling_in = env_config["count"]["cooldown"]["in"]
|
|
245
|
+
env_config["count"]["cooldown"]["in"] = int(
|
|
246
|
+
scaling_in.rstrip("s")
|
|
247
|
+
)
|
|
248
|
+
if "out" in env_config["count"]["cooldown"]:
|
|
249
|
+
scaling_out = env_config["count"]["cooldown"]["out"]
|
|
250
|
+
env_config["count"]["cooldown"]["out"] = int(
|
|
251
|
+
scaling_out.rstrip("s")
|
|
252
|
+
)
|
|
253
|
+
if "network" in env_config:
|
|
254
|
+
del env_config["network"]
|
|
255
|
+
if "observability" in env_config:
|
|
256
|
+
del env_config["observability"]
|
|
257
|
+
|
|
258
|
+
if "healthcheck" in service_manifest.get("http", {}):
|
|
259
|
+
if "interval" in service_manifest["http"]["healthcheck"]:
|
|
260
|
+
interval = service_manifest["http"]["healthcheck"]["interval"]
|
|
261
|
+
service_manifest["http"]["healthcheck"]["interval"] = int(
|
|
262
|
+
interval.rstrip("s")
|
|
263
|
+
)
|
|
264
|
+
if "timeout" in service_manifest["http"]["healthcheck"]:
|
|
265
|
+
timeout = service_manifest["http"]["healthcheck"]["timeout"]
|
|
266
|
+
service_manifest["http"]["healthcheck"]["timeout"] = int(
|
|
267
|
+
timeout.rstrip("s")
|
|
268
|
+
)
|
|
269
|
+
if "grace_period" in service_manifest["http"]["healthcheck"]:
|
|
270
|
+
grace_period = service_manifest["http"]["healthcheck"]["grace_period"]
|
|
271
|
+
service_manifest["http"]["healthcheck"]["grace_period"] = int(
|
|
272
|
+
grace_period.rstrip("s")
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
if "image" in service_manifest:
|
|
276
|
+
if "location" in service_manifest["image"]:
|
|
277
|
+
image_without_tag = service_manifest["image"]["location"].rsplit(":", 1)[0]
|
|
278
|
+
service_manifest["image"]["location"] = image_without_tag
|
|
279
|
+
if "healthcheck" in service_manifest["image"]:
|
|
280
|
+
if "interval" in service_manifest["image"]["healthcheck"]:
|
|
281
|
+
interval = service_manifest["image"]["healthcheck"]["interval"]
|
|
282
|
+
service_manifest["image"]["healthcheck"]["interval"] = int(
|
|
283
|
+
interval.rstrip("s")
|
|
284
|
+
)
|
|
285
|
+
if "timeout" in service_manifest["image"]["healthcheck"]:
|
|
286
|
+
timeout = service_manifest["image"]["healthcheck"]["timeout"]
|
|
287
|
+
service_manifest["image"]["healthcheck"]["timeout"] = int(
|
|
288
|
+
timeout.rstrip("s")
|
|
289
|
+
)
|
|
290
|
+
if "start_period" in service_manifest["image"]["healthcheck"]:
|
|
291
|
+
start_period = service_manifest["image"]["healthcheck"]["start_period"]
|
|
292
|
+
service_manifest["image"]["healthcheck"]["start_period"] = int(
|
|
293
|
+
start_period.rstrip("s")
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
if "count" in service_manifest:
|
|
297
|
+
if not isinstance(service_manifest.get("count"), int):
|
|
298
|
+
if "cooldown" in service_manifest.get("count"):
|
|
299
|
+
if "in" in service_manifest["count"]["cooldown"]:
|
|
300
|
+
scaling_in = service_manifest["count"]["cooldown"]["in"]
|
|
301
|
+
service_manifest["count"]["cooldown"]["in"] = int(
|
|
302
|
+
scaling_in.rstrip("s")
|
|
303
|
+
)
|
|
304
|
+
if "out" in service_manifest["count"]["cooldown"]:
|
|
305
|
+
scaling_out = service_manifest["count"]["cooldown"]["out"]
|
|
306
|
+
service_manifest["count"]["cooldown"]["out"] = int(
|
|
307
|
+
scaling_out.rstrip("s")
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
if "network" in service_manifest:
|
|
311
|
+
del service_manifest["network"]
|
|
312
|
+
if "observability" in service_manifest:
|
|
313
|
+
del service_manifest["observability"]
|
|
314
|
+
|
|
315
|
+
if "entrypoint" in service_manifest:
|
|
316
|
+
if isinstance(service_manifest["entrypoint"], str):
|
|
317
|
+
service_manifest["entrypoint"] = [service_manifest["entrypoint"]]
|
|
318
|
+
|
|
319
|
+
if "alias" in service_manifest.get("http", {}):
|
|
320
|
+
if isinstance(service_manifest["http"]["alias"], str):
|
|
321
|
+
service_manifest["http"]["alias"] = [service_manifest["http"]["alias"]]
|
|
322
|
+
|
|
323
|
+
service_manifest = self.file_provider.find_and_replace(
|
|
324
|
+
config=service_manifest,
|
|
325
|
+
strings=["${COPILOT_APPLICATION_NAME}", "${COPILOT_ENVIRONMENT_NAME}"],
|
|
326
|
+
replacements=["${PLATFORM_APPLICATION_NAME}", "${PLATFORM_ENVIRONMENT_NAME}"],
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
service_manifest = self.file_provider.remove_empty_keys(service_manifest)
|
|
330
|
+
|
|
331
|
+
if "sidecars" in service_manifest:
|
|
332
|
+
new_sidecars = {}
|
|
333
|
+
writable_directories = []
|
|
334
|
+
|
|
335
|
+
for sidecar_name, sidecar in service_manifest["sidecars"].items():
|
|
336
|
+
if "chown" not in sidecar.get("command", "") and "chmod" not in sidecar.get(
|
|
337
|
+
"command", ""
|
|
338
|
+
):
|
|
339
|
+
new_sidecars[sidecar_name] = sidecar
|
|
340
|
+
if "chown" in sidecar.get("command", "") and "mount_points" in sidecar:
|
|
341
|
+
for mountpoint in sidecar["mount_points"]:
|
|
342
|
+
writable_directories.append(mountpoint["path"])
|
|
343
|
+
|
|
344
|
+
service_manifest["sidecars"] = new_sidecars
|
|
345
|
+
if "storage" in service_manifest:
|
|
346
|
+
service_manifest["storage"]["writable_directories"] = writable_directories
|
|
347
|
+
|
|
348
|
+
service_path = service_directory / service_manifest["name"]
|
|
349
|
+
|
|
350
|
+
self.io.info(
|
|
351
|
+
FileProvider.mkfile(
|
|
352
|
+
service_path,
|
|
353
|
+
"service-config.yml",
|
|
354
|
+
"",
|
|
355
|
+
overwrite=True,
|
|
356
|
+
)
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
current_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
360
|
+
message = f"# Generated by platform-helper {self.installed_version_provider.get_semantic_version(PLATFORM_HELPER_PACKAGE_NAME)} / {current_date}.\n\n"
|
|
361
|
+
|
|
362
|
+
self.file_provider.write(
|
|
363
|
+
f"{service_path}/service-config.yml", dict(service_manifest), message
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
def deploy(
|
|
367
|
+
self,
|
|
368
|
+
service: str,
|
|
369
|
+
environment: str,
|
|
370
|
+
application: str,
|
|
371
|
+
image_tag: str = None,
|
|
372
|
+
):
|
|
373
|
+
"""Register a new ECS task definition revision, update the ECS service
|
|
374
|
+
with it, monitor service, task and container logs, and wait until
|
|
375
|
+
deployment is complete."""
|
|
376
|
+
|
|
377
|
+
start_time = datetime.now(timezone.utc)
|
|
378
|
+
cluster_name = f"{application}-{environment}-cluster"
|
|
379
|
+
ecs_service_name = f"{application}-{environment}-{service}"
|
|
380
|
+
|
|
381
|
+
s3_response = self.s3_provider.get_object(
|
|
382
|
+
bucket_name=f"ecs-task-definitions-{application}-{environment}",
|
|
383
|
+
object_key=f"{application}/{environment}/{service}.json",
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
task_definition = json.loads(s3_response)
|
|
387
|
+
|
|
388
|
+
self.io.info(
|
|
389
|
+
f"Deploying image tag '{image_tag}' to service '{ecs_service_name}' in environment '{environment}'.\n"
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
task_def_arn = self.ecs_provider.register_task_definition(
|
|
393
|
+
service=service,
|
|
394
|
+
image_tag=image_tag,
|
|
395
|
+
task_definition=task_definition,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
self._output_with_timestamp(
|
|
399
|
+
f"Task definition successfully registered with ARN '{task_def_arn}'."
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
autoscaling_response = self.autoscaling_provider.describe_autoscaling_target(
|
|
403
|
+
cluster_name=cluster_name, ecs_service_name=ecs_service_name
|
|
404
|
+
)
|
|
405
|
+
desired_count = autoscaling_response.get("MinCapacity", 1)
|
|
406
|
+
|
|
407
|
+
update_response = self.ecs_provider.update_service(
|
|
408
|
+
service=service,
|
|
409
|
+
task_def_arn=task_def_arn,
|
|
410
|
+
environment=environment,
|
|
411
|
+
application=application,
|
|
412
|
+
desired_count=desired_count,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
self._output_with_timestamp(
|
|
416
|
+
f"Successfully updated ECS service '{update_response['serviceName']}'."
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
primary_deployment_id = self._get_primary_deployment_id(service_response=update_response)
|
|
420
|
+
|
|
421
|
+
self._output_with_timestamp(
|
|
422
|
+
f"New deployment with ID '{primary_deployment_id}' has been triggered."
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
seen_events = set()
|
|
426
|
+
deadline = time.monotonic() + DEPLOYMENT_TIMEOUT_SECONDS
|
|
427
|
+
|
|
428
|
+
while time.monotonic() < deadline:
|
|
429
|
+
service_response = self.ecs_provider.describe_service(
|
|
430
|
+
application=application, environment=environment, service=service
|
|
431
|
+
)
|
|
432
|
+
primary_deployment_id = self._get_primary_deployment_id(
|
|
433
|
+
service_response=service_response
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
self._monitor_service_events(
|
|
437
|
+
service_response=service_response, seen_events=seen_events, start_time=start_time
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
task_ids = self._wait_for_new_tasks(
|
|
441
|
+
cluster_name=cluster_name, deployment_id=primary_deployment_id
|
|
442
|
+
)
|
|
443
|
+
task_response = self.ecs_provider.describe_tasks(
|
|
444
|
+
cluster_name=f"{application}-{environment}-cluster",
|
|
445
|
+
task_ids=task_ids,
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
log_group = f"/platform/ecs/service/{application}/{environment}/{service}"
|
|
449
|
+
self._monitor_task_events(
|
|
450
|
+
task_response=task_response, seen_events=seen_events, log_group=log_group
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
state, reason = self.ecs_provider.get_service_deployment_state(
|
|
454
|
+
cluster_name=cluster_name,
|
|
455
|
+
service_name=ecs_service_name,
|
|
456
|
+
start_time=start_time.timestamp(),
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
if state == "SUCCESSFUL":
|
|
460
|
+
self._output_with_timestamp("Deployment complete.")
|
|
461
|
+
return
|
|
462
|
+
if state in ["STOPPED", "ROLLBACK_SUCCESSFUL", "ROLLBACK_FAILED"]:
|
|
463
|
+
raise PlatformException(f"Deployment failed: {reason or 'unknown reason'}")
|
|
464
|
+
|
|
465
|
+
time.sleep(POLL_INTERVAL_SECONDS)
|
|
466
|
+
|
|
467
|
+
raise PlatformException("Timed out waiting for service to stabilise.")
|
|
468
|
+
|
|
469
|
+
@staticmethod
|
|
470
|
+
def _get_primary_deployment_id(service_response: dict[str, Any]):
|
|
471
|
+
for dep in service_response["deployments"]:
|
|
472
|
+
if dep["status"] == "PRIMARY":
|
|
473
|
+
return dep["id"]
|
|
474
|
+
raise PlatformException(
|
|
475
|
+
f"Unable to find primary ECS deployment for service '{service_response['serviceName']}'."
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
def _monitor_service_events(
|
|
479
|
+
self, service_response: dict[str, Any], seen_events: set[str], start_time: datetime
|
|
480
|
+
):
|
|
481
|
+
"""Output ECS service events during deployment."""
|
|
482
|
+
|
|
483
|
+
for event in reversed(service_response.get("events", [])):
|
|
484
|
+
if event["id"] not in seen_events and event["createdAt"] > start_time:
|
|
485
|
+
seen_events.add(event["id"])
|
|
486
|
+
timestamp = event["createdAt"].strftime("%H:%M:%S")
|
|
487
|
+
message = event["message"]
|
|
488
|
+
self._output_with_timestamp(
|
|
489
|
+
message=message,
|
|
490
|
+
error=("error" in message or "failed" in message),
|
|
491
|
+
timestamp=timestamp,
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
def _monitor_task_events(
|
|
495
|
+
self, task_response: list[dict[str, Any]], seen_events: set[str], log_group: str
|
|
496
|
+
):
|
|
497
|
+
"""Output ECS task and container errors during deployment."""
|
|
498
|
+
|
|
499
|
+
for task in task_response:
|
|
500
|
+
for container in task["containers"]:
|
|
501
|
+
if container.get("exitCode", 0) != 1:
|
|
502
|
+
continue
|
|
503
|
+
|
|
504
|
+
task_id = task["taskArn"].split("/")[-1]
|
|
505
|
+
container_name = container["name"]
|
|
506
|
+
log_stream = f"platform/{container_name}/{task_id}"
|
|
507
|
+
|
|
508
|
+
if f"{task_id}-{container_name}" not in seen_events:
|
|
509
|
+
seen_events.add(f"{task_id}-{container_name}")
|
|
510
|
+
self._output_with_timestamp(
|
|
511
|
+
message=f"Container '{container_name}' stopped in task '{task_id}'.",
|
|
512
|
+
error=True,
|
|
513
|
+
)
|
|
514
|
+
log_url = f"https://eu-west-2.console.aws.amazon.com/cloudwatch/home?region=eu-west-2#logsV2:log-groups/log-group/{urllib.parse.quote_plus(log_group)}/log-events/{urllib.parse.quote_plus(log_stream)}"
|
|
515
|
+
self._output_with_timestamp(
|
|
516
|
+
message=f"View CloudWatch log: {log_url}", error=True
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
log_events = self.logs_provider.get_log_stream_events(
|
|
520
|
+
log_group=log_group,
|
|
521
|
+
log_stream=log_stream,
|
|
522
|
+
limit=20,
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
for event in log_events:
|
|
526
|
+
try:
|
|
527
|
+
message = json.loads(event["message"])
|
|
528
|
+
except json.decoder.JSONDecodeError:
|
|
529
|
+
message = event["message"]
|
|
530
|
+
|
|
531
|
+
if f"{task_id}-{message}" not in seen_events:
|
|
532
|
+
seen_events.add(f"{task_id}-{message}")
|
|
533
|
+
self._output_with_timestamp(message=message, error=True)
|
|
534
|
+
|
|
535
|
+
def _wait_for_new_tasks(self, cluster_name: str, deployment_id: str) -> list[str]:
|
|
536
|
+
"""Return first ECS task ID started by the PRIMARY ECS deployment."""
|
|
537
|
+
|
|
538
|
+
timeout_seconds = 300
|
|
539
|
+
deadline = time.monotonic() + timeout_seconds
|
|
540
|
+
|
|
541
|
+
while time.monotonic() < deadline:
|
|
542
|
+
task_arns = self.ecs_provider.get_ecs_task_arns(
|
|
543
|
+
cluster=cluster_name,
|
|
544
|
+
started_by=deployment_id,
|
|
545
|
+
desired_status="RUNNING",
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
if task_arns:
|
|
549
|
+
break
|
|
550
|
+
|
|
551
|
+
time.sleep(POLL_INTERVAL_SECONDS)
|
|
552
|
+
|
|
553
|
+
if not task_arns:
|
|
554
|
+
raise PlatformException(
|
|
555
|
+
f"Timed out waiting for RUNNING ECS tasks to spin up after {timeout_seconds}s."
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
task_ids = []
|
|
559
|
+
for arn in task_arns:
|
|
560
|
+
task_ids.append(arn.rsplit("/", 1)[-1])
|
|
561
|
+
return task_ids
|
|
562
|
+
|
|
563
|
+
def _output_with_timestamp(self, message: str, error: bool = False, timestamp: datetime = None):
|
|
564
|
+
if not timestamp:
|
|
565
|
+
timestamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
|
|
566
|
+
|
|
567
|
+
if error:
|
|
568
|
+
self.io.deploy_error(f"[{timestamp}] {message}")
|
|
569
|
+
else:
|
|
570
|
+
self.io.info(f"[{timestamp}] {message}")
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
1
2
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
2
3
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
3
4
|
from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
|
|
4
|
-
from dbt_platform_helper.utils.versioning import (
|
|
5
|
-
get_required_terraform_platform_modules_version,
|
|
6
|
-
)
|
|
7
5
|
|
|
8
6
|
|
|
9
7
|
class TerraformEnvironmentException(PlatformException):
|
|
@@ -20,12 +18,20 @@ class TerraformEnvironment:
|
|
|
20
18
|
config_provider,
|
|
21
19
|
manifest_provider: TerraformManifestProvider = None,
|
|
22
20
|
io: ClickIOProvider = ClickIOProvider(),
|
|
21
|
+
platform_helper_versioning: PlatformHelperVersioning = None,
|
|
23
22
|
):
|
|
24
23
|
self.io = io
|
|
25
24
|
self.config_provider = config_provider
|
|
26
25
|
self.manifest_provider = manifest_provider or TerraformManifestProvider()
|
|
26
|
+
self.platform_helper_versioning = platform_helper_versioning
|
|
27
|
+
|
|
28
|
+
def generate(
|
|
29
|
+
self,
|
|
30
|
+
environment_name: str,
|
|
31
|
+
):
|
|
32
|
+
|
|
33
|
+
self.platform_helper_versioning.check_platform_helper_version_mismatch()
|
|
27
34
|
|
|
28
|
-
def generate(self, environment_name, terraform_platform_modules_version_override=None):
|
|
29
35
|
config = self.config_provider.get_enriched_config()
|
|
30
36
|
|
|
31
37
|
if environment_name not in config.get("environments").keys():
|
|
@@ -33,14 +39,9 @@ class TerraformEnvironment:
|
|
|
33
39
|
f"cannot generate terraform for environment {environment_name}. It does not exist in your configuration"
|
|
34
40
|
)
|
|
35
41
|
|
|
36
|
-
platform_config_terraform_modules_default_version = config.get("default_versions", {}).get(
|
|
37
|
-
"terraform-platform-modules", ""
|
|
38
|
-
)
|
|
39
|
-
terraform_platform_modules_version = get_required_terraform_platform_modules_version(
|
|
40
|
-
terraform_platform_modules_version_override,
|
|
41
|
-
platform_config_terraform_modules_default_version,
|
|
42
|
-
)
|
|
43
|
-
|
|
44
42
|
self.manifest_provider.generate_environment_config(
|
|
45
|
-
config,
|
|
43
|
+
config,
|
|
44
|
+
environment_name,
|
|
45
|
+
self.platform_helper_versioning.get_template_version(),
|
|
46
|
+
self.platform_helper_versioning.get_extensions_module_source(),
|
|
46
47
|
)
|