dbt-platform-helper 12.0.2__py3-none-any.whl → 12.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +2 -4
- dbt_platform_helper/commands/codebase.py +62 -228
- dbt_platform_helper/constants.py +1 -0
- dbt_platform_helper/domain/codebase.py +222 -0
- dbt_platform_helper/domain/database_copy.py +1 -1
- dbt_platform_helper/exceptions.py +28 -0
- dbt_platform_helper/utils/application.py +1 -4
- dbt_platform_helper/utils/aws.py +132 -0
- dbt_platform_helper/utils/files.py +70 -0
- dbt_platform_helper/utils/git.py +13 -0
- dbt_platform_helper/utils/validation.py +99 -2
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/METADATA +1 -1
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/RECORD +16 -24
- dbt_platform_helper/templates/env/overrides/.gitignore +0 -12
- dbt_platform_helper/templates/env/overrides/README.md +0 -11
- dbt_platform_helper/templates/env/overrides/bin/override.ts +0 -9
- dbt_platform_helper/templates/env/overrides/cdk.json +0 -20
- dbt_platform_helper/templates/env/overrides/log_resource_policy.json +0 -68
- dbt_platform_helper/templates/env/overrides/package-lock.json +0 -4307
- dbt_platform_helper/templates/env/overrides/package.json +0 -27
- dbt_platform_helper/templates/env/overrides/stack.ts +0 -51
- dbt_platform_helper/templates/env/overrides/tsconfig.json +0 -32
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-12.0.2.dist-info → dbt_platform_helper-12.1.0.dist-info}/entry_points.txt +0 -0
|
@@ -8,6 +8,7 @@ import yaml
|
|
|
8
8
|
from boto3 import Session
|
|
9
9
|
from yaml.parser import ParserError
|
|
10
10
|
|
|
11
|
+
from dbt_platform_helper.exceptions import ApplicationNotFoundError
|
|
11
12
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
12
13
|
from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
|
|
13
14
|
from dbt_platform_helper.utils.aws import get_ssm_secrets
|
|
@@ -67,10 +68,6 @@ class Application:
|
|
|
67
68
|
return str(self) == str(other)
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
class ApplicationNotFoundError(Exception):
|
|
71
|
-
pass
|
|
72
|
-
|
|
73
|
-
|
|
74
71
|
def load_application(app: str = None, default_session: Session = None) -> Application:
|
|
75
72
|
application = Application(app if app else get_application_name())
|
|
76
73
|
current_session = default_session if default_session else get_aws_session_or_abort()
|
dbt_platform_helper/utils/aws.py
CHANGED
|
@@ -13,7 +13,12 @@ import yaml
|
|
|
13
13
|
from boto3 import Session
|
|
14
14
|
|
|
15
15
|
from dbt_platform_helper.exceptions import AWSException
|
|
16
|
+
from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
|
|
17
|
+
from dbt_platform_helper.exceptions import ImageNotFoundError
|
|
16
18
|
from dbt_platform_helper.exceptions import ValidationException
|
|
19
|
+
from dbt_platform_helper.utils.files import cache_refresh_required
|
|
20
|
+
from dbt_platform_helper.utils.files import read_supported_versions_from_cache
|
|
21
|
+
from dbt_platform_helper.utils.files import write_to_cache
|
|
17
22
|
|
|
18
23
|
SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
|
|
19
24
|
SSM_PATH = "/copilot/{app}/{env}/secrets/{name}"
|
|
@@ -351,6 +356,59 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
|
|
|
351
356
|
return parameter_data
|
|
352
357
|
|
|
353
358
|
|
|
359
|
+
def get_supported_redis_versions():
|
|
360
|
+
|
|
361
|
+
if cache_refresh_required("redis"):
|
|
362
|
+
|
|
363
|
+
supported_versions = []
|
|
364
|
+
|
|
365
|
+
session = get_aws_session_or_abort()
|
|
366
|
+
elasticache_client = session.client("elasticache")
|
|
367
|
+
|
|
368
|
+
supported_versions_response = elasticache_client.describe_cache_engine_versions(
|
|
369
|
+
Engine="redis"
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
supported_versions = [
|
|
373
|
+
version["EngineVersion"]
|
|
374
|
+
for version in supported_versions_response["CacheEngineVersions"]
|
|
375
|
+
]
|
|
376
|
+
|
|
377
|
+
write_to_cache("redis", supported_versions)
|
|
378
|
+
|
|
379
|
+
return supported_versions
|
|
380
|
+
|
|
381
|
+
else:
|
|
382
|
+
return read_supported_versions_from_cache("redis")
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def get_supported_opensearch_versions():
|
|
386
|
+
|
|
387
|
+
if cache_refresh_required("opensearch"):
|
|
388
|
+
|
|
389
|
+
supported_versions = []
|
|
390
|
+
|
|
391
|
+
session = get_aws_session_or_abort()
|
|
392
|
+
opensearch_client = session.client("opensearch")
|
|
393
|
+
|
|
394
|
+
response = opensearch_client.list_versions()
|
|
395
|
+
all_versions = response["Versions"]
|
|
396
|
+
|
|
397
|
+
opensearch_versions = [
|
|
398
|
+
version for version in all_versions if not version.startswith("Elasticsearch_")
|
|
399
|
+
]
|
|
400
|
+
supported_versions = [
|
|
401
|
+
version.removeprefix("OpenSearch_") for version in opensearch_versions
|
|
402
|
+
]
|
|
403
|
+
|
|
404
|
+
write_to_cache("opensearch", supported_versions)
|
|
405
|
+
|
|
406
|
+
return supported_versions
|
|
407
|
+
|
|
408
|
+
else:
|
|
409
|
+
return read_supported_versions_from_cache("opensearch")
|
|
410
|
+
|
|
411
|
+
|
|
354
412
|
def get_connection_string(
|
|
355
413
|
session: Session,
|
|
356
414
|
app: str,
|
|
@@ -420,3 +478,77 @@ def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) ->
|
|
|
420
478
|
raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
|
|
421
479
|
|
|
422
480
|
return Vpc(subnets, sec_groups)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def start_build_extraction(codebuild_client, build_options):
|
|
484
|
+
response = codebuild_client.start_build(**build_options)
|
|
485
|
+
return response["build"]["arn"]
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def check_codebase_exists(session: Session, application, codebase: str):
|
|
489
|
+
try:
|
|
490
|
+
ssm_client = session.client("ssm")
|
|
491
|
+
ssm_client.get_parameter(
|
|
492
|
+
Name=f"/copilot/applications/{application.name}/codebases/{codebase}"
|
|
493
|
+
)["Parameter"]["Value"]
|
|
494
|
+
except (
|
|
495
|
+
KeyError,
|
|
496
|
+
ValueError,
|
|
497
|
+
ssm_client.exceptions.ParameterNotFound,
|
|
498
|
+
):
|
|
499
|
+
raise CopilotCodebaseNotFoundError
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def check_image_exists(session, application, codebase, commit):
|
|
503
|
+
ecr_client = session.client("ecr")
|
|
504
|
+
try:
|
|
505
|
+
ecr_client.describe_images(
|
|
506
|
+
repositoryName=f"{application.name}/{codebase}",
|
|
507
|
+
imageIds=[{"imageTag": f"commit-{commit}"}],
|
|
508
|
+
)
|
|
509
|
+
except (
|
|
510
|
+
ecr_client.exceptions.RepositoryNotFoundException,
|
|
511
|
+
ecr_client.exceptions.ImageNotFoundException,
|
|
512
|
+
):
|
|
513
|
+
raise ImageNotFoundError
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def get_build_url_from_arn(build_arn: str) -> str:
|
|
517
|
+
_, _, _, region, account_id, project_name, build_id = build_arn.split(":")
|
|
518
|
+
project_name = project_name.removeprefix("build/")
|
|
519
|
+
return (
|
|
520
|
+
f"https://eu-west-2.console.aws.amazon.com/codesuite/codebuild/{account_id}/projects/"
|
|
521
|
+
f"{project_name}/build/{project_name}%3A{build_id}"
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def list_latest_images(ecr_client, ecr_repository_name, codebase_repository, echo_fn):
|
|
526
|
+
paginator = ecr_client.get_paginator("describe_images")
|
|
527
|
+
describe_images_response_iterator = paginator.paginate(
|
|
528
|
+
repositoryName=ecr_repository_name,
|
|
529
|
+
filter={"tagStatus": "TAGGED"},
|
|
530
|
+
)
|
|
531
|
+
images = []
|
|
532
|
+
for page in describe_images_response_iterator:
|
|
533
|
+
images += page["imageDetails"]
|
|
534
|
+
|
|
535
|
+
sorted_images = sorted(
|
|
536
|
+
images,
|
|
537
|
+
key=lambda i: i["imagePushedAt"],
|
|
538
|
+
reverse=True,
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
MAX_RESULTS = 20
|
|
542
|
+
|
|
543
|
+
for image in sorted_images[:MAX_RESULTS]:
|
|
544
|
+
try:
|
|
545
|
+
commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
|
|
546
|
+
if not commit_tag:
|
|
547
|
+
continue
|
|
548
|
+
|
|
549
|
+
commit_hash = commit_tag.replace("commit-", "")
|
|
550
|
+
echo_fn(
|
|
551
|
+
f" - https://github.com/{codebase_repository}/commit/{commit_hash} - published: {image['imagePushedAt']}"
|
|
552
|
+
)
|
|
553
|
+
except StopIteration:
|
|
554
|
+
continue
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import os
|
|
1
2
|
from copy import deepcopy
|
|
3
|
+
from datetime import datetime
|
|
2
4
|
from os import makedirs
|
|
3
5
|
from pathlib import Path
|
|
4
6
|
|
|
@@ -7,6 +9,8 @@ import yaml
|
|
|
7
9
|
from jinja2 import Environment
|
|
8
10
|
from jinja2 import FileSystemLoader
|
|
9
11
|
|
|
12
|
+
from dbt_platform_helper.constants import PLATFORM_HELPER_CACHE_FILE
|
|
13
|
+
|
|
10
14
|
|
|
11
15
|
def to_yaml(value):
|
|
12
16
|
return yaml.dump(value, sort_keys=False)
|
|
@@ -102,3 +106,69 @@ def apply_environment_defaults(config):
|
|
|
102
106
|
enriched_config["environments"] = defaulted_envs
|
|
103
107
|
|
|
104
108
|
return enriched_config
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def read_supported_versions_from_cache(resource_name):
|
|
112
|
+
|
|
113
|
+
platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
|
|
114
|
+
|
|
115
|
+
return platform_helper_config.get(resource_name).get("versions")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def write_to_cache(resource_name, supported_versions):
|
|
119
|
+
|
|
120
|
+
platform_helper_config = {}
|
|
121
|
+
|
|
122
|
+
if os.path.exists(PLATFORM_HELPER_CACHE_FILE):
|
|
123
|
+
platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
|
|
124
|
+
|
|
125
|
+
cache_dict = {
|
|
126
|
+
resource_name: {
|
|
127
|
+
"versions": supported_versions,
|
|
128
|
+
"date-retrieved": datetime.now().strftime("%d-%m-%y %H:%M:%S"),
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
platform_helper_config.update(cache_dict)
|
|
133
|
+
|
|
134
|
+
with open(PLATFORM_HELPER_CACHE_FILE, "w") as file:
|
|
135
|
+
file.write("# [!] This file is autogenerated via the platform-helper. Do not edit.\n")
|
|
136
|
+
yaml.dump(platform_helper_config, file)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cache_refresh_required(resource_name) -> bool:
|
|
140
|
+
"""
|
|
141
|
+
Checks if the platform-helper should reach out to AWS to 'refresh' its
|
|
142
|
+
cached values.
|
|
143
|
+
|
|
144
|
+
An API call is needed if any of the following conditions are met:
|
|
145
|
+
1. No cache file (.platform-helper-config.yml) exists.
|
|
146
|
+
2. The resource name (e.g. redis, opensearch) does not exist within the cache file.
|
|
147
|
+
3. The date-retrieved value of the cached data is > than a time interval. In this case 1 day.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
if not os.path.exists(PLATFORM_HELPER_CACHE_FILE):
|
|
151
|
+
return True
|
|
152
|
+
|
|
153
|
+
platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
|
|
154
|
+
|
|
155
|
+
if platform_helper_config.get(resource_name):
|
|
156
|
+
return check_if_cached_datetime_is_greater_than_interval(
|
|
157
|
+
platform_helper_config[resource_name].get("date-retrieved"), 1
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def check_if_cached_datetime_is_greater_than_interval(date_retrieved, interval_in_days):
|
|
164
|
+
|
|
165
|
+
current_datetime = datetime.now()
|
|
166
|
+
cached_datetime = datetime.strptime(date_retrieved, "%d-%m-%y %H:%M:%S")
|
|
167
|
+
delta = current_datetime - cached_datetime
|
|
168
|
+
|
|
169
|
+
return delta.days > interval_in_days
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def read_file_as_yaml(file_name):
|
|
173
|
+
|
|
174
|
+
return yaml.safe_load(Path(file_name).read_text())
|
dbt_platform_helper/utils/git.py
CHANGED
|
@@ -2,6 +2,10 @@ import re
|
|
|
2
2
|
import subprocess
|
|
3
3
|
|
|
4
4
|
|
|
5
|
+
class CommitNotFoundError(Exception):
|
|
6
|
+
pass
|
|
7
|
+
|
|
8
|
+
|
|
5
9
|
def git_remote():
|
|
6
10
|
git_repo = subprocess.run(
|
|
7
11
|
["git", "remote", "get-url", "origin"], capture_output=True, text=True
|
|
@@ -14,3 +18,12 @@ def extract_repository_name(repository_url):
|
|
|
14
18
|
return
|
|
15
19
|
|
|
16
20
|
return re.search(r"([^/:]*/[^/]*)\.git", repository_url).group(1)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def check_if_commit_exists(commit):
|
|
24
|
+
branches_containing_commit = subprocess.run(
|
|
25
|
+
["git", "branch", "-r", "--contains", f"{commit}"], capture_output=True, text=True
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
if branches_containing_commit.stderr:
|
|
29
|
+
raise CommitNotFoundError()
|
|
@@ -18,6 +18,8 @@ from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
|
|
|
18
18
|
from dbt_platform_helper.constants import ENVIRONMENTS_KEY
|
|
19
19
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
20
20
|
from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
|
|
21
|
+
from dbt_platform_helper.utils.aws import get_supported_opensearch_versions
|
|
22
|
+
from dbt_platform_helper.utils.aws import get_supported_redis_versions
|
|
21
23
|
from dbt_platform_helper.utils.files import apply_environment_defaults
|
|
22
24
|
from dbt_platform_helper.utils.messages import abort_with_error
|
|
23
25
|
|
|
@@ -98,6 +100,19 @@ def validate_addons(addons: dict):
|
|
|
98
100
|
except SchemaError as ex:
|
|
99
101
|
errors[addon_name] = f"Error in {addon_name}: {ex.code}"
|
|
100
102
|
|
|
103
|
+
_validate_extension_supported_versions(
|
|
104
|
+
config={"extensions": addons},
|
|
105
|
+
extension_type="redis",
|
|
106
|
+
version_key="engine",
|
|
107
|
+
get_supported_versions_fn=get_supported_redis_versions,
|
|
108
|
+
)
|
|
109
|
+
_validate_extension_supported_versions(
|
|
110
|
+
config={"extensions": addons},
|
|
111
|
+
extension_type="opensearch",
|
|
112
|
+
version_key="engine",
|
|
113
|
+
get_supported_versions_fn=get_supported_opensearch_versions,
|
|
114
|
+
)
|
|
115
|
+
|
|
101
116
|
return errors
|
|
102
117
|
|
|
103
118
|
|
|
@@ -152,7 +167,7 @@ REDIS_PLANS = Or(
|
|
|
152
167
|
"x-large-ha",
|
|
153
168
|
)
|
|
154
169
|
|
|
155
|
-
REDIS_ENGINE_VERSIONS =
|
|
170
|
+
REDIS_ENGINE_VERSIONS = str
|
|
156
171
|
|
|
157
172
|
REDIS_DEFINITION = {
|
|
158
173
|
"type": "redis",
|
|
@@ -300,7 +315,7 @@ MONITORING_DEFINITION = {
|
|
|
300
315
|
OPENSEARCH_PLANS = Or(
|
|
301
316
|
"tiny", "small", "small-ha", "medium", "medium-ha", "large", "large-ha", "x-large", "x-large-ha"
|
|
302
317
|
)
|
|
303
|
-
OPENSEARCH_ENGINE_VERSIONS =
|
|
318
|
+
OPENSEARCH_ENGINE_VERSIONS = str
|
|
304
319
|
OPENSEARCH_MIN_VOLUME_SIZE = 10
|
|
305
320
|
OPENSEARCH_MAX_VOLUME_SIZE = {
|
|
306
321
|
"tiny": 100,
|
|
@@ -337,6 +352,32 @@ OPENSEARCH_DEFINITION = {
|
|
|
337
352
|
},
|
|
338
353
|
}
|
|
339
354
|
|
|
355
|
+
CACHE_POLICY_DEFINITION = {
|
|
356
|
+
"min_ttl": int,
|
|
357
|
+
"max_ttl": int,
|
|
358
|
+
"default_ttl": int,
|
|
359
|
+
"cookies_config": Or("none", "whitelist", "allExcept", "all"),
|
|
360
|
+
"header": Or("none", "whitelist"),
|
|
361
|
+
"query_string_behavior": Or("none", "whitelist", "allExcept", "all"),
|
|
362
|
+
Optional("cookie_list"): list,
|
|
363
|
+
Optional("headers_list"): list,
|
|
364
|
+
Optional("cache_policy_query_strings"): list,
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
PATHS_DEFINITION = {
|
|
368
|
+
Optional("default"): {
|
|
369
|
+
"cache": str,
|
|
370
|
+
"request": str,
|
|
371
|
+
},
|
|
372
|
+
Optional("additional"): list[
|
|
373
|
+
{
|
|
374
|
+
"path": str,
|
|
375
|
+
"cache": str,
|
|
376
|
+
"request": str,
|
|
377
|
+
}
|
|
378
|
+
],
|
|
379
|
+
}
|
|
380
|
+
|
|
340
381
|
ALB_DEFINITION = {
|
|
341
382
|
"type": "alb",
|
|
342
383
|
Optional("environments"): {
|
|
@@ -364,6 +405,9 @@ ALB_DEFINITION = {
|
|
|
364
405
|
Optional("viewer_certificate_minimum_protocol_version"): str,
|
|
365
406
|
Optional("viewer_certificate_ssl_support_method"): str,
|
|
366
407
|
Optional("viewer_protocol_policy"): str,
|
|
408
|
+
Optional("cache_policy"): dict({str: CACHE_POLICY_DEFINITION}),
|
|
409
|
+
Optional("origin_request_policy"): dict({str: {}}),
|
|
410
|
+
Optional("paths"): dict({str: PATHS_DEFINITION}),
|
|
367
411
|
},
|
|
368
412
|
None,
|
|
369
413
|
)
|
|
@@ -489,6 +533,59 @@ def validate_platform_config(config):
|
|
|
489
533
|
_validate_codebase_pipelines(enriched_config)
|
|
490
534
|
validate_database_copy_section(enriched_config)
|
|
491
535
|
|
|
536
|
+
_validate_extension_supported_versions(
|
|
537
|
+
config=config,
|
|
538
|
+
extension_type="redis",
|
|
539
|
+
version_key="engine",
|
|
540
|
+
get_supported_versions_fn=get_supported_redis_versions,
|
|
541
|
+
)
|
|
542
|
+
_validate_extension_supported_versions(
|
|
543
|
+
config=config,
|
|
544
|
+
extension_type="opensearch",
|
|
545
|
+
version_key="engine",
|
|
546
|
+
get_supported_versions_fn=get_supported_opensearch_versions,
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _validate_extension_supported_versions(
|
|
551
|
+
config, extension_type, version_key, get_supported_versions_fn
|
|
552
|
+
):
|
|
553
|
+
extensions = config.get("extensions", {})
|
|
554
|
+
if not extensions:
|
|
555
|
+
return
|
|
556
|
+
|
|
557
|
+
extensions_for_type = [
|
|
558
|
+
extension
|
|
559
|
+
for extension in config.get("extensions", {}).values()
|
|
560
|
+
if extension.get("type") == extension_type
|
|
561
|
+
]
|
|
562
|
+
|
|
563
|
+
supported_extension_versions = get_supported_versions_fn()
|
|
564
|
+
extensions_with_invalid_version = []
|
|
565
|
+
|
|
566
|
+
for extension in extensions_for_type:
|
|
567
|
+
|
|
568
|
+
environments = extension.get("environments", {})
|
|
569
|
+
|
|
570
|
+
if not isinstance(environments, dict):
|
|
571
|
+
click.secho(
|
|
572
|
+
"Error: Opensearch extension definition is invalid type, expected dictionary",
|
|
573
|
+
fg="red",
|
|
574
|
+
)
|
|
575
|
+
continue
|
|
576
|
+
for environment, env_config in environments.items():
|
|
577
|
+
extension_version = env_config.get(version_key)
|
|
578
|
+
if extension_version not in supported_extension_versions:
|
|
579
|
+
extensions_with_invalid_version.append(
|
|
580
|
+
{"environment": environment, "version": extension_version}
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
for version_failure in extensions_with_invalid_version:
|
|
584
|
+
click.secho(
|
|
585
|
+
f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
|
|
586
|
+
fg="red",
|
|
587
|
+
)
|
|
588
|
+
|
|
492
589
|
|
|
493
590
|
def validate_database_copy_section(config):
|
|
494
591
|
extensions = config.get("extensions", {})
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dbt-platform-helper
|
|
3
|
-
Version: 12.0
|
|
3
|
+
Version: 12.1.0
|
|
4
4
|
Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Department for Business and Trade Platform Team
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
dbt_platform_helper/COMMANDS.md,sha256=
|
|
1
|
+
dbt_platform_helper/COMMANDS.md,sha256=rQzvfy7xAtRP5mb-2DrvG0k_pcal1SO6kNL3cJTGzyk,21745
|
|
2
2
|
dbt_platform_helper/README.md,sha256=B0qN2_u_ASqqgkGDWY2iwNGZt_9tUgMb9XqtaTuzYjw,1530
|
|
3
3
|
dbt_platform_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
dbt_platform_helper/addon-plans.yml,sha256=O46a_ODsGG9KXmQY_1XbSGqrpSaHSLDe-SdROzHx8Go,4545
|
|
5
5
|
dbt_platform_helper/addons-template-map.yml,sha256=kYv_ZoZGWNeNBCnR_9wSeLhJuWOTHx-vn7ub74MgGb4,546
|
|
6
6
|
dbt_platform_helper/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
7
|
dbt_platform_helper/commands/application.py,sha256=idHLbEIJ3DH_7oL2_5ZID3BInLZjPYLSdFST0x7kvFE,10076
|
|
8
|
-
dbt_platform_helper/commands/codebase.py,sha256=
|
|
8
|
+
dbt_platform_helper/commands/codebase.py,sha256=IMaCYHqOHg8xNmGT1sU1pSrFUA-tO7Bj-mst-fKDpKI,5080
|
|
9
9
|
dbt_platform_helper/commands/conduit.py,sha256=cuD1obH_daebnLIsouDJNlxKXVmVjYJ0d3XZudqy1JM,15291
|
|
10
10
|
dbt_platform_helper/commands/config.py,sha256=NOHea7OAjrl6XHlW6HMLn0m0T5lFPyNH3HXoyCOWsJk,12070
|
|
11
11
|
dbt_platform_helper/commands/copilot.py,sha256=i7FLSF-p9P5JQ36e_V8THXxdXG_g1hI7fHxemxQG82A,12927
|
|
@@ -16,12 +16,13 @@ dbt_platform_helper/commands/notify.py,sha256=kVJ0s78QMiaEWPVKu_bbMko4DW2uJy2fu8
|
|
|
16
16
|
dbt_platform_helper/commands/pipeline.py,sha256=_52bDSDa8DoyOA4VFxFJhwaiKCPHKqPtK2LWDLFaKlA,9452
|
|
17
17
|
dbt_platform_helper/commands/secrets.py,sha256=2NtV5FGx-ErkMg2QMiDvFOp03cKVbLzgmY8Y504EKJw,3860
|
|
18
18
|
dbt_platform_helper/commands/version.py,sha256=XVfSd53TDti4cceBqTmRfq_yZnvxs14RbrOjNJHW75U,1444
|
|
19
|
-
dbt_platform_helper/constants.py,sha256=
|
|
19
|
+
dbt_platform_helper/constants.py,sha256=cDgvAwRc2bpkH_Aob4HrQyagL_6G9X0y3RjzLtvyjoE,297
|
|
20
20
|
dbt_platform_helper/default-extensions.yml,sha256=SU1ZitskbuEBpvE7efc3s56eAUF11j70brhj_XrNMMo,493
|
|
21
21
|
dbt_platform_helper/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
dbt_platform_helper/domain/
|
|
22
|
+
dbt_platform_helper/domain/codebase.py,sha256=hWzx9GaaMqAlkwhKXKFAVrTluNdhSyme4v6KJL15rRY,9380
|
|
23
|
+
dbt_platform_helper/domain/database_copy.py,sha256=QUw8XeUzcKccu4U33b9AKSOwCuGsONIzdmInlOzsmA4,8525
|
|
23
24
|
dbt_platform_helper/domain/maintenance_page.py,sha256=NFHN_J0NthhJ1YkcOTJ8c0R8y33TrDZq3ka2fmMRM1g,15708
|
|
24
|
-
dbt_platform_helper/exceptions.py,sha256=
|
|
25
|
+
dbt_platform_helper/exceptions.py,sha256=cf_GpBN9_JxwhPm-l6s9xQAuwKiA6WHX2Nt-5muXWx4,945
|
|
25
26
|
dbt_platform_helper/jinja2_tags.py,sha256=jFyN_Sxmko1GSfvrqRIGQ80CCW8EwlCV3su0ahJPfoE,541
|
|
26
27
|
dbt_platform_helper/providers/load_balancers.py,sha256=e1SPrWbBWq95paSVd3Y5yORIrHAZxcVabBYDjPyUTsU,1430
|
|
27
28
|
dbt_platform_helper/templates/.copilot/config.yml,sha256=J_bA9sCtBdCPBRImpCBRnYvhQd4vpLYIXIU-lq9vbkA,158
|
|
@@ -41,15 +42,6 @@ dbt_platform_helper/templates/ci-codebuild-role-policy.json,sha256=hNE-wGrraWxsJ
|
|
|
41
42
|
dbt_platform_helper/templates/create-codebuild-role.json,sha256=THJgIKi8rWwDzhg5ZxT8a0UkXKBfXZ-zsXm8St_ixPg,197
|
|
42
43
|
dbt_platform_helper/templates/custom-codebuild-role-policy.json,sha256=8xyCofilPhV1Yjt3OnQLcI2kZ35mk2c07GcqYrKxuoI,1180
|
|
43
44
|
dbt_platform_helper/templates/env/manifest.yml,sha256=VCEj_y3jdfnPYi6gmyrwiEqzHYjpaJDANbvswmkiLA0,802
|
|
44
|
-
dbt_platform_helper/templates/env/overrides/.gitignore,sha256=duFipE4GPjdkrYo-sO1jjCXdmSMoYNoFJLZ19lUKfL0,169
|
|
45
|
-
dbt_platform_helper/templates/env/overrides/README.md,sha256=n-wlfeSEqtY1Ajq4RVYl3xD6x804lz0YtezKTgB3Z00,443
|
|
46
|
-
dbt_platform_helper/templates/env/overrides/bin/override.ts,sha256=7_FT8mvS-yc5OcqJvVMvDQ4CEh5L_7hWFmApUXN2bbk,284
|
|
47
|
-
dbt_platform_helper/templates/env/overrides/cdk.json,sha256=ZbvoQdcj_k9k1GAD9qHUQcDfQPbMcBPjJwt2mu_S6ho,339
|
|
48
|
-
dbt_platform_helper/templates/env/overrides/log_resource_policy.json,sha256=_WBPaEuX_ImPNuFuhXTgLBY6P-FsnCLKhCYKgxGFyGk,1760
|
|
49
|
-
dbt_platform_helper/templates/env/overrides/package-lock.json,sha256=faLqSIOwLWCwz-9HWYGL5ovqmyMfv2w317FWQcO6_vA,155387
|
|
50
|
-
dbt_platform_helper/templates/env/overrides/package.json,sha256=dLDirCV_IljHPl1jfGsG7cH18YuTCeg95i3SBSOm8CI,536
|
|
51
|
-
dbt_platform_helper/templates/env/overrides/stack.ts,sha256=mh5xBJ_enJGUflIDlD-aRLKB_IE5OcvM6W8dGUdB-io,1910
|
|
52
|
-
dbt_platform_helper/templates/env/overrides/tsconfig.json,sha256=xOAruVlYLYq9vU1MiIruhrIXDWYVXjbxy9Y42bHh5xE,710
|
|
53
45
|
dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml,sha256=cFlg69fvi9kzpz13ZAeY1asseZ6TuUex-6s76jG3oL4,259
|
|
54
46
|
dbt_platform_helper/templates/environment-pipelines/main.tf,sha256=R5xM5OHmWD1BjUyl67uOVQ_PbgR96PuiYXlCDXoaj8c,1914
|
|
55
47
|
dbt_platform_helper/templates/environments/main.tf,sha256=6tgxyGLvllZCiN2ryYwNM6fcm6wijb-LMjwdb9UckyU,1579
|
|
@@ -74,22 +66,22 @@ dbt_platform_helper/templates/svc/manifest-backend.yml,sha256=aAD9ndkbXnF7JBAKS2
|
|
|
74
66
|
dbt_platform_helper/templates/svc/manifest-public.yml,sha256=6NHVR_onBu5hbwynLrB6roDRce7JcylSc0qeYvzlPdI,3664
|
|
75
67
|
dbt_platform_helper/templates/svc/overrides/cfn.patches.yml,sha256=W7-d017akuUq9kda64DQxazavcRcCPDjaAik6t1EZqM,742
|
|
76
68
|
dbt_platform_helper/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
|
-
dbt_platform_helper/utils/application.py,sha256=
|
|
69
|
+
dbt_platform_helper/utils/application.py,sha256=C-t8LIFqE-rrVBNqWEGRtCd6uOt41Y2t0z9_SEZm7oE,4127
|
|
78
70
|
dbt_platform_helper/utils/arn_parser.py,sha256=1jY0elpAe4YL3ulrrCf1YiKmjI-7YXz4gJASqkIFHTc,1294
|
|
79
|
-
dbt_platform_helper/utils/aws.py,sha256=
|
|
71
|
+
dbt_platform_helper/utils/aws.py,sha256=TkgDBelEcfBKQwiYiYZD1RMCxC4tZkRn0PyLyiM19Eo,18555
|
|
80
72
|
dbt_platform_helper/utils/click.py,sha256=Fx4y4bbve1zypvog_sgK7tJtCocmzheoEFLBRv1lfdM,2943
|
|
81
73
|
dbt_platform_helper/utils/cloudfoundry.py,sha256=GnQ4fVLnDfOdNSrsJjI6ElZHqpgwINeoPn77cUH2UFY,484
|
|
82
|
-
dbt_platform_helper/utils/files.py,sha256=
|
|
83
|
-
dbt_platform_helper/utils/git.py,sha256=
|
|
74
|
+
dbt_platform_helper/utils/files.py,sha256=pa5uwwrEFIAgXZUyH2KEr2Yiu6bKuV4EMlhn9BcUMsY,5296
|
|
75
|
+
dbt_platform_helper/utils/git.py,sha256=aI7pCst5hm4XACiubsTpGJV1UEBLnxLpUGXu4h4hofM,696
|
|
84
76
|
dbt_platform_helper/utils/manifests.py,sha256=ji3UYHCxq9tTpkm4MlRa2y0-JOYYqq1pWZ2h_zpj0UU,507
|
|
85
77
|
dbt_platform_helper/utils/messages.py,sha256=aLx6s9utt__IqlDdeIYq4n82ERwludu2Zfqy0Q2t-x8,115
|
|
86
78
|
dbt_platform_helper/utils/platform_config.py,sha256=2RfIxBAT5fv7WR4YuP3yomUK7sKZFL77xevuHnUALdg,676
|
|
87
79
|
dbt_platform_helper/utils/template.py,sha256=raRx4QUCVJtKfvJK08Egg6gwWcs3r3V4nPWcJW4xNhA,574
|
|
88
|
-
dbt_platform_helper/utils/validation.py,sha256=
|
|
80
|
+
dbt_platform_helper/utils/validation.py,sha256=LlSzg0kw3RuHp0xm1A8Jgo0pHJDJvpZVnBe5rlYy0LY,28175
|
|
89
81
|
dbt_platform_helper/utils/versioning.py,sha256=IBxdocJ8ZyJib38d1ja87tTuFE0iJ4npaDcAHQAKQ58,10825
|
|
90
82
|
platform_helper.py,sha256=bly3JkwbfwnWTZSZziu40dbgzQItsK-DIMMvL6ArFDY,1893
|
|
91
|
-
dbt_platform_helper-12.0.
|
|
92
|
-
dbt_platform_helper-12.0.
|
|
93
|
-
dbt_platform_helper-12.0.
|
|
94
|
-
dbt_platform_helper-12.0.
|
|
95
|
-
dbt_platform_helper-12.0.
|
|
83
|
+
dbt_platform_helper-12.1.0.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
|
|
84
|
+
dbt_platform_helper-12.1.0.dist-info/METADATA,sha256=hR2ugIXj8avyObLCR7BLN29snhNT4nxuTaixXrIgkEI,3161
|
|
85
|
+
dbt_platform_helper-12.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
86
|
+
dbt_platform_helper-12.1.0.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
|
|
87
|
+
dbt_platform_helper-12.1.0.dist-info/RECORD,,
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# Answers to some obvious questions
|
|
2
|
-
|
|
3
|
-
## Why not regular cfn.patches.yml for the overrides?
|
|
4
|
-
|
|
5
|
-
We need to add the contents of `extensions.yml` to Parameter Store.
|
|
6
|
-
|
|
7
|
-
This is then used by is used by `platform-helper conduit` so that you don't have to be in the `*-deploy` directory to run it.
|
|
8
|
-
|
|
9
|
-
## Why TypeScript and not Python?
|
|
10
|
-
|
|
11
|
-
Because [TypeScript is what AWS Copilot supports](https://aws.github.io/copilot-cli/en/docs/developing/overrides/cdk/).
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import * as cdk from 'aws-cdk-lib';
|
|
3
|
-
import { TransformedStack } from '../stack';
|
|
4
|
-
|
|
5
|
-
const app = new cdk.App();
|
|
6
|
-
new TransformedStack(app, 'Stack', {
|
|
7
|
-
appName: process.env.COPILOT_APPLICATION_NAME || "",
|
|
8
|
-
envName: process.env.COPILOT_ENVIRONMENT_NAME || "",
|
|
9
|
-
});
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"app": "npx ts-node --prefer-ts-exts bin/override.ts",
|
|
3
|
-
"versionReporting": false,
|
|
4
|
-
"watch": {
|
|
5
|
-
"include": [
|
|
6
|
-
"**"
|
|
7
|
-
],
|
|
8
|
-
"exclude": [
|
|
9
|
-
"README.md",
|
|
10
|
-
"cdk*.json",
|
|
11
|
-
"**/*.d.ts",
|
|
12
|
-
"**/*.js",
|
|
13
|
-
"tsconfig.json",
|
|
14
|
-
"package*.json",
|
|
15
|
-
"yarn.lock",
|
|
16
|
-
"node_modules",
|
|
17
|
-
"test"
|
|
18
|
-
]
|
|
19
|
-
}
|
|
20
|
-
}
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"Version": "2012-10-17",
|
|
3
|
-
"Statement": [
|
|
4
|
-
{
|
|
5
|
-
"Sid": "StateMachineToCloudWatchLogs123",
|
|
6
|
-
"Effect": "Allow",
|
|
7
|
-
"Principal": {
|
|
8
|
-
"Service": [
|
|
9
|
-
"delivery.logs.amazonaws.com"
|
|
10
|
-
]
|
|
11
|
-
},
|
|
12
|
-
"Action": [
|
|
13
|
-
"logs:CreateLogStream",
|
|
14
|
-
"logs:PutLogEvents"
|
|
15
|
-
],
|
|
16
|
-
"Resource": [
|
|
17
|
-
"arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/copilot/${AppName}-${EnvironmentName}-*:log-stream:*"
|
|
18
|
-
],
|
|
19
|
-
"Condition": {
|
|
20
|
-
"StringEquals": {
|
|
21
|
-
"aws:SourceAccount": "${AWS::AccountId}"
|
|
22
|
-
},
|
|
23
|
-
"ArnLike": {
|
|
24
|
-
"aws:SourceArn": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*"
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
},
|
|
28
|
-
{
|
|
29
|
-
"Effect": "Allow",
|
|
30
|
-
"Principal": {
|
|
31
|
-
"Service": [
|
|
32
|
-
"delivery.logs.amazonaws.com"
|
|
33
|
-
]
|
|
34
|
-
},
|
|
35
|
-
"Action": [
|
|
36
|
-
"logs:CreateLogStream",
|
|
37
|
-
"logs:PutLogEvents"
|
|
38
|
-
],
|
|
39
|
-
"Resource": "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/elasticache/${AppName}/${EnvironmentName}/*",
|
|
40
|
-
"Condition": {
|
|
41
|
-
"StringEquals": {
|
|
42
|
-
"aws:SourceAccount": "${AWS::AccountId}"
|
|
43
|
-
},
|
|
44
|
-
"ArnLike": {
|
|
45
|
-
"aws:SourceArn": "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*"
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"Effect": "Allow",
|
|
51
|
-
"Principal": {
|
|
52
|
-
"Service": [
|
|
53
|
-
"es.amazonaws.com"
|
|
54
|
-
]
|
|
55
|
-
},
|
|
56
|
-
"Action": [
|
|
57
|
-
"logs:PutLogEvents",
|
|
58
|
-
"logs:CreateLogStream"
|
|
59
|
-
],
|
|
60
|
-
"Resource": "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/opensearch/${AppName}/${EnvironmentName}/*",
|
|
61
|
-
"Condition": {
|
|
62
|
-
"StringEquals": {
|
|
63
|
-
"aws:SourceAccount": "${AWS::AccountId}"
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
]
|
|
68
|
-
}
|