dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +107 -27
- dbt_platform_helper/commands/application.py +5 -6
- dbt_platform_helper/commands/codebase.py +31 -10
- dbt_platform_helper/commands/conduit.py +3 -5
- dbt_platform_helper/commands/config.py +20 -311
- dbt_platform_helper/commands/copilot.py +18 -391
- dbt_platform_helper/commands/database.py +17 -9
- dbt_platform_helper/commands/environment.py +20 -14
- dbt_platform_helper/commands/generate.py +0 -3
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/notify.py +58 -78
- dbt_platform_helper/commands/pipeline.py +23 -19
- dbt_platform_helper/commands/secrets.py +39 -93
- dbt_platform_helper/commands/version.py +7 -12
- dbt_platform_helper/constants.py +52 -7
- dbt_platform_helper/domain/codebase.py +89 -39
- dbt_platform_helper/domain/conduit.py +335 -76
- dbt_platform_helper/domain/config.py +381 -0
- dbt_platform_helper/domain/copilot.py +398 -0
- dbt_platform_helper/domain/copilot_environment.py +8 -8
- dbt_platform_helper/domain/database_copy.py +2 -2
- dbt_platform_helper/domain/maintenance_page.py +254 -430
- dbt_platform_helper/domain/notify.py +64 -0
- dbt_platform_helper/domain/pipelines.py +43 -35
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +14 -13
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +249 -0
- dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
- dbt_platform_helper/entities/semantic_version.py +83 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/__init__.py +0 -0
- dbt_platform_helper/providers/aws/exceptions.py +70 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/aws/sso_auth.py +75 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/cloudformation.py +1 -1
- dbt_platform_helper/providers/config.py +137 -19
- dbt_platform_helper/providers/config_validator.py +112 -51
- dbt_platform_helper/providers/copilot.py +24 -16
- dbt_platform_helper/providers/ecr.py +89 -7
- dbt_platform_helper/providers/ecs.py +228 -36
- dbt_platform_helper/providers/environment_variable.py +24 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +36 -4
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +402 -42
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +134 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
- dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
- dbt_platform_helper/providers/schema_migrator.py +77 -0
- dbt_platform_helper/providers/secrets.py +5 -5
- dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
- dbt_platform_helper/providers/terraform_manifest.py +121 -19
- dbt_platform_helper/providers/version.py +106 -23
- dbt_platform_helper/providers/version_status.py +27 -0
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +58 -2
- dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utilities/decorators.py +103 -0
- dbt_platform_helper/utils/application.py +119 -22
- dbt_platform_helper/utils/aws.py +39 -150
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -14
- dbt_platform_helper/utils/validation.py +1 -1
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
- dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +3 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- dbt_platform_helper/providers/aws.py +0 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/providers/semantic_version.py +0 -126
- dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
- dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
- dbt_platform_helper/utils/cloudfoundry.py +0 -14
- dbt_platform_helper/utils/files.py +0 -53
- dbt_platform_helper/utils/manifests.py +0 -18
- dbt_platform_helper/utils/versioning.py +0 -238
- dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
from botocore.exceptions import ClientError
|
|
3
|
+
|
|
4
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class S3Provider:
|
|
8
|
+
|
|
9
|
+
def __init__(self, client: boto3.client):
|
|
10
|
+
self.client = client
|
|
11
|
+
|
|
12
|
+
def get_object(self, bucket_name: str, object_key: str) -> str:
|
|
13
|
+
"""Returns an object from an S3 bucket."""
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
content = self.client.get_object(Bucket=bucket_name, Key=object_key)
|
|
17
|
+
return content["Body"].read().decode("utf-8")
|
|
18
|
+
except ClientError as e:
|
|
19
|
+
raise PlatformException(
|
|
20
|
+
f"Failed to get '{object_key}' from '{bucket_name}'. Error: {e}"
|
|
21
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SchemaV0ToV1Migration:
|
|
5
|
+
def from_version(self) -> int:
|
|
6
|
+
return 0
|
|
7
|
+
|
|
8
|
+
def migrate(self, platform_config: dict) -> dict:
|
|
9
|
+
migrated_config = deepcopy(platform_config)
|
|
10
|
+
|
|
11
|
+
self._remove_terraform_platform_modules_default_version(migrated_config)
|
|
12
|
+
self._remove_versions_from_env_config(migrated_config)
|
|
13
|
+
self._remove_to_account_and_from_account_from_database_copy(migrated_config)
|
|
14
|
+
self._remove_pipeline_platform_helper_override(migrated_config)
|
|
15
|
+
|
|
16
|
+
return migrated_config
|
|
17
|
+
|
|
18
|
+
def _remove_versions_from_env_config(self, migrated_config: dict) -> None:
|
|
19
|
+
for env_name, env in migrated_config.get("environments", {}).items():
|
|
20
|
+
if env and "versions" in env:
|
|
21
|
+
del env["versions"]
|
|
22
|
+
|
|
23
|
+
def _remove_terraform_platform_modules_default_version(self, migrated_config: dict) -> None:
|
|
24
|
+
if "default_versions" in migrated_config:
|
|
25
|
+
default_versions = migrated_config["default_versions"]
|
|
26
|
+
if "terraform-platform-modules" in default_versions:
|
|
27
|
+
del default_versions["terraform-platform-modules"]
|
|
28
|
+
|
|
29
|
+
def _remove_to_account_and_from_account_from_database_copy(self, migrated_config: dict) -> None:
|
|
30
|
+
for extension_name, extension in migrated_config.get("extensions", {}).items():
|
|
31
|
+
if extension.get("type") == "postgres" and "database_copy" in extension:
|
|
32
|
+
for database_copy_block in extension["database_copy"]:
|
|
33
|
+
if "from_account" in database_copy_block:
|
|
34
|
+
del database_copy_block["from_account"]
|
|
35
|
+
if "to_account" in database_copy_block:
|
|
36
|
+
del database_copy_block["to_account"]
|
|
37
|
+
|
|
38
|
+
def _remove_pipeline_platform_helper_override(self, migrated_config: dict) -> None:
|
|
39
|
+
for pipeline_name, pipeline_config in migrated_config.get(
|
|
40
|
+
"environment_pipelines", {}
|
|
41
|
+
).items():
|
|
42
|
+
if "versions" in pipeline_config:
|
|
43
|
+
del pipeline_config["versions"]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from collections import Counter
|
|
2
|
+
from collections import OrderedDict
|
|
3
|
+
from copy import deepcopy
|
|
4
|
+
from typing import Protocol
|
|
5
|
+
|
|
6
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
7
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
8
|
+
from dbt_platform_helper.providers.schema_migrations.schema_v0_to_v1_migration import (
|
|
9
|
+
SchemaV0ToV1Migration,
|
|
10
|
+
)
|
|
11
|
+
from dbt_platform_helper.providers.version import InstalledVersionProvider
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class InvalidMigrationConfigurationException(PlatformException):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SchemaMigrationProtocol(Protocol):
|
|
19
|
+
def from_version(self) -> int: ...
|
|
20
|
+
|
|
21
|
+
def migrate(self, platform_config: dict) -> dict: ...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# TODO: Possibly get this programmatically?
|
|
25
|
+
ALL_MIGRATIONS = [SchemaV0ToV1Migration()]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Migrator:
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
migrations: list[SchemaMigrationProtocol],
|
|
32
|
+
installed_version_provider: InstalledVersionProvider = InstalledVersionProvider,
|
|
33
|
+
io_provider: ClickIOProvider = ClickIOProvider(),
|
|
34
|
+
):
|
|
35
|
+
self.migrations = sorted(migrations, key=lambda m: m.from_version())
|
|
36
|
+
self.installed_version_provider = installed_version_provider
|
|
37
|
+
self.io_provider = io_provider
|
|
38
|
+
from_version_counts = Counter([migration.from_version() for migration in self.migrations])
|
|
39
|
+
duplicate_from_versions = [count for count in from_version_counts.values() if count > 1]
|
|
40
|
+
|
|
41
|
+
if duplicate_from_versions:
|
|
42
|
+
raise InvalidMigrationConfigurationException(
|
|
43
|
+
"`from_version` parameters must be unique amongst migrations"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
def migrate(self, platform_config: dict) -> dict:
|
|
47
|
+
out = OrderedDict(deepcopy(platform_config))
|
|
48
|
+
if "schema_version" not in out:
|
|
49
|
+
out["schema_version"] = 0
|
|
50
|
+
|
|
51
|
+
if "default_versions" in out:
|
|
52
|
+
out.move_to_end("default_versions", last=False)
|
|
53
|
+
if "schema_version" in out:
|
|
54
|
+
out.move_to_end("schema_version", last=False)
|
|
55
|
+
if "application" in out:
|
|
56
|
+
out.move_to_end("application", last=False)
|
|
57
|
+
|
|
58
|
+
for migration in self.migrations:
|
|
59
|
+
migration_can_be_applied = migration.from_version() == out["schema_version"]
|
|
60
|
+
if migration_can_be_applied:
|
|
61
|
+
out = migration.migrate(out)
|
|
62
|
+
schema_version = out["schema_version"]
|
|
63
|
+
self.io_provider.info(
|
|
64
|
+
f"Migrating from platform config schema version {schema_version} to version {schema_version + 1}"
|
|
65
|
+
)
|
|
66
|
+
out["schema_version"] += 1
|
|
67
|
+
|
|
68
|
+
if "default_versions" not in out:
|
|
69
|
+
out["default_versions"] = {}
|
|
70
|
+
|
|
71
|
+
out["default_versions"]["platform-helper"] = str(
|
|
72
|
+
self.installed_version_provider.get_semantic_version("dbt-platform-helper")
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
self.io_provider.info("\nMigration complete")
|
|
76
|
+
|
|
77
|
+
return dict(out)
|
|
@@ -41,7 +41,7 @@ class Secrets:
|
|
|
41
41
|
|
|
42
42
|
raise SecretNotFoundException(secret_name)
|
|
43
43
|
|
|
44
|
-
#
|
|
44
|
+
# TODO: DBTP-1946: This probably does not belong in the secrets provider. When it moves, take the Todoed exceptions from below
|
|
45
45
|
def get_addon_type(self, addon_name: str) -> str:
|
|
46
46
|
addon_type = None
|
|
47
47
|
try:
|
|
@@ -82,18 +82,18 @@ class Secrets:
|
|
|
82
82
|
return addon_name.replace("-", "_").upper()
|
|
83
83
|
|
|
84
84
|
|
|
85
|
-
#
|
|
85
|
+
# TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
|
|
86
86
|
class AddonException(PlatformException):
|
|
87
87
|
pass
|
|
88
88
|
|
|
89
89
|
|
|
90
|
-
#
|
|
90
|
+
# TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
|
|
91
91
|
class AddonNotFoundException(AddonException):
|
|
92
92
|
def __init__(self, addon_name: str):
|
|
93
93
|
super().__init__(f"""Addon "{addon_name}" does not exist.""")
|
|
94
94
|
|
|
95
95
|
|
|
96
|
-
#
|
|
96
|
+
# TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
|
|
97
97
|
class AddonTypeMissingFromConfigException(AddonException):
|
|
98
98
|
def __init__(self, addon_name: str):
|
|
99
99
|
super().__init__(
|
|
@@ -101,7 +101,7 @@ class AddonTypeMissingFromConfigException(AddonException):
|
|
|
101
101
|
)
|
|
102
102
|
|
|
103
103
|
|
|
104
|
-
#
|
|
104
|
+
# TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
|
|
105
105
|
class InvalidAddonTypeException(AddonException):
|
|
106
106
|
def __init__(self, addon_type):
|
|
107
107
|
self.addon_type = addon_type
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from slack_sdk import WebClient
|
|
2
|
+
from slack_sdk.errors import SlackApiError
|
|
3
|
+
from slack_sdk.models import blocks
|
|
4
|
+
|
|
5
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SlackChannelNotifierException(PlatformException):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SlackChannelNotifier:
|
|
13
|
+
def __init__(self, slack_token: str, slack_channel_id: str):
|
|
14
|
+
self.client = WebClient(slack_token)
|
|
15
|
+
self.slack_channel_id = slack_channel_id
|
|
16
|
+
|
|
17
|
+
def post_update(self, message_ref, message, context=None):
|
|
18
|
+
args = {
|
|
19
|
+
"channel": self.slack_channel_id,
|
|
20
|
+
"blocks": self._build_message_blocks(message, context),
|
|
21
|
+
"text": message,
|
|
22
|
+
"unfurl_links": False,
|
|
23
|
+
"unfurl_media": False,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
response = self.client.chat_update(ts=message_ref, **args)
|
|
28
|
+
return response["ts"]
|
|
29
|
+
except SlackApiError as e:
|
|
30
|
+
raise SlackChannelNotifierException(f"Slack notification unsuccessful: {e}")
|
|
31
|
+
|
|
32
|
+
def post_new(self, message, context=None, title=None, reply_broadcast=None, thread_ref=None):
|
|
33
|
+
args = {
|
|
34
|
+
"channel": self.slack_channel_id,
|
|
35
|
+
"blocks": self._build_message_blocks(message, context),
|
|
36
|
+
"text": title if title else message,
|
|
37
|
+
"reply_broadcast": reply_broadcast,
|
|
38
|
+
"unfurl_links": False,
|
|
39
|
+
"unfurl_media": False,
|
|
40
|
+
"thread_ts": thread_ref,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
response = self.client.chat_postMessage(ts=None, **args)
|
|
45
|
+
return response["ts"]
|
|
46
|
+
except SlackApiError as e:
|
|
47
|
+
raise SlackChannelNotifierException(f"Slack notification unsuccessful: {e}")
|
|
48
|
+
|
|
49
|
+
def _build_message_blocks(self, message, context):
|
|
50
|
+
message_blocks = [
|
|
51
|
+
blocks.SectionBlock(
|
|
52
|
+
text=blocks.TextObject(type="mrkdwn", text=message),
|
|
53
|
+
),
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
if context:
|
|
57
|
+
message_blocks.append(
|
|
58
|
+
blocks.ContextBlock(
|
|
59
|
+
elements=[blocks.TextObject(type="mrkdwn", text=element) for element in context]
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
return message_blocks
|
|
@@ -3,6 +3,7 @@ from datetime import datetime
|
|
|
3
3
|
from importlib.metadata import version
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
|
+
from dbt_platform_helper.constants import EXTENSIONS_MODULE_PATH
|
|
6
7
|
from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
|
|
7
8
|
from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
|
|
8
9
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
@@ -17,23 +18,98 @@ class TerraformManifestProvider:
|
|
|
17
18
|
self.file_provider = file_provider
|
|
18
19
|
self.io = io
|
|
19
20
|
|
|
21
|
+
def generate_service_config(
|
|
22
|
+
self,
|
|
23
|
+
config_object,
|
|
24
|
+
environment,
|
|
25
|
+
platform_helper_version: str,
|
|
26
|
+
platform_config,
|
|
27
|
+
module_source_override: str = None,
|
|
28
|
+
):
|
|
29
|
+
|
|
30
|
+
service_dir = f"terraform/services/{environment}/{config_object.name}"
|
|
31
|
+
platform_config = ConfigProvider.apply_environment_defaults(platform_config)
|
|
32
|
+
account = self._get_account_for_env(environment, platform_config)
|
|
33
|
+
deploy_to_account_id = self._get_account_id_for_account(account, platform_config)
|
|
34
|
+
application_name = platform_config["application"]
|
|
35
|
+
|
|
36
|
+
terraform = {}
|
|
37
|
+
self._add_header(terraform)
|
|
38
|
+
|
|
39
|
+
self._add_service_locals(terraform, environment)
|
|
40
|
+
|
|
41
|
+
self._add_provider(terraform, account, deploy_to_account_id)
|
|
42
|
+
self._add_backend(
|
|
43
|
+
terraform,
|
|
44
|
+
platform_config,
|
|
45
|
+
account,
|
|
46
|
+
f"tfstate/application/{application_name}/services/{environment}/{config_object.name}.tfstate",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
self._add_service_module(terraform, platform_helper_version, module_source_override)
|
|
50
|
+
|
|
51
|
+
self._write_terraform_json(terraform, service_dir)
|
|
52
|
+
|
|
53
|
+
def _add_service_locals(self, terraform, environment):
|
|
54
|
+
terraform["locals"] = {
|
|
55
|
+
"environment": environment,
|
|
56
|
+
"platform_config": '${yamldecode(file("../../../../platform-config.yml"))}',
|
|
57
|
+
"application": '${local.platform_config["application"]}',
|
|
58
|
+
"environments": '${local.platform_config["environments"]}',
|
|
59
|
+
"env_config": '${{for name, config in local.environments: name => merge(lookup(local.environments, "*", {}), config)}}',
|
|
60
|
+
"service_config": '${yamldecode(file("./service-config.yml"))}',
|
|
61
|
+
"raw_env_config": '${local.platform_config["environments"]}',
|
|
62
|
+
"combined_env_config": '${{for name, config in local.raw_env_config: name => merge(lookup(local.raw_env_config, "*", {}), config)}}',
|
|
63
|
+
"service_deployment_mode": '${lookup(local.combined_env_config[local.environment], "service-deployment-mode", "copilot")}',
|
|
64
|
+
"non_copilot_service_deployment_mode": '${local.service_deployment_mode == "dual-deploy-copilot-traffic" || local.service_deployment_mode == "dual-deploy-platform-traffic" || local.service_deployment_mode == "platform" ? 1 : 0}',
|
|
65
|
+
"custom_iam_policy_path": '${abspath(format("%s/../../../../services/%s/custom-iam-policy/%s.yml", path.module, local.service_config.name, local.environment))}',
|
|
66
|
+
"custom_iam_policy_json": "${fileexists(local.custom_iam_policy_path) ? jsonencode(yamldecode(file(local.custom_iam_policy_path))) : null}",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
def _add_service_module(
|
|
70
|
+
self, terraform: dict, platform_helper_version: str, module_source_override: str = None
|
|
71
|
+
):
|
|
72
|
+
source = (
|
|
73
|
+
module_source_override
|
|
74
|
+
or f"git::git@github.com:uktrade/platform-tools.git//terraform/ecs-service?depth=1&ref={platform_helper_version}"
|
|
75
|
+
)
|
|
76
|
+
terraform["module"] = {
|
|
77
|
+
"ecs-service": {
|
|
78
|
+
"source": source,
|
|
79
|
+
"count": "${local.non_copilot_service_deployment_mode}",
|
|
80
|
+
"application": "${local.application}",
|
|
81
|
+
"environment": "${local.environment}",
|
|
82
|
+
"service_config": "${local.service_config}",
|
|
83
|
+
"env_config": "${local.env_config}",
|
|
84
|
+
"platform_extensions": '${local.platform_config["extensions"]}',
|
|
85
|
+
"custom_iam_policy_json": "${local.custom_iam_policy_json}",
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
20
89
|
def generate_codebase_pipeline_config(
|
|
21
90
|
self,
|
|
22
91
|
platform_config: dict,
|
|
23
|
-
|
|
92
|
+
platform_helper_version: str,
|
|
24
93
|
ecr_imports: dict[str, str],
|
|
25
94
|
deploy_repository: str,
|
|
95
|
+
module_source: str,
|
|
26
96
|
):
|
|
27
97
|
default_account = self._get_account_for_env("*", platform_config)
|
|
98
|
+
deploy_to_account_id = self._get_account_id_for_account(default_account, platform_config)
|
|
28
99
|
state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
|
|
29
100
|
|
|
30
101
|
terraform = {}
|
|
31
102
|
self._add_header(terraform)
|
|
32
103
|
self._add_codebase_pipeline_locals(terraform)
|
|
33
|
-
self._add_provider(terraform, default_account)
|
|
34
|
-
self._add_backend(
|
|
104
|
+
self._add_provider(terraform, default_account, deploy_to_account_id)
|
|
105
|
+
self._add_backend(
|
|
106
|
+
terraform,
|
|
107
|
+
platform_config,
|
|
108
|
+
default_account,
|
|
109
|
+
f"tfstate/application/{state_key_suffix}.tfstate",
|
|
110
|
+
)
|
|
35
111
|
self._add_codebase_pipeline_module(
|
|
36
|
-
terraform,
|
|
112
|
+
terraform, platform_helper_version, deploy_repository, module_source
|
|
37
113
|
)
|
|
38
114
|
self._add_imports(terraform, ecr_imports)
|
|
39
115
|
self._write_terraform_json(terraform, "terraform/codebase-pipelines")
|
|
@@ -42,7 +118,8 @@ class TerraformManifestProvider:
|
|
|
42
118
|
self,
|
|
43
119
|
platform_config: dict,
|
|
44
120
|
env: str,
|
|
45
|
-
|
|
121
|
+
platform_helper_version: str,
|
|
122
|
+
module_source_override: str = None,
|
|
46
123
|
):
|
|
47
124
|
platform_config = ConfigProvider.apply_environment_defaults(platform_config)
|
|
48
125
|
account = self._get_account_for_env(env, platform_config)
|
|
@@ -54,8 +131,10 @@ class TerraformManifestProvider:
|
|
|
54
131
|
terraform = {}
|
|
55
132
|
self._add_header(terraform)
|
|
56
133
|
self._add_environment_locals(terraform, application_name)
|
|
57
|
-
self._add_backend(
|
|
58
|
-
|
|
134
|
+
self._add_backend(
|
|
135
|
+
terraform, platform_config, account, f"tfstate/application/{state_key_suffix}.tfstate"
|
|
136
|
+
)
|
|
137
|
+
self._add_extensions_module(terraform, platform_helper_version, env, module_source_override)
|
|
59
138
|
self._add_moved(terraform, platform_config)
|
|
60
139
|
self._ensure_no_hcl_manifest_file(env_dir)
|
|
61
140
|
self._write_terraform_json(terraform, env_dir)
|
|
@@ -71,6 +150,16 @@ class TerraformManifestProvider:
|
|
|
71
150
|
)
|
|
72
151
|
return account
|
|
73
152
|
|
|
153
|
+
@staticmethod
|
|
154
|
+
def _get_account_id_for_account(account_name, platform_config):
|
|
155
|
+
environment_config = platform_config["environments"]
|
|
156
|
+
account_id_lookup = {
|
|
157
|
+
env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
|
|
158
|
+
for env in environment_config.values()
|
|
159
|
+
if env is not None and "accounts" in env and "deploy" in env["accounts"]
|
|
160
|
+
}
|
|
161
|
+
return account_id_lookup.get(account_name)
|
|
162
|
+
|
|
74
163
|
@staticmethod
|
|
75
164
|
def _add_header(terraform: dict):
|
|
76
165
|
time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
@@ -89,21 +178,20 @@ class TerraformManifestProvider:
|
|
|
89
178
|
}
|
|
90
179
|
|
|
91
180
|
@staticmethod
|
|
92
|
-
def _add_provider(terraform: dict,
|
|
181
|
+
def _add_provider(terraform: dict, deploy_to_account: str, deploy_to_account_id: str):
|
|
93
182
|
terraform["provider"] = {"aws": {}}
|
|
94
183
|
terraform["provider"]["aws"]["region"] = "eu-west-2"
|
|
95
|
-
terraform["provider"]["aws"]["profile"] =
|
|
96
|
-
terraform["provider"]["aws"]["
|
|
97
|
-
terraform["provider"]["aws"]["shared_credentials_files"] = ["~/.aws/config"]
|
|
184
|
+
terraform["provider"]["aws"]["profile"] = deploy_to_account
|
|
185
|
+
terraform["provider"]["aws"]["allowed_account_ids"] = [deploy_to_account_id]
|
|
98
186
|
|
|
99
187
|
@staticmethod
|
|
100
|
-
def _add_backend(terraform: dict, platform_config: dict, account: str,
|
|
188
|
+
def _add_backend(terraform: dict, platform_config: dict, account: str, state_key: str):
|
|
101
189
|
terraform["terraform"] = {
|
|
102
190
|
"required_version": SUPPORTED_TERRAFORM_VERSION,
|
|
103
191
|
"backend": {
|
|
104
192
|
"s3": {
|
|
105
193
|
"bucket": f"terraform-platform-state-{account}",
|
|
106
|
-
"key":
|
|
194
|
+
"key": state_key,
|
|
107
195
|
"region": "eu-west-2",
|
|
108
196
|
"encrypt": True,
|
|
109
197
|
"kms_key_id": f"alias/terraform-platform-state-s3-key-{account}",
|
|
@@ -117,9 +205,12 @@ class TerraformManifestProvider:
|
|
|
117
205
|
|
|
118
206
|
@staticmethod
|
|
119
207
|
def _add_codebase_pipeline_module(
|
|
120
|
-
terraform: dict,
|
|
208
|
+
terraform: dict,
|
|
209
|
+
platform_helper_version: str,
|
|
210
|
+
deploy_repository: str,
|
|
211
|
+
module_source: str,
|
|
121
212
|
):
|
|
122
|
-
source =
|
|
213
|
+
source = module_source
|
|
123
214
|
terraform["module"] = {
|
|
124
215
|
"codebase-pipelines": {
|
|
125
216
|
"source": source,
|
|
@@ -128,20 +219,30 @@ class TerraformManifestProvider:
|
|
|
128
219
|
"codebase": "${each.key}",
|
|
129
220
|
"repository": "${each.value.repository}",
|
|
130
221
|
"deploy_repository": f"{deploy_repository}",
|
|
222
|
+
"deploy_repository_branch": '${lookup(each.value, "deploy_repository_branch", "main")}',
|
|
131
223
|
"additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
|
|
224
|
+
"cache_invalidation": '${lookup(each.value, "cache_invalidation", null)}',
|
|
132
225
|
"pipelines": '${lookup(each.value, "pipelines", [])}',
|
|
133
226
|
"services": "${each.value.services}",
|
|
134
227
|
"requires_image_build": '${lookup(each.value, "requires_image_build", true)}',
|
|
135
228
|
"slack_channel": '${lookup(each.value, "slack_channel", "/codebuild/slack_oauth_channel")}',
|
|
136
229
|
"env_config": "${local.environments}",
|
|
230
|
+
"platform_tools_version": f"{platform_helper_version}",
|
|
137
231
|
}
|
|
138
232
|
}
|
|
139
233
|
|
|
140
234
|
@staticmethod
|
|
141
|
-
def _add_extensions_module(
|
|
142
|
-
|
|
235
|
+
def _add_extensions_module(
|
|
236
|
+
terraform: dict, platform_helper_version: str, env: str, module_source_override: str = None
|
|
237
|
+
):
|
|
238
|
+
source = module_source_override or f"{EXTENSIONS_MODULE_PATH}{platform_helper_version}"
|
|
143
239
|
terraform["module"] = {
|
|
144
|
-
"extensions": {
|
|
240
|
+
"extensions": {
|
|
241
|
+
"source": source,
|
|
242
|
+
"args": "${local.args}",
|
|
243
|
+
"environment": env,
|
|
244
|
+
"repos": "${local.codebase_pipeline_repos != null ? (distinct(values(local.codebase_pipeline_repos))) : null}",
|
|
245
|
+
}
|
|
145
246
|
}
|
|
146
247
|
|
|
147
248
|
@staticmethod
|
|
@@ -164,6 +265,7 @@ class TerraformManifestProvider:
|
|
|
164
265
|
"services": '${local.config["extensions"]}',
|
|
165
266
|
"env_config": "${local.env_config}",
|
|
166
267
|
},
|
|
268
|
+
"codebase_pipeline_repos": '${try({for k, v in local.config["codebase_pipelines"]: k => v.repository}, null)}',
|
|
167
269
|
}
|
|
168
270
|
|
|
169
271
|
@staticmethod
|
|
@@ -204,7 +306,7 @@ class TerraformManifestProvider:
|
|
|
204
306
|
|
|
205
307
|
def _write_terraform_json(self, terraform: dict, env_dir: str):
|
|
206
308
|
message = self.file_provider.mkfile(
|
|
207
|
-
str(Path(env_dir)
|
|
309
|
+
str(Path(env_dir)),
|
|
208
310
|
"main.tf.json",
|
|
209
311
|
json.dumps(terraform, indent=2),
|
|
210
312
|
True,
|
|
@@ -1,36 +1,119 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import subprocess
|
|
1
3
|
from abc import ABC
|
|
4
|
+
from abc import abstractmethod
|
|
5
|
+
from importlib.metadata import PackageNotFoundError
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
from typing import Union
|
|
2
8
|
|
|
3
|
-
import
|
|
9
|
+
from requests import Session
|
|
10
|
+
from requests.adapters import HTTPAdapter
|
|
11
|
+
from urllib3.util import Retry
|
|
4
12
|
|
|
5
|
-
from dbt_platform_helper.
|
|
13
|
+
from dbt_platform_helper.entities.semantic_version import SemanticVersion
|
|
14
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
15
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
6
16
|
|
|
7
17
|
|
|
8
|
-
|
|
18
|
+
def set_up_retry():
|
|
19
|
+
session = Session()
|
|
20
|
+
retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[403, 500, 502, 503, 504])
|
|
21
|
+
session.mount("https://", HTTPAdapter(max_retries=retries))
|
|
22
|
+
return session
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class InstalledVersionProviderException(PlatformException):
|
|
9
26
|
pass
|
|
10
27
|
|
|
11
28
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
29
|
+
class InstalledToolNotFoundException(InstalledVersionProviderException):
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
tool_name: str,
|
|
33
|
+
):
|
|
34
|
+
super().__init__(f"Package '{tool_name}' not found.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class VersionProvider(ABC):
|
|
38
|
+
@abstractmethod
|
|
39
|
+
def get_semantic_version() -> SemanticVersion:
|
|
40
|
+
raise NotImplementedError("Must be implemented in subclasses")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class InstalledVersionProvider:
|
|
44
|
+
@staticmethod
|
|
45
|
+
def get_semantic_version(tool_name: str) -> SemanticVersion:
|
|
46
|
+
try:
|
|
47
|
+
return SemanticVersion.from_string(version(tool_name))
|
|
48
|
+
|
|
49
|
+
except PackageNotFoundError:
|
|
50
|
+
raise InstalledToolNotFoundException(tool_name)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class GithubLatestVersionProvider(VersionProvider):
|
|
15
54
|
@staticmethod
|
|
16
|
-
def
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
55
|
+
def get_semantic_version(
|
|
56
|
+
repo_name: str, tags: bool = False, request_session=set_up_retry(), io=ClickIOProvider()
|
|
57
|
+
) -> Union[SemanticVersion, None]:
|
|
58
|
+
|
|
59
|
+
semantic_version = None
|
|
60
|
+
try:
|
|
61
|
+
if tags:
|
|
62
|
+
response = request_session.get(f"https://api.github.com/repos/{repo_name}/tags")
|
|
63
|
+
|
|
64
|
+
versions = [SemanticVersion.from_string(v["name"]) for v in response.json()]
|
|
65
|
+
versions.sort(reverse=True)
|
|
66
|
+
semantic_version = versions[0]
|
|
67
|
+
else:
|
|
68
|
+
package_info = request_session.get(
|
|
69
|
+
f"https://api.github.com/repos/{repo_name}/releases/latest"
|
|
70
|
+
).json()
|
|
71
|
+
semantic_version = SemanticVersion.from_string(package_info["tag_name"])
|
|
72
|
+
except Exception as e:
|
|
73
|
+
io.error(f"Exception occured when calling Github with:\n{str(e)}")
|
|
22
74
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
75
|
+
return semantic_version
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class PyPiLatestVersionProvider(VersionProvider):
|
|
79
|
+
@staticmethod
|
|
80
|
+
def get_semantic_version(
|
|
81
|
+
project_name: str, request_session=set_up_retry(), io=ClickIOProvider()
|
|
82
|
+
) -> Union[SemanticVersion, None]:
|
|
83
|
+
semantic_version = None
|
|
84
|
+
try:
|
|
85
|
+
package_info = request_session.get(f"https://pypi.org/pypi/{project_name}/json").json()
|
|
86
|
+
released_versions = package_info["releases"].keys()
|
|
87
|
+
parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
|
|
88
|
+
parsed_released_versions.sort(reverse=True)
|
|
89
|
+
semantic_version = parsed_released_versions[0]
|
|
90
|
+
except Exception as e:
|
|
91
|
+
io.error(f"Exception occured when calling PyPi with:\n{str(e)}")
|
|
92
|
+
return semantic_version
|
|
27
93
|
|
|
28
94
|
|
|
29
|
-
class
|
|
95
|
+
class AWSCLIInstalledVersionProvider(VersionProvider):
|
|
30
96
|
@staticmethod
|
|
31
|
-
def
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
97
|
+
def get_semantic_version() -> Union[SemanticVersion, None]:
|
|
98
|
+
installed_aws_version = None
|
|
99
|
+
try:
|
|
100
|
+
response = subprocess.run("aws --version", capture_output=True, shell=True)
|
|
101
|
+
matched = re.match(r"aws-cli/([0-9.]+)", response.stdout.decode("utf8"))
|
|
102
|
+
installed_aws_version = matched.group(1)
|
|
103
|
+
except (ValueError, AttributeError):
|
|
104
|
+
pass
|
|
105
|
+
return SemanticVersion.from_string(installed_aws_version)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class CopilotInstalledVersionProvider(VersionProvider):
|
|
109
|
+
@staticmethod
|
|
110
|
+
def get_semantic_version() -> Union[SemanticVersion, None]:
|
|
111
|
+
copilot_version = None
|
|
112
|
+
|
|
113
|
+
try:
|
|
114
|
+
response = subprocess.run("copilot --version", capture_output=True, shell=True)
|
|
115
|
+
[copilot_version] = re.findall(r"[0-9.]+", response.stdout.decode("utf8"))
|
|
116
|
+
except ValueError:
|
|
117
|
+
pass
|
|
118
|
+
|
|
119
|
+
return SemanticVersion.from_string(copilot_version)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from dbt_platform_helper.entities.semantic_version import SemanticVersion
|
|
4
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class UnsupportedVersionException(PlatformException):
|
|
8
|
+
def __init__(self, version: str):
|
|
9
|
+
super().__init__(
|
|
10
|
+
f"""Platform-helper version {version} is not compatible with platform-helper. Please install version platform-helper version 14 or later."""
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class VersionStatus:
|
|
16
|
+
installed: SemanticVersion = None
|
|
17
|
+
latest: SemanticVersion = None
|
|
18
|
+
|
|
19
|
+
def __str__(self):
|
|
20
|
+
attrs = {
|
|
21
|
+
key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
|
|
22
|
+
}
|
|
23
|
+
attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
|
|
24
|
+
return f"{self.__class__.__name__}: {attrs_str}"
|
|
25
|
+
|
|
26
|
+
def is_outdated(self):
|
|
27
|
+
return self.installed != self.latest
|