dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. dbt_platform_helper/COMMANDS.md +107 -27
  2. dbt_platform_helper/commands/application.py +5 -6
  3. dbt_platform_helper/commands/codebase.py +31 -10
  4. dbt_platform_helper/commands/conduit.py +3 -5
  5. dbt_platform_helper/commands/config.py +20 -311
  6. dbt_platform_helper/commands/copilot.py +18 -391
  7. dbt_platform_helper/commands/database.py +17 -9
  8. dbt_platform_helper/commands/environment.py +20 -14
  9. dbt_platform_helper/commands/generate.py +0 -3
  10. dbt_platform_helper/commands/internal.py +140 -0
  11. dbt_platform_helper/commands/notify.py +58 -78
  12. dbt_platform_helper/commands/pipeline.py +23 -19
  13. dbt_platform_helper/commands/secrets.py +39 -93
  14. dbt_platform_helper/commands/version.py +7 -12
  15. dbt_platform_helper/constants.py +52 -7
  16. dbt_platform_helper/domain/codebase.py +89 -39
  17. dbt_platform_helper/domain/conduit.py +335 -76
  18. dbt_platform_helper/domain/config.py +381 -0
  19. dbt_platform_helper/domain/copilot.py +398 -0
  20. dbt_platform_helper/domain/copilot_environment.py +8 -8
  21. dbt_platform_helper/domain/database_copy.py +2 -2
  22. dbt_platform_helper/domain/maintenance_page.py +254 -430
  23. dbt_platform_helper/domain/notify.py +64 -0
  24. dbt_platform_helper/domain/pipelines.py +43 -35
  25. dbt_platform_helper/domain/plans.py +41 -0
  26. dbt_platform_helper/domain/secrets.py +279 -0
  27. dbt_platform_helper/domain/service.py +570 -0
  28. dbt_platform_helper/domain/terraform_environment.py +14 -13
  29. dbt_platform_helper/domain/update_alb_rules.py +412 -0
  30. dbt_platform_helper/domain/versioning.py +249 -0
  31. dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
  32. dbt_platform_helper/entities/semantic_version.py +83 -0
  33. dbt_platform_helper/entities/service.py +339 -0
  34. dbt_platform_helper/platform_exception.py +4 -0
  35. dbt_platform_helper/providers/autoscaling.py +24 -0
  36. dbt_platform_helper/providers/aws/__init__.py +0 -0
  37. dbt_platform_helper/providers/aws/exceptions.py +70 -0
  38. dbt_platform_helper/providers/aws/interfaces.py +13 -0
  39. dbt_platform_helper/providers/aws/opensearch.py +23 -0
  40. dbt_platform_helper/providers/aws/redis.py +21 -0
  41. dbt_platform_helper/providers/aws/sso_auth.py +75 -0
  42. dbt_platform_helper/providers/cache.py +40 -4
  43. dbt_platform_helper/providers/cloudformation.py +1 -1
  44. dbt_platform_helper/providers/config.py +137 -19
  45. dbt_platform_helper/providers/config_validator.py +112 -51
  46. dbt_platform_helper/providers/copilot.py +24 -16
  47. dbt_platform_helper/providers/ecr.py +89 -7
  48. dbt_platform_helper/providers/ecs.py +228 -36
  49. dbt_platform_helper/providers/environment_variable.py +24 -0
  50. dbt_platform_helper/providers/files.py +1 -1
  51. dbt_platform_helper/providers/io.py +36 -4
  52. dbt_platform_helper/providers/kms.py +22 -0
  53. dbt_platform_helper/providers/load_balancers.py +402 -42
  54. dbt_platform_helper/providers/logs.py +72 -0
  55. dbt_platform_helper/providers/parameter_store.py +134 -0
  56. dbt_platform_helper/providers/s3.py +21 -0
  57. dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
  58. dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
  59. dbt_platform_helper/providers/schema_migrator.py +77 -0
  60. dbt_platform_helper/providers/secrets.py +5 -5
  61. dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
  62. dbt_platform_helper/providers/terraform_manifest.py +121 -19
  63. dbt_platform_helper/providers/version.py +106 -23
  64. dbt_platform_helper/providers/version_status.py +27 -0
  65. dbt_platform_helper/providers/vpc.py +36 -5
  66. dbt_platform_helper/providers/yaml_file.py +58 -2
  67. dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
  68. dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
  69. dbt_platform_helper/utilities/decorators.py +103 -0
  70. dbt_platform_helper/utils/application.py +119 -22
  71. dbt_platform_helper/utils/aws.py +39 -150
  72. dbt_platform_helper/utils/deep_merge.py +10 -0
  73. dbt_platform_helper/utils/git.py +1 -14
  74. dbt_platform_helper/utils/validation.py +1 -1
  75. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
  76. dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
  77. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
  78. platform_helper.py +3 -1
  79. terraform/elasticache-redis/plans.yml +85 -0
  80. terraform/opensearch/plans.yml +71 -0
  81. terraform/postgres/plans.yml +128 -0
  82. dbt_platform_helper/addon-plans.yml +0 -224
  83. dbt_platform_helper/providers/aws.py +0 -37
  84. dbt_platform_helper/providers/opensearch.py +0 -36
  85. dbt_platform_helper/providers/redis.py +0 -34
  86. dbt_platform_helper/providers/semantic_version.py +0 -126
  87. dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
  88. dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
  89. dbt_platform_helper/utils/cloudfoundry.py +0 -14
  90. dbt_platform_helper/utils/files.py +0 -53
  91. dbt_platform_helper/utils/manifests.py +0 -18
  92. dbt_platform_helper/utils/versioning.py +0 -238
  93. dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
  94. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
  95. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,21 @@
1
+ import boto3
2
+ from botocore.exceptions import ClientError
3
+
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
+
6
+
7
+ class S3Provider:
8
+
9
+ def __init__(self, client: boto3.client):
10
+ self.client = client
11
+
12
+ def get_object(self, bucket_name: str, object_key: str) -> str:
13
+ """Returns an object from an S3 bucket."""
14
+
15
+ try:
16
+ content = self.client.get_object(Bucket=bucket_name, Key=object_key)
17
+ return content["Body"].read().decode("utf-8")
18
+ except ClientError as e:
19
+ raise PlatformException(
20
+ f"Failed to get '{object_key}' from '{bucket_name}'. Error: {e}"
21
+ )
@@ -0,0 +1,43 @@
1
+ from copy import deepcopy
2
+
3
+
4
+ class SchemaV0ToV1Migration:
5
+ def from_version(self) -> int:
6
+ return 0
7
+
8
+ def migrate(self, platform_config: dict) -> dict:
9
+ migrated_config = deepcopy(platform_config)
10
+
11
+ self._remove_terraform_platform_modules_default_version(migrated_config)
12
+ self._remove_versions_from_env_config(migrated_config)
13
+ self._remove_to_account_and_from_account_from_database_copy(migrated_config)
14
+ self._remove_pipeline_platform_helper_override(migrated_config)
15
+
16
+ return migrated_config
17
+
18
+ def _remove_versions_from_env_config(self, migrated_config: dict) -> None:
19
+ for env_name, env in migrated_config.get("environments", {}).items():
20
+ if env and "versions" in env:
21
+ del env["versions"]
22
+
23
+ def _remove_terraform_platform_modules_default_version(self, migrated_config: dict) -> None:
24
+ if "default_versions" in migrated_config:
25
+ default_versions = migrated_config["default_versions"]
26
+ if "terraform-platform-modules" in default_versions:
27
+ del default_versions["terraform-platform-modules"]
28
+
29
+ def _remove_to_account_and_from_account_from_database_copy(self, migrated_config: dict) -> None:
30
+ for extension_name, extension in migrated_config.get("extensions", {}).items():
31
+ if extension.get("type") == "postgres" and "database_copy" in extension:
32
+ for database_copy_block in extension["database_copy"]:
33
+ if "from_account" in database_copy_block:
34
+ del database_copy_block["from_account"]
35
+ if "to_account" in database_copy_block:
36
+ del database_copy_block["to_account"]
37
+
38
+ def _remove_pipeline_platform_helper_override(self, migrated_config: dict) -> None:
39
+ for pipeline_name, pipeline_config in migrated_config.get(
40
+ "environment_pipelines", {}
41
+ ).items():
42
+ if "versions" in pipeline_config:
43
+ del pipeline_config["versions"]
@@ -0,0 +1,77 @@
1
+ from collections import Counter
2
+ from collections import OrderedDict
3
+ from copy import deepcopy
4
+ from typing import Protocol
5
+
6
+ from dbt_platform_helper.platform_exception import PlatformException
7
+ from dbt_platform_helper.providers.io import ClickIOProvider
8
+ from dbt_platform_helper.providers.schema_migrations.schema_v0_to_v1_migration import (
9
+ SchemaV0ToV1Migration,
10
+ )
11
+ from dbt_platform_helper.providers.version import InstalledVersionProvider
12
+
13
+
14
+ class InvalidMigrationConfigurationException(PlatformException):
15
+ pass
16
+
17
+
18
+ class SchemaMigrationProtocol(Protocol):
19
+ def from_version(self) -> int: ...
20
+
21
+ def migrate(self, platform_config: dict) -> dict: ...
22
+
23
+
24
+ # TODO: Possibly get this programmatically?
25
+ ALL_MIGRATIONS = [SchemaV0ToV1Migration()]
26
+
27
+
28
+ class Migrator:
29
+ def __init__(
30
+ self,
31
+ migrations: list[SchemaMigrationProtocol],
32
+ installed_version_provider: InstalledVersionProvider = InstalledVersionProvider,
33
+ io_provider: ClickIOProvider = ClickIOProvider(),
34
+ ):
35
+ self.migrations = sorted(migrations, key=lambda m: m.from_version())
36
+ self.installed_version_provider = installed_version_provider
37
+ self.io_provider = io_provider
38
+ from_version_counts = Counter([migration.from_version() for migration in self.migrations])
39
+ duplicate_from_versions = [count for count in from_version_counts.values() if count > 1]
40
+
41
+ if duplicate_from_versions:
42
+ raise InvalidMigrationConfigurationException(
43
+ "`from_version` parameters must be unique amongst migrations"
44
+ )
45
+
46
+ def migrate(self, platform_config: dict) -> dict:
47
+ out = OrderedDict(deepcopy(platform_config))
48
+ if "schema_version" not in out:
49
+ out["schema_version"] = 0
50
+
51
+ if "default_versions" in out:
52
+ out.move_to_end("default_versions", last=False)
53
+ if "schema_version" in out:
54
+ out.move_to_end("schema_version", last=False)
55
+ if "application" in out:
56
+ out.move_to_end("application", last=False)
57
+
58
+ for migration in self.migrations:
59
+ migration_can_be_applied = migration.from_version() == out["schema_version"]
60
+ if migration_can_be_applied:
61
+ out = migration.migrate(out)
62
+ schema_version = out["schema_version"]
63
+ self.io_provider.info(
64
+ f"Migrating from platform config schema version {schema_version} to version {schema_version + 1}"
65
+ )
66
+ out["schema_version"] += 1
67
+
68
+ if "default_versions" not in out:
69
+ out["default_versions"] = {}
70
+
71
+ out["default_versions"]["platform-helper"] = str(
72
+ self.installed_version_provider.get_semantic_version("dbt-platform-helper")
73
+ )
74
+
75
+ self.io_provider.info("\nMigration complete")
76
+
77
+ return dict(out)
@@ -41,7 +41,7 @@ class Secrets:
41
41
 
42
42
  raise SecretNotFoundException(secret_name)
43
43
 
44
- # Todo: This probably does not belong in the secrets provider. When it moves, take the Todoed exceptions from below
44
+ # TODO: DBTP-1946: This probably does not belong in the secrets provider. When it moves, take the Todoed exceptions from below
45
45
  def get_addon_type(self, addon_name: str) -> str:
46
46
  addon_type = None
47
47
  try:
@@ -82,18 +82,18 @@ class Secrets:
82
82
  return addon_name.replace("-", "_").upper()
83
83
 
84
84
 
85
- # Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
85
+ # TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
86
86
  class AddonException(PlatformException):
87
87
  pass
88
88
 
89
89
 
90
- # Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
90
+ # TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
91
91
  class AddonNotFoundException(AddonException):
92
92
  def __init__(self, addon_name: str):
93
93
  super().__init__(f"""Addon "{addon_name}" does not exist.""")
94
94
 
95
95
 
96
- # Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
96
+ # TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
97
97
  class AddonTypeMissingFromConfigException(AddonException):
98
98
  def __init__(self, addon_name: str):
99
99
  super().__init__(
@@ -101,7 +101,7 @@ class AddonTypeMissingFromConfigException(AddonException):
101
101
  )
102
102
 
103
103
 
104
- # Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
104
+ # TODO: DBTP-1946: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
105
105
  class InvalidAddonTypeException(AddonException):
106
106
  def __init__(self, addon_type):
107
107
  self.addon_type = addon_type
@@ -0,0 +1,62 @@
1
+ from slack_sdk import WebClient
2
+ from slack_sdk.errors import SlackApiError
3
+ from slack_sdk.models import blocks
4
+
5
+ from dbt_platform_helper.platform_exception import PlatformException
6
+
7
+
8
+ class SlackChannelNotifierException(PlatformException):
9
+ pass
10
+
11
+
12
+ class SlackChannelNotifier:
13
+ def __init__(self, slack_token: str, slack_channel_id: str):
14
+ self.client = WebClient(slack_token)
15
+ self.slack_channel_id = slack_channel_id
16
+
17
+ def post_update(self, message_ref, message, context=None):
18
+ args = {
19
+ "channel": self.slack_channel_id,
20
+ "blocks": self._build_message_blocks(message, context),
21
+ "text": message,
22
+ "unfurl_links": False,
23
+ "unfurl_media": False,
24
+ }
25
+
26
+ try:
27
+ response = self.client.chat_update(ts=message_ref, **args)
28
+ return response["ts"]
29
+ except SlackApiError as e:
30
+ raise SlackChannelNotifierException(f"Slack notification unsuccessful: {e}")
31
+
32
+ def post_new(self, message, context=None, title=None, reply_broadcast=None, thread_ref=None):
33
+ args = {
34
+ "channel": self.slack_channel_id,
35
+ "blocks": self._build_message_blocks(message, context),
36
+ "text": title if title else message,
37
+ "reply_broadcast": reply_broadcast,
38
+ "unfurl_links": False,
39
+ "unfurl_media": False,
40
+ "thread_ts": thread_ref,
41
+ }
42
+
43
+ try:
44
+ response = self.client.chat_postMessage(ts=None, **args)
45
+ return response["ts"]
46
+ except SlackApiError as e:
47
+ raise SlackChannelNotifierException(f"Slack notification unsuccessful: {e}")
48
+
49
+ def _build_message_blocks(self, message, context):
50
+ message_blocks = [
51
+ blocks.SectionBlock(
52
+ text=blocks.TextObject(type="mrkdwn", text=message),
53
+ ),
54
+ ]
55
+
56
+ if context:
57
+ message_blocks.append(
58
+ blocks.ContextBlock(
59
+ elements=[blocks.TextObject(type="mrkdwn", text=element) for element in context]
60
+ )
61
+ )
62
+ return message_blocks
@@ -3,6 +3,7 @@ from datetime import datetime
3
3
  from importlib.metadata import version
4
4
  from pathlib import Path
5
5
 
6
+ from dbt_platform_helper.constants import EXTENSIONS_MODULE_PATH
6
7
  from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
7
8
  from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
8
9
  from dbt_platform_helper.providers.config import ConfigProvider
@@ -17,23 +18,98 @@ class TerraformManifestProvider:
17
18
  self.file_provider = file_provider
18
19
  self.io = io
19
20
 
21
+ def generate_service_config(
22
+ self,
23
+ config_object,
24
+ environment,
25
+ platform_helper_version: str,
26
+ platform_config,
27
+ module_source_override: str = None,
28
+ ):
29
+
30
+ service_dir = f"terraform/services/{environment}/{config_object.name}"
31
+ platform_config = ConfigProvider.apply_environment_defaults(platform_config)
32
+ account = self._get_account_for_env(environment, platform_config)
33
+ deploy_to_account_id = self._get_account_id_for_account(account, platform_config)
34
+ application_name = platform_config["application"]
35
+
36
+ terraform = {}
37
+ self._add_header(terraform)
38
+
39
+ self._add_service_locals(terraform, environment)
40
+
41
+ self._add_provider(terraform, account, deploy_to_account_id)
42
+ self._add_backend(
43
+ terraform,
44
+ platform_config,
45
+ account,
46
+ f"tfstate/application/{application_name}/services/{environment}/{config_object.name}.tfstate",
47
+ )
48
+
49
+ self._add_service_module(terraform, platform_helper_version, module_source_override)
50
+
51
+ self._write_terraform_json(terraform, service_dir)
52
+
53
+ def _add_service_locals(self, terraform, environment):
54
+ terraform["locals"] = {
55
+ "environment": environment,
56
+ "platform_config": '${yamldecode(file("../../../../platform-config.yml"))}',
57
+ "application": '${local.platform_config["application"]}',
58
+ "environments": '${local.platform_config["environments"]}',
59
+ "env_config": '${{for name, config in local.environments: name => merge(lookup(local.environments, "*", {}), config)}}',
60
+ "service_config": '${yamldecode(file("./service-config.yml"))}',
61
+ "raw_env_config": '${local.platform_config["environments"]}',
62
+ "combined_env_config": '${{for name, config in local.raw_env_config: name => merge(lookup(local.raw_env_config, "*", {}), config)}}',
63
+ "service_deployment_mode": '${lookup(local.combined_env_config[local.environment], "service-deployment-mode", "copilot")}',
64
+ "non_copilot_service_deployment_mode": '${local.service_deployment_mode == "dual-deploy-copilot-traffic" || local.service_deployment_mode == "dual-deploy-platform-traffic" || local.service_deployment_mode == "platform" ? 1 : 0}',
65
+ "custom_iam_policy_path": '${abspath(format("%s/../../../../services/%s/custom-iam-policy/%s.yml", path.module, local.service_config.name, local.environment))}',
66
+ "custom_iam_policy_json": "${fileexists(local.custom_iam_policy_path) ? jsonencode(yamldecode(file(local.custom_iam_policy_path))) : null}",
67
+ }
68
+
69
+ def _add_service_module(
70
+ self, terraform: dict, platform_helper_version: str, module_source_override: str = None
71
+ ):
72
+ source = (
73
+ module_source_override
74
+ or f"git::git@github.com:uktrade/platform-tools.git//terraform/ecs-service?depth=1&ref={platform_helper_version}"
75
+ )
76
+ terraform["module"] = {
77
+ "ecs-service": {
78
+ "source": source,
79
+ "count": "${local.non_copilot_service_deployment_mode}",
80
+ "application": "${local.application}",
81
+ "environment": "${local.environment}",
82
+ "service_config": "${local.service_config}",
83
+ "env_config": "${local.env_config}",
84
+ "platform_extensions": '${local.platform_config["extensions"]}',
85
+ "custom_iam_policy_json": "${local.custom_iam_policy_json}",
86
+ }
87
+ }
88
+
20
89
  def generate_codebase_pipeline_config(
21
90
  self,
22
91
  platform_config: dict,
23
- terraform_platform_modules_version: str,
92
+ platform_helper_version: str,
24
93
  ecr_imports: dict[str, str],
25
94
  deploy_repository: str,
95
+ module_source: str,
26
96
  ):
27
97
  default_account = self._get_account_for_env("*", platform_config)
98
+ deploy_to_account_id = self._get_account_id_for_account(default_account, platform_config)
28
99
  state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
29
100
 
30
101
  terraform = {}
31
102
  self._add_header(terraform)
32
103
  self._add_codebase_pipeline_locals(terraform)
33
- self._add_provider(terraform, default_account)
34
- self._add_backend(terraform, platform_config, default_account, state_key_suffix)
104
+ self._add_provider(terraform, default_account, deploy_to_account_id)
105
+ self._add_backend(
106
+ terraform,
107
+ platform_config,
108
+ default_account,
109
+ f"tfstate/application/{state_key_suffix}.tfstate",
110
+ )
35
111
  self._add_codebase_pipeline_module(
36
- terraform, terraform_platform_modules_version, deploy_repository
112
+ terraform, platform_helper_version, deploy_repository, module_source
37
113
  )
38
114
  self._add_imports(terraform, ecr_imports)
39
115
  self._write_terraform_json(terraform, "terraform/codebase-pipelines")
@@ -42,7 +118,8 @@ class TerraformManifestProvider:
42
118
  self,
43
119
  platform_config: dict,
44
120
  env: str,
45
- terraform_platform_modules_version: str,
121
+ platform_helper_version: str,
122
+ module_source_override: str = None,
46
123
  ):
47
124
  platform_config = ConfigProvider.apply_environment_defaults(platform_config)
48
125
  account = self._get_account_for_env(env, platform_config)
@@ -54,8 +131,10 @@ class TerraformManifestProvider:
54
131
  terraform = {}
55
132
  self._add_header(terraform)
56
133
  self._add_environment_locals(terraform, application_name)
57
- self._add_backend(terraform, platform_config, account, state_key_suffix)
58
- self._add_extensions_module(terraform, terraform_platform_modules_version, env)
134
+ self._add_backend(
135
+ terraform, platform_config, account, f"tfstate/application/{state_key_suffix}.tfstate"
136
+ )
137
+ self._add_extensions_module(terraform, platform_helper_version, env, module_source_override)
59
138
  self._add_moved(terraform, platform_config)
60
139
  self._ensure_no_hcl_manifest_file(env_dir)
61
140
  self._write_terraform_json(terraform, env_dir)
@@ -71,6 +150,16 @@ class TerraformManifestProvider:
71
150
  )
72
151
  return account
73
152
 
153
+ @staticmethod
154
+ def _get_account_id_for_account(account_name, platform_config):
155
+ environment_config = platform_config["environments"]
156
+ account_id_lookup = {
157
+ env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
158
+ for env in environment_config.values()
159
+ if env is not None and "accounts" in env and "deploy" in env["accounts"]
160
+ }
161
+ return account_id_lookup.get(account_name)
162
+
74
163
  @staticmethod
75
164
  def _add_header(terraform: dict):
76
165
  time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -89,21 +178,20 @@ class TerraformManifestProvider:
89
178
  }
90
179
 
91
180
  @staticmethod
92
- def _add_provider(terraform: dict, default_account: str):
181
+ def _add_provider(terraform: dict, deploy_to_account: str, deploy_to_account_id: str):
93
182
  terraform["provider"] = {"aws": {}}
94
183
  terraform["provider"]["aws"]["region"] = "eu-west-2"
95
- terraform["provider"]["aws"]["profile"] = default_account
96
- terraform["provider"]["aws"]["alias"] = default_account
97
- terraform["provider"]["aws"]["shared_credentials_files"] = ["~/.aws/config"]
184
+ terraform["provider"]["aws"]["profile"] = deploy_to_account
185
+ terraform["provider"]["aws"]["allowed_account_ids"] = [deploy_to_account_id]
98
186
 
99
187
  @staticmethod
100
- def _add_backend(terraform: dict, platform_config: dict, account: str, state_key_suffix: str):
188
+ def _add_backend(terraform: dict, platform_config: dict, account: str, state_key: str):
101
189
  terraform["terraform"] = {
102
190
  "required_version": SUPPORTED_TERRAFORM_VERSION,
103
191
  "backend": {
104
192
  "s3": {
105
193
  "bucket": f"terraform-platform-state-{account}",
106
- "key": f"tfstate/application/{state_key_suffix}.tfstate",
194
+ "key": state_key,
107
195
  "region": "eu-west-2",
108
196
  "encrypt": True,
109
197
  "kms_key_id": f"alias/terraform-platform-state-s3-key-{account}",
@@ -117,9 +205,12 @@ class TerraformManifestProvider:
117
205
 
118
206
  @staticmethod
119
207
  def _add_codebase_pipeline_module(
120
- terraform: dict, terraform_platform_modules_version: str, deploy_repository: str
208
+ terraform: dict,
209
+ platform_helper_version: str,
210
+ deploy_repository: str,
211
+ module_source: str,
121
212
  ):
122
- source = f"git::https://github.com/uktrade/terraform-platform-modules.git//codebase-pipelines?depth=1&ref={terraform_platform_modules_version}"
213
+ source = module_source
123
214
  terraform["module"] = {
124
215
  "codebase-pipelines": {
125
216
  "source": source,
@@ -128,20 +219,30 @@ class TerraformManifestProvider:
128
219
  "codebase": "${each.key}",
129
220
  "repository": "${each.value.repository}",
130
221
  "deploy_repository": f"{deploy_repository}",
222
+ "deploy_repository_branch": '${lookup(each.value, "deploy_repository_branch", "main")}',
131
223
  "additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
224
+ "cache_invalidation": '${lookup(each.value, "cache_invalidation", null)}',
132
225
  "pipelines": '${lookup(each.value, "pipelines", [])}',
133
226
  "services": "${each.value.services}",
134
227
  "requires_image_build": '${lookup(each.value, "requires_image_build", true)}',
135
228
  "slack_channel": '${lookup(each.value, "slack_channel", "/codebuild/slack_oauth_channel")}',
136
229
  "env_config": "${local.environments}",
230
+ "platform_tools_version": f"{platform_helper_version}",
137
231
  }
138
232
  }
139
233
 
140
234
  @staticmethod
141
- def _add_extensions_module(terraform: dict, terraform_platform_modules_version: str, env: str):
142
- source = f"git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref={terraform_platform_modules_version}"
235
+ def _add_extensions_module(
236
+ terraform: dict, platform_helper_version: str, env: str, module_source_override: str = None
237
+ ):
238
+ source = module_source_override or f"{EXTENSIONS_MODULE_PATH}{platform_helper_version}"
143
239
  terraform["module"] = {
144
- "extensions": {"source": source, "args": "${local.args}", "environment": env}
240
+ "extensions": {
241
+ "source": source,
242
+ "args": "${local.args}",
243
+ "environment": env,
244
+ "repos": "${local.codebase_pipeline_repos != null ? (distinct(values(local.codebase_pipeline_repos))) : null}",
245
+ }
145
246
  }
146
247
 
147
248
  @staticmethod
@@ -164,6 +265,7 @@ class TerraformManifestProvider:
164
265
  "services": '${local.config["extensions"]}',
165
266
  "env_config": "${local.env_config}",
166
267
  },
268
+ "codebase_pipeline_repos": '${try({for k, v in local.config["codebase_pipelines"]: k => v.repository}, null)}',
167
269
  }
168
270
 
169
271
  @staticmethod
@@ -204,7 +306,7 @@ class TerraformManifestProvider:
204
306
 
205
307
  def _write_terraform_json(self, terraform: dict, env_dir: str):
206
308
  message = self.file_provider.mkfile(
207
- str(Path(env_dir).absolute()),
309
+ str(Path(env_dir)),
208
310
  "main.tf.json",
209
311
  json.dumps(terraform, indent=2),
210
312
  True,
@@ -1,36 +1,119 @@
1
+ import re
2
+ import subprocess
1
3
  from abc import ABC
4
+ from abc import abstractmethod
5
+ from importlib.metadata import PackageNotFoundError
6
+ from importlib.metadata import version
7
+ from typing import Union
2
8
 
3
- import requests
9
+ from requests import Session
10
+ from requests.adapters import HTTPAdapter
11
+ from urllib3.util import Retry
4
12
 
5
- from dbt_platform_helper.providers.semantic_version import SemanticVersion
13
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
14
+ from dbt_platform_helper.platform_exception import PlatformException
15
+ from dbt_platform_helper.providers.io import ClickIOProvider
6
16
 
7
17
 
8
- class VersionProvider(ABC):
18
+ def set_up_retry():
19
+ session = Session()
20
+ retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[403, 500, 502, 503, 504])
21
+ session.mount("https://", HTTPAdapter(max_retries=retries))
22
+ return session
23
+
24
+
25
+ class InstalledVersionProviderException(PlatformException):
9
26
  pass
10
27
 
11
28
 
12
- # TODO add timeouts and exception handling for requests
13
- # TODO Alternatively use the gitpython package?
14
- class GithubVersionProvider(VersionProvider):
29
+ class InstalledToolNotFoundException(InstalledVersionProviderException):
30
+ def __init__(
31
+ self,
32
+ tool_name: str,
33
+ ):
34
+ super().__init__(f"Package '{tool_name}' not found.")
35
+
36
+
37
+ class VersionProvider(ABC):
38
+ @abstractmethod
39
+ def get_semantic_version() -> SemanticVersion:
40
+ raise NotImplementedError("Must be implemented in subclasses")
41
+
42
+
43
+ class InstalledVersionProvider:
44
+ @staticmethod
45
+ def get_semantic_version(tool_name: str) -> SemanticVersion:
46
+ try:
47
+ return SemanticVersion.from_string(version(tool_name))
48
+
49
+ except PackageNotFoundError:
50
+ raise InstalledToolNotFoundException(tool_name)
51
+
52
+
53
+ class GithubLatestVersionProvider(VersionProvider):
15
54
  @staticmethod
16
- def get_latest_version(repo_name: str, tags: bool = False) -> SemanticVersion:
17
- if tags:
18
- tags_list = requests.get(f"https://api.github.com/repos/{repo_name}/tags").json()
19
- versions = [SemanticVersion.from_string(v["name"]) for v in tags_list]
20
- versions.sort(reverse=True)
21
- return versions[0]
55
+ def get_semantic_version(
56
+ repo_name: str, tags: bool = False, request_session=set_up_retry(), io=ClickIOProvider()
57
+ ) -> Union[SemanticVersion, None]:
58
+
59
+ semantic_version = None
60
+ try:
61
+ if tags:
62
+ response = request_session.get(f"https://api.github.com/repos/{repo_name}/tags")
63
+
64
+ versions = [SemanticVersion.from_string(v["name"]) for v in response.json()]
65
+ versions.sort(reverse=True)
66
+ semantic_version = versions[0]
67
+ else:
68
+ package_info = request_session.get(
69
+ f"https://api.github.com/repos/{repo_name}/releases/latest"
70
+ ).json()
71
+ semantic_version = SemanticVersion.from_string(package_info["tag_name"])
72
+ except Exception as e:
73
+ io.error(f"Exception occured when calling Github with:\n{str(e)}")
22
74
 
23
- package_info = requests.get(
24
- f"https://api.github.com/repos/{repo_name}/releases/latest"
25
- ).json()
26
- return SemanticVersion.from_string(package_info["tag_name"])
75
+ return semantic_version
76
+
77
+
78
+ class PyPiLatestVersionProvider(VersionProvider):
79
+ @staticmethod
80
+ def get_semantic_version(
81
+ project_name: str, request_session=set_up_retry(), io=ClickIOProvider()
82
+ ) -> Union[SemanticVersion, None]:
83
+ semantic_version = None
84
+ try:
85
+ package_info = request_session.get(f"https://pypi.org/pypi/{project_name}/json").json()
86
+ released_versions = package_info["releases"].keys()
87
+ parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
88
+ parsed_released_versions.sort(reverse=True)
89
+ semantic_version = parsed_released_versions[0]
90
+ except Exception as e:
91
+ io.error(f"Exception occured when calling PyPi with:\n{str(e)}")
92
+ return semantic_version
27
93
 
28
94
 
29
- class PyPiVersionProvider(VersionProvider):
95
+ class AWSCLIInstalledVersionProvider(VersionProvider):
30
96
  @staticmethod
31
- def get_latest_version(project_name: str) -> SemanticVersion:
32
- package_info = requests.get(f"https://pypi.org/pypi/{project_name}/json").json()
33
- released_versions = package_info["releases"].keys()
34
- parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
35
- parsed_released_versions.sort(reverse=True)
36
- return parsed_released_versions[0]
97
+ def get_semantic_version() -> Union[SemanticVersion, None]:
98
+ installed_aws_version = None
99
+ try:
100
+ response = subprocess.run("aws --version", capture_output=True, shell=True)
101
+ matched = re.match(r"aws-cli/([0-9.]+)", response.stdout.decode("utf8"))
102
+ installed_aws_version = matched.group(1)
103
+ except (ValueError, AttributeError):
104
+ pass
105
+ return SemanticVersion.from_string(installed_aws_version)
106
+
107
+
108
+ class CopilotInstalledVersionProvider(VersionProvider):
109
+ @staticmethod
110
+ def get_semantic_version() -> Union[SemanticVersion, None]:
111
+ copilot_version = None
112
+
113
+ try:
114
+ response = subprocess.run("copilot --version", capture_output=True, shell=True)
115
+ [copilot_version] = re.findall(r"[0-9.]+", response.stdout.decode("utf8"))
116
+ except ValueError:
117
+ pass
118
+
119
+ return SemanticVersion.from_string(copilot_version)
@@ -0,0 +1,27 @@
1
+ from dataclasses import dataclass
2
+
3
+ from dbt_platform_helper.entities.semantic_version import SemanticVersion
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
+
6
+
7
+ class UnsupportedVersionException(PlatformException):
8
+ def __init__(self, version: str):
9
+ super().__init__(
10
+ f"""Platform-helper version {version} is not compatible with platform-helper. Please install version platform-helper version 14 or later."""
11
+ )
12
+
13
+
14
+ @dataclass
15
+ class VersionStatus:
16
+ installed: SemanticVersion = None
17
+ latest: SemanticVersion = None
18
+
19
+ def __str__(self):
20
+ attrs = {
21
+ key: value for key, value in vars(self).items() if isinstance(value, SemanticVersion)
22
+ }
23
+ attrs_str = ", ".join(f"{key}: {value}" for key, value in attrs.items())
24
+ return f"{self.__class__.__name__}: {attrs_str}"
25
+
26
+ def is_outdated(self):
27
+ return self.installed != self.latest