dbt-platform-helper 15.3.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. dbt_platform_helper/COMMANDS.md +36 -11
  2. dbt_platform_helper/commands/application.py +2 -1
  3. dbt_platform_helper/commands/conduit.py +1 -1
  4. dbt_platform_helper/commands/environment.py +12 -1
  5. dbt_platform_helper/commands/generate.py +0 -2
  6. dbt_platform_helper/commands/internal.py +140 -0
  7. dbt_platform_helper/commands/pipeline.py +15 -3
  8. dbt_platform_helper/commands/secrets.py +37 -89
  9. dbt_platform_helper/commands/version.py +3 -2
  10. dbt_platform_helper/constants.py +38 -2
  11. dbt_platform_helper/domain/conduit.py +22 -9
  12. dbt_platform_helper/domain/config.py +30 -1
  13. dbt_platform_helper/domain/database_copy.py +1 -1
  14. dbt_platform_helper/domain/maintenance_page.py +27 -3
  15. dbt_platform_helper/domain/pipelines.py +36 -60
  16. dbt_platform_helper/domain/secrets.py +279 -0
  17. dbt_platform_helper/domain/service.py +570 -0
  18. dbt_platform_helper/domain/terraform_environment.py +7 -29
  19. dbt_platform_helper/domain/update_alb_rules.py +412 -0
  20. dbt_platform_helper/domain/versioning.py +124 -13
  21. dbt_platform_helper/entities/platform_config_schema.py +31 -11
  22. dbt_platform_helper/entities/semantic_version.py +2 -0
  23. dbt_platform_helper/entities/service.py +339 -0
  24. dbt_platform_helper/providers/autoscaling.py +24 -0
  25. dbt_platform_helper/providers/aws/exceptions.py +5 -0
  26. dbt_platform_helper/providers/aws/sso_auth.py +14 -0
  27. dbt_platform_helper/providers/config.py +17 -2
  28. dbt_platform_helper/providers/config_validator.py +87 -2
  29. dbt_platform_helper/providers/ecs.py +131 -11
  30. dbt_platform_helper/providers/environment_variable.py +2 -2
  31. dbt_platform_helper/providers/io.py +9 -2
  32. dbt_platform_helper/providers/load_balancers.py +122 -16
  33. dbt_platform_helper/providers/logs.py +72 -0
  34. dbt_platform_helper/providers/parameter_store.py +97 -10
  35. dbt_platform_helper/providers/s3.py +21 -0
  36. dbt_platform_helper/providers/terraform_manifest.py +97 -13
  37. dbt_platform_helper/providers/vpc.py +36 -5
  38. dbt_platform_helper/providers/yaml_file.py +35 -0
  39. dbt_platform_helper/templates/environment-pipelines/main.tf +3 -2
  40. dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
  41. dbt_platform_helper/utils/application.py +104 -21
  42. dbt_platform_helper/utils/aws.py +11 -10
  43. dbt_platform_helper/utils/deep_merge.py +10 -0
  44. dbt_platform_helper/utils/git.py +1 -1
  45. {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +8 -17
  46. {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/RECORD +50 -41
  47. {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
  48. platform_helper.py +2 -0
  49. {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
  50. {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
@@ -25,6 +25,7 @@
25
25
  - [platform-helper pipeline](#platform-helper-pipeline)
26
26
  - [platform-helper pipeline generate](#platform-helper-pipeline-generate)
27
27
  - [platform-helper secrets](#platform-helper-secrets)
28
+ - [platform-helper secrets create](#platform-helper-secrets-create)
28
29
  - [platform-helper secrets copy](#platform-helper-secrets-copy)
29
30
  - [platform-helper secrets list](#platform-helper-secrets-list)
30
31
  - [platform-helper notify](#platform-helper-notify)
@@ -63,6 +64,7 @@ platform-helper <command> [--version]
63
64
  - [`database` ↪](#platform-helper-database)
64
65
  - [`environment` ↪](#platform-helper-environment)
65
66
  - [`generate` ↪](#platform-helper-generate)
67
+ - [`internal` ↪](#platform-helper-internal)
66
68
  - [`notify` ↪](#platform-helper-notify)
67
69
  - [`pipeline` ↪](#platform-helper-pipeline)
68
70
  - [`secrets` ↪](#platform-helper-secrets)
@@ -600,7 +602,7 @@ This is generated from the terraform/environments-pipeline/<aws_account>/main.tf
600
602
  ## Usage
601
603
 
602
604
  ```
603
- platform-helper secrets (copy|list)
605
+ platform-helper secrets (create|copy|list)
604
606
  ```
605
607
 
606
608
  ## Options
@@ -611,30 +613,53 @@ platform-helper secrets (copy|list)
611
613
  ## Commands
612
614
 
613
615
  - [`copy` ↪](#platform-helper-secrets-copy)
616
+ - [`create` ↪](#platform-helper-secrets-create)
614
617
  - [`list` ↪](#platform-helper-secrets-list)
615
618
 
616
- # platform-helper secrets copy
619
+ # platform-helper secrets create
617
620
 
618
621
  [↩ Parent](#platform-helper-secrets)
619
622
 
620
- Copy secrets from one environment to a new environment.
623
+ Create a Parameter Store secret for all environments of an
624
+ application.
621
625
 
622
626
  ## Usage
623
627
 
624
628
  ```
625
- platform-helper secrets copy <source_environment> <target_environment>
626
- --project-profile <project_profile>
629
+ platform-helper secrets create --app <application> --name <name> [--overwrite]
627
630
  ```
628
631
 
629
- ## Arguments
632
+ ## Options
630
633
 
631
- - `source_environment <text>`
632
- - `target_environment <text>`
634
+ - `--app <text>`
635
+ - Application name.
636
+ - `--name <text>`
637
+ - Secret name (automatically uppercased).
638
+ - `--overwrite <boolean>` _Defaults to False._
639
+ - Allows overwriting the value of secrets if they already exist.
640
+ - `--help <boolean>` _Defaults to False._
641
+ - Show this message and exit.
642
+
643
+ # platform-helper secrets copy
644
+
645
+ [↩ Parent](#platform-helper-secrets)
646
+
647
+ Copy secrets from one environment to another.
648
+
649
+ ## Usage
650
+
651
+ ```
652
+ platform-helper secrets copy --app <application> --source <source> --target <target>
653
+ ```
633
654
 
634
655
  ## Options
635
656
 
636
- - `--project-profile <text>`
637
- - AWS account profile name
657
+ - `--app <text>`
658
+ - Application name.
659
+ - `--source <text>`
660
+ - Source environment where to copy secrets from.
661
+ - `--target <text>`
662
+ - Destination environment where to copy secrets to.
638
663
  - `--help <boolean>` _Defaults to False._
639
664
  - Show this message and exit.
640
665
 
@@ -642,7 +667,7 @@ platform-helper secrets copy <source_environment> <target_environment>
642
667
 
643
668
  [↩ Parent](#platform-helper-secrets)
644
669
 
645
- List secret names and values for an environment.
670
+ [DELETED] List secret names and values for an environment.
646
671
 
647
672
  ## Usage
648
673
 
@@ -281,12 +281,13 @@ def task_stats(env, app, disk, storage, network):
281
281
  memory,
282
282
  dsk,
283
283
  ) in cpu_response["results"]:
284
+ cpu_value = f"{float(cpu['value']):.1f}" + "%"
284
285
  values = [
285
286
  f"{YELLOW}{taskdef['value'].split('-')[-1]}",
286
287
  f"{YELLOW}{task['value']}",
287
288
  f"{YELLOW}{task_def_revision['value']}",
288
289
  f"{YELLOW}{status['value']}",
289
- f"{YELLOW}{'%.1f' % float(cpu['value']) + '%'}",
290
+ f"{YELLOW}{cpu_value}",
290
291
  f"{YELLOW}{memory['value'] + 'M'}",
291
292
  ]
292
293
 
@@ -27,7 +27,7 @@ def conduit(addon_name: str, app: str, env: str, access: str):
27
27
  """Opens a shell for a given addon_name create a conduit connection to
28
28
  interact with postgres, opensearch or redis."""
29
29
  PlatformHelperVersioning().check_if_needs_update()
30
- application = load_application(app)
30
+ application = load_application(app=app, env=env)
31
31
 
32
32
  try:
33
33
  secrets_provider: Secrets = Secrets(
@@ -8,7 +8,11 @@ from dbt_platform_helper.platform_exception import PlatformException
8
8
  from dbt_platform_helper.providers.cloudformation import CloudFormation
9
9
  from dbt_platform_helper.providers.config import ConfigProvider
10
10
  from dbt_platform_helper.providers.config_validator import ConfigValidator
11
+ from dbt_platform_helper.providers.environment_variable import (
12
+ EnvironmentVariableProvider,
13
+ )
11
14
  from dbt_platform_helper.providers.io import ClickIOProvider
15
+ from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
12
16
  from dbt_platform_helper.providers.vpc import VpcProvider
13
17
  from dbt_platform_helper.utils.application import load_application
14
18
  from dbt_platform_helper.utils.aws import get_aws_session_or_abort
@@ -91,7 +95,14 @@ def generate_terraform(name):
91
95
  try:
92
96
  session = get_aws_session_or_abort()
93
97
  config_provider = ConfigProvider(ConfigValidator(session=session))
94
- TerraformEnvironment(config_provider).generate(name)
98
+ platform_helper_versioning = PlatformHelperVersioning(
99
+ click_io,
100
+ config_provider,
101
+ EnvironmentVariableProvider(),
102
+ )
103
+ TerraformEnvironment(
104
+ config_provider, TerraformManifestProvider(), click_io, platform_helper_versioning
105
+ ).generate(name)
95
106
 
96
107
  except PlatformException as err:
97
108
  click_io.abort_with_error(str(err))
@@ -3,7 +3,6 @@ import click
3
3
 
4
4
  from dbt_platform_helper.commands.copilot import make_addons
5
5
  from dbt_platform_helper.commands.pipeline import generate as pipeline_generate
6
- from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
7
6
  from dbt_platform_helper.utils.click import ClickDocOptCommand
8
7
 
9
8
 
@@ -16,6 +15,5 @@ def generate(ctx: click.Context):
16
15
 
17
16
  Wraps pipeline generate and make-addons.
18
17
  """
19
- PlatformHelperVersioning().check_platform_helper_version_mismatch()
20
18
  ctx.invoke(pipeline_generate)
21
19
  ctx.invoke(make_addons)
@@ -0,0 +1,140 @@
1
+ import click
2
+
3
+ from dbt_platform_helper.domain.service import ServiceManager
4
+ from dbt_platform_helper.domain.update_alb_rules import UpdateALBRules
5
+ from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
6
+ from dbt_platform_helper.platform_exception import PlatformException
7
+ from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
8
+ from dbt_platform_helper.providers.config import ConfigProvider
9
+ from dbt_platform_helper.providers.config_validator import ConfigValidator
10
+ from dbt_platform_helper.providers.ecs import ECS
11
+ from dbt_platform_helper.providers.io import ClickIOProvider
12
+ from dbt_platform_helper.providers.logs import LogsProvider
13
+ from dbt_platform_helper.providers.s3 import S3Provider
14
+ from dbt_platform_helper.utils.application import load_application
15
+ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
16
+ from dbt_platform_helper.utils.click import ClickDocOptGroup
17
+
18
+
19
+ @click.group(cls=ClickDocOptGroup)
20
+ def internal():
21
+ """Internal commands for use within pipelines or by Platform Team."""
22
+
23
+
24
+ @internal.command()
25
+ def migrate_service_manifests():
26
+ """Migrate copilot manifests to service manifests."""
27
+ click_io = ClickIOProvider()
28
+
29
+ try:
30
+ service_manager = ServiceManager()
31
+ service_manager.migrate_copilot_manifests()
32
+ except PlatformException as error:
33
+ click_io.abort_with_error(str(error))
34
+
35
+
36
+ @internal.group(cls=ClickDocOptGroup)
37
+ def service():
38
+ """Subgroup for 'internal service' commands."""
39
+
40
+
41
+ @service.command(help="Trigger an ECS deployment.")
42
+ @click.option("--name", required=True, help="The name of the ECS service to create or update.")
43
+ @click.option(
44
+ "--env",
45
+ required=True,
46
+ help="The name of the environment where the ECS service will be created or updated.",
47
+ )
48
+ @click.option(
49
+ "--image-tag",
50
+ required=True,
51
+ help="Image tag to deploy for the service(s). Takes precedence over the $IMAGE_TAG environment variable.",
52
+ )
53
+ def deploy(name, env, image_tag):
54
+ """Register a new ECS task definition from an S3 JSON template, update the
55
+ ECS service, and tail CloudWatch logs until the ECS rollout is complete."""
56
+ click_io = ClickIOProvider()
57
+
58
+ try:
59
+
60
+ config = ConfigProvider(ConfigValidator()).get_enriched_config()
61
+ application_name = config.get("application", "")
62
+ application = load_application(app=application_name, env=env)
63
+
64
+ ecs_client = application.environments[env].session.client("ecs")
65
+ ssm_client = application.environments[env].session.client("ssm")
66
+ s3_client = application.environments[env].session.client("s3")
67
+ logs_client = application.environments[env].session.client("logs")
68
+ autoscaling_client = application.environments[env].session.client("application-autoscaling")
69
+
70
+ ecs_provider = ECS(
71
+ ecs_client=ecs_client,
72
+ ssm_client=ssm_client,
73
+ application_name=application.name,
74
+ env=env,
75
+ )
76
+ s3_provider = S3Provider(client=s3_client)
77
+ logs_provider = LogsProvider(client=logs_client)
78
+ autoscaling_provider = AutoscalingProvider(client=autoscaling_client)
79
+
80
+ service_manager = ServiceManager(
81
+ ecs_provider=ecs_provider,
82
+ s3_provider=s3_provider,
83
+ logs_provider=logs_provider,
84
+ autoscaling_provider=autoscaling_provider,
85
+ )
86
+ service_manager.deploy(
87
+ service=name,
88
+ environment=env,
89
+ application=application.name,
90
+ image_tag=image_tag,
91
+ )
92
+ except PlatformException as error:
93
+ click_io.abort_with_error(str(error))
94
+
95
+
96
+ @service.command(help="Generate Terraform manifest for the specified service(s).")
97
+ @click.option(
98
+ "--name",
99
+ required=False,
100
+ help="The name of the service(s) to generate service manifest(s) for.",
101
+ multiple=True,
102
+ )
103
+ @click.option(
104
+ "--env",
105
+ required=True,
106
+ help="The name of the environment to generate service manifests for.",
107
+ )
108
+ def generate(name, env):
109
+ """Validates the service-config.yml format, applies the environment-specific
110
+ overrides, and generates a Terraform manifest at
111
+ /terraform/services/<environment>/<service>/main.tf.json."""
112
+
113
+ services = list(name)
114
+ click_io = ClickIOProvider()
115
+
116
+ try:
117
+ service_manager = ServiceManager()
118
+ service_manager.generate(environment=env, services=services)
119
+
120
+ except PlatformException as err:
121
+ click_io.abort_with_error(str(err))
122
+
123
+
124
+ @internal.group(cls=ClickDocOptGroup)
125
+ def alb():
126
+ """Load Balancer related commands."""
127
+ PlatformHelperVersioning().check_if_needs_update()
128
+
129
+
130
+ @alb.command()
131
+ @click.option("--env", type=str, required=True)
132
+ def update_rules(env: str):
133
+ """Update alb rules based on service-deployment-mode for a given
134
+ environment."""
135
+ try:
136
+ session = get_aws_session_or_abort()
137
+ update_aws = UpdateALBRules(session)
138
+ update_aws.update_alb_rules(environment=env)
139
+ except PlatformException as err:
140
+ ClickIOProvider().abort_with_error(str(err))
@@ -6,9 +6,12 @@ from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
6
6
  from dbt_platform_helper.providers.config import ConfigProvider
7
7
  from dbt_platform_helper.providers.config_validator import ConfigValidator
8
8
  from dbt_platform_helper.providers.ecr import ECRProvider
9
+ from dbt_platform_helper.providers.environment_variable import (
10
+ EnvironmentVariableProvider,
11
+ )
12
+ from dbt_platform_helper.providers.files import FileProvider
9
13
  from dbt_platform_helper.providers.io import ClickIOProvider
10
14
  from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
11
- from dbt_platform_helper.utils.aws import get_codestar_connection_arn
12
15
  from dbt_platform_helper.utils.click import ClickDocOptGroup
13
16
  from dbt_platform_helper.utils.git import git_remote
14
17
 
@@ -43,15 +46,24 @@ def generate(deploy_branch: str):
43
46
  The `terraform/codebase-pipelines/main.tf.json` file is generated using this configuration.
44
47
  The `main.tf.json` file is then used to generate Terraform for creating a codebase pipeline resource.
45
48
  """
49
+ config_provider = ConfigProvider(ConfigValidator())
50
+ environment_variable_provider = EnvironmentVariableProvider()
46
51
  io = ClickIOProvider()
52
+ platform_helper_versioning = PlatformHelperVersioning(
53
+ io,
54
+ config_provider,
55
+ environment_variable_provider,
56
+ )
57
+
47
58
  try:
48
59
  pipelines = Pipelines(
49
- ConfigProvider(ConfigValidator()),
60
+ config_provider,
50
61
  TerraformManifestProvider(),
51
62
  ECRProvider(),
52
63
  git_remote,
53
- get_codestar_connection_arn,
54
64
  io,
65
+ FileProvider(),
66
+ platform_helper_versioning,
55
67
  )
56
68
  pipelines.generate(deploy_branch)
57
69
  except Exception as exc:
@@ -1,17 +1,12 @@
1
1
  #!/usr/bin/env python
2
2
 
3
- from pathlib import Path
4
3
 
5
4
  import click
6
- from botocore.exceptions import ClientError
7
- from cloudfoundry_client.client import CloudFoundryClient
8
5
 
6
+ from dbt_platform_helper.domain.secrets import Secrets
9
7
  from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
10
- from dbt_platform_helper.utils.application import get_application_name
11
- from dbt_platform_helper.utils.aws import SSM_BASE_PATH
12
- from dbt_platform_helper.utils.aws import get_aws_session_or_abort
13
- from dbt_platform_helper.utils.aws import get_ssm_secrets
14
- from dbt_platform_helper.utils.aws import set_ssm_param
8
+ from dbt_platform_helper.platform_exception import PlatformException
9
+ from dbt_platform_helper.providers.io import ClickIOProvider
15
10
  from dbt_platform_helper.utils.click import ClickDocOptGroup
16
11
 
17
12
 
@@ -19,100 +14,53 @@ def secret_should_be_skipped(secret_name):
19
14
  return "AWS_" in secret_name
20
15
 
21
16
 
22
- def get_paas_env_vars(client: CloudFoundryClient, paas: str) -> dict:
23
- org, space, app = paas.split("/")
24
-
25
- env_vars = None
26
-
27
- for paas_org in client.v2.organizations:
28
- if paas_org["entity"]["name"] == org:
29
- for paas_space in paas_org.spaces():
30
- if paas_space["entity"]["name"] == space:
31
- for paas_app in paas_space.apps():
32
- if paas_app["entity"]["name"] == app:
33
- env_vars = paas_app["entity"]["environment_json"]
34
-
35
- if not env_vars:
36
- raise Exception(f"Application {paas} not found")
37
-
38
- return dict(env_vars)
39
-
40
-
41
17
  @click.group(chain=True, cls=ClickDocOptGroup)
42
18
  def secrets():
43
19
  PlatformHelperVersioning().check_if_needs_update()
44
20
 
45
21
 
46
22
  @secrets.command()
47
- @click.argument("source_environment")
48
- @click.argument("target_environment")
49
- @click.option("--project-profile", required=True, help="AWS account profile name")
50
- def copy(project_profile, source_environment, target_environment):
51
- """Copy secrets from one environment to a new environment."""
52
- get_aws_session_or_abort(project_profile)
53
-
54
- if not Path(f"copilot/environments/{target_environment}").exists():
55
- click.echo(f"""Target environment manifest for "{target_environment}" does not exist.""")
56
- exit(1)
57
-
58
- app_name = get_application_name()
59
- secrets = get_ssm_secrets(app_name, source_environment)
60
-
61
- for secret in secrets:
62
- secret_name = secret[0].replace(f"/{source_environment}/", f"/{target_environment}/")
63
-
64
- if secret_should_be_skipped(secret_name):
65
- continue
66
-
67
- click.echo(secret_name)
68
-
69
- try:
70
- set_ssm_param(
71
- app_name,
72
- target_environment,
73
- secret_name,
74
- secret[1],
75
- False,
76
- False,
77
- f"Copied from {source_environment} environment.",
78
- )
79
- except ClientError as e:
80
- if e.response["Error"]["Code"] == "ParameterAlreadyExists":
81
- click.secho(
82
- f"""The "{secret_name.split("/")[-1]}" parameter already exists for the "{target_environment}" environment.""",
83
- fg="yellow",
84
- )
85
- else:
86
- raise e
23
+ @click.option("--app", help="Application name.", required=True)
24
+ @click.option("--name", help="Secret name (automatically uppercased).", required=True)
25
+ @click.option(
26
+ "--overwrite",
27
+ is_flag=True,
28
+ default=False,
29
+ help="Allows overwriting the value of secrets if they already exist.",
30
+ )
31
+ def create(app: str, name: str, overwrite: bool):
32
+ """Create a Parameter Store secret for all environments of an
33
+ application."""
34
+
35
+ try:
36
+ Secrets().create(app, name, overwrite)
37
+ except PlatformException as err:
38
+ ClickIOProvider().abort_with_error(str(err))
87
39
 
88
40
 
89
41
  @secrets.command()
90
- @click.argument("app", type=str, required=True)
91
- @click.argument("env", type=str, required=True)
92
- def list(app, env):
93
- """List secret names and values for an environment."""
94
-
95
- session = get_aws_session_or_abort()
96
- client = session.client("ssm")
97
-
98
- path = SSM_BASE_PATH.format(app=app, env=env)
99
-
100
- params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
101
- secrets = []
42
+ @click.option("--app", help="Application name.", required=True)
43
+ @click.option("--source", help="Source environment where to copy secrets from.", required=True)
44
+ @click.option("--target", help="Destination environment where to copy secrets to.", required=True)
45
+ def copy(app, source, target):
46
+ """Copy secrets from one environment to another."""
102
47
 
103
- while True:
104
- response = client.get_parameters_by_path(**params)
48
+ try:
49
+ Secrets().copy(app, source, target)
50
+ except PlatformException as err:
51
+ ClickIOProvider().abort_with_error(str(err))
105
52
 
106
- for secret in response["Parameters"]:
107
- secrets.append(f"{secret['Name']:<8}: {secret['Value']:<15}")
108
53
 
109
- if "NextToken" in response:
110
- params["NextToken"] = response["NextToken"]
111
- else:
112
- break
54
+ @secrets.command()
55
+ @click.argument("app", type=str, required=True)
56
+ @click.argument("env", type=str, required=True)
57
+ def list(app, env):
58
+ """[DELETED] List secret names and values for an environment."""
113
59
 
114
- # TODO: DBTP-1953: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
115
- print("\n".join(sorted(secrets)))
60
+ click.secho(
61
+ message="\nThis command has been removed to prevent accidental exposure of secret values in local terminals and logs. To view secrets, log into your AWS account and head over to AWS Parameter Store https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/\n",
62
+ fg="magenta",
63
+ )
116
64
 
117
65
 
118
66
  if __name__ == "__main__":
@@ -25,7 +25,8 @@ def get_platform_helper_for_project(pipeline):
25
25
 
26
26
  Version preferences removed, requires specifying platform-helper version in platform-config.yml
27
27
  """
28
+ io = ClickIOProvider()
28
29
  try:
29
- PlatformHelperVersioning().get_required_version()
30
+ io.info(PlatformHelperVersioning().get_required_version())
30
31
  except PlatformException as err:
31
- ClickIOProvider().abort_with_error(str(err))
32
+ io.abort_with_error(str(err))
@@ -1,5 +1,18 @@
1
1
  # TODO: DBTP-1888: Move to Config provider
2
+ REPO_ORG = "uktrade"
2
3
  PLATFORM_CONFIG_FILE = "platform-config.yml"
4
+ SERVICE_DIRECTORY = "services"
5
+ SERVICE_CONFIG_FILE = "service-config.yml"
6
+ ENVIRONMENT_PIPELINE_MODULE_PATH = (
7
+ "git::git@github.com:uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref="
8
+ )
9
+
10
+ CODEBASE_PIPELINE_MODULE_PATH = (
11
+ "git::git@github.com:uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref="
12
+ )
13
+ EXTENSIONS_MODULE_PATH = (
14
+ "git::git@github.com:uktrade/platform-tools.git//terraform/extensions?depth=1&ref="
15
+ )
3
16
  PLATFORM_HELPER_VERSION_OVERRIDE_KEY = "PLATFORM_HELPER_VERSION_OVERRIDE"
4
17
  TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR = "TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE"
5
18
  TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
@@ -8,13 +21,19 @@ TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
8
21
  TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
9
22
  "TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE"
10
23
  )
24
+ TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
25
+ "TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE"
26
+ )
27
+ TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR = "TERRAFORM_MODULE_SOURCE_TYPE" # "LOCAL", "SSH", "OVERRIDE"
28
+ IMAGE_TAG_ENV_VAR = "IMAGE_TAG"
11
29
  PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
12
- SUPPORTED_TERRAFORM_VERSION = "~> 1.8"
13
- SUPPORTED_AWS_PROVIDER_VERSION = "~> 5"
30
+ SUPPORTED_TERRAFORM_VERSION = "~> 1.11"
31
+ SUPPORTED_AWS_PROVIDER_VERSION = "~> 6"
14
32
  FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR_VERSION = 13
15
33
 
16
34
  MERGED_TPM_PLATFORM_HELPER_VERSION = 14
17
35
  PLATFORM_CONFIG_SCHEMA_VERSION = 1
36
+ SERVICE_CONFIG_SCHEMA_VERSION = 1
18
37
 
19
38
  # Keys
20
39
  CODEBASE_PIPELINES_KEY = "codebase_pipelines"
@@ -37,3 +56,20 @@ SERVICE_NAME_SUFFIX = f"Service-{COPILOT_IDENTIFIER}"
37
56
  REFRESH_TOKEN_MESSAGE = (
38
57
  "To refresh this SSO session run `aws sso login` with the corresponding profile"
39
58
  )
59
+ COPILOT_RULE_PRIORITY = 48000
60
+ PLATFORM_RULE_STARTING_PRIORITY = 10000
61
+ RULE_PRIORITY_INCREMENT = 100
62
+ DUMMY_RULE_REASON = "DummyRule"
63
+ MAINTENANCE_PAGE_TAGS = ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]
64
+ MAINTENANCE_PAGE_REASON = "MaintenancePage"
65
+ MANAGED_BY_PLATFORM = "DBT Platform"
66
+ MANAGED_BY_SERVICE_TERRAFORM = "DBT Platform - Service Terraform"
67
+ MANAGED_BY_PLATFORM_TERRAFORM = "DBT Platform - Terraform"
68
+ STANDARD_PLATFORM_SSO_ROLES = [
69
+ "AdministratorAccess",
70
+ "DBTPlatformDeveloperWrite",
71
+ "DBTPlatformDeveloperRead",
72
+ ]
73
+ ROUTED_TO_PLATFORM_MODES = ["platform", "dual-deploy-platform-traffic"]
74
+ ROUTED_TO_COPILOT_MODES = ["copilot", "dual-deploy-copilot-traffic"]
75
+ HTTP_SERVICE_TYPES = ["Load Balanced Web Service"]
@@ -1,3 +1,4 @@
1
+ import json
1
2
  from abc import ABC
2
3
  from abc import abstractmethod
3
4
  from typing import Callable
@@ -56,10 +57,16 @@ class TerraformConduitStrategy(ConduitECSStrategy):
56
57
 
57
58
  def get_data(self):
58
59
  self.io.info("Starting conduit in Terraform mode.")
59
- return {
60
- "cluster_arn": self.ecs_provider.get_cluster_arn_by_name(
60
+ try:
61
+ cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
62
+ f"{self.application.name}-{self.env}-cluster"
63
+ )
64
+ except:
65
+ cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
61
66
  f"{self.application.name}-{self.env}"
62
- ),
67
+ )
68
+ return {
69
+ "cluster_arn": cluster_arn,
63
70
  "task_def_family": self._generate_container_name(),
64
71
  "vpc_name": self._resolve_vpc_name(),
65
72
  "addon_type": self.addon_type,
@@ -94,8 +101,10 @@ class TerraformConduitStrategy(ConduitECSStrategy):
94
101
  },
95
102
  ]
96
103
 
104
+ cluster_name = data_context["cluster_arn"].split("/")[-1]
105
+
97
106
  self.ecs_provider.start_ecs_task(
98
- f"{self.application.name}-{self.env}",
107
+ cluster_name,
99
108
  self._generate_container_name(),
100
109
  data_context["task_def_family"],
101
110
  vpc_config,
@@ -110,14 +119,18 @@ class TerraformConduitStrategy(ConduitECSStrategy):
110
119
 
111
120
  def _resolve_vpc_name(self):
112
121
  ssm_client = self.clients["ssm"]
113
- parameter_key = f"/conduit/{self.application.name}/{self.env}/{_normalise_secret_name(self.addon_name)}_VPC_NAME"
122
+ parameter_key = f"/platform/applications/{self.application.name}/environments/{self.env}"
114
123
 
115
124
  try:
116
- response = ssm_client.get_parameter(Name=parameter_key)
117
- return response["Parameter"]["Value"]
125
+ response = ssm_client.get_parameter(Name=parameter_key)["Parameter"]["Value"]
126
+ return json.loads(response)["vpc_name"]
118
127
  except ssm_client.exceptions.ParameterNotFound:
119
128
  self.io.abort_with_error(
120
- f"Could not find VPC name for {self.addon_name}. Missing SSM param: {parameter_key}"
129
+ f"Could not find AWS SSM parameter {parameter_key}. Please ensure your environment Terraform is up to date."
130
+ )
131
+ except KeyError:
132
+ self.io.abort_with_error(
133
+ f"The parameter {parameter_key} exists but does not contain the 'vpc_name' field. Please ensure your environment Terraform is up to date."
121
134
  )
122
135
 
123
136
 
@@ -326,7 +339,7 @@ class Conduit:
326
339
  data_context = strategy.get_data()
327
340
 
328
341
  data_context["task_arns"] = self.ecs_provider.get_ecs_task_arns(
329
- data_context["cluster_arn"], data_context["task_def_family"]
342
+ cluster=data_context["cluster_arn"], task_def_family=data_context["task_def_family"]
330
343
  )
331
344
 
332
345
  info_log = (