dbt-platform-helper 15.3.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +36 -11
- dbt_platform_helper/commands/application.py +2 -1
- dbt_platform_helper/commands/conduit.py +1 -1
- dbt_platform_helper/commands/environment.py +12 -1
- dbt_platform_helper/commands/generate.py +0 -2
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/pipeline.py +15 -3
- dbt_platform_helper/commands/secrets.py +37 -89
- dbt_platform_helper/commands/version.py +3 -2
- dbt_platform_helper/constants.py +38 -2
- dbt_platform_helper/domain/conduit.py +22 -9
- dbt_platform_helper/domain/config.py +30 -1
- dbt_platform_helper/domain/database_copy.py +1 -1
- dbt_platform_helper/domain/maintenance_page.py +27 -3
- dbt_platform_helper/domain/pipelines.py +36 -60
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +7 -29
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +124 -13
- dbt_platform_helper/entities/platform_config_schema.py +31 -11
- dbt_platform_helper/entities/semantic_version.py +2 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/exceptions.py +5 -0
- dbt_platform_helper/providers/aws/sso_auth.py +14 -0
- dbt_platform_helper/providers/config.py +17 -2
- dbt_platform_helper/providers/config_validator.py +87 -2
- dbt_platform_helper/providers/ecs.py +131 -11
- dbt_platform_helper/providers/environment_variable.py +2 -2
- dbt_platform_helper/providers/io.py +9 -2
- dbt_platform_helper/providers/load_balancers.py +122 -16
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +97 -10
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/terraform_manifest.py +97 -13
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +35 -0
- dbt_platform_helper/templates/environment-pipelines/main.tf +3 -2
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utils/application.py +104 -21
- dbt_platform_helper/utils/aws.py +11 -10
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -1
- {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +8 -17
- {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/RECORD +50 -41
- {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +2 -0
- {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
dbt_platform_helper/COMMANDS.md
CHANGED
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
- [platform-helper pipeline](#platform-helper-pipeline)
|
|
26
26
|
- [platform-helper pipeline generate](#platform-helper-pipeline-generate)
|
|
27
27
|
- [platform-helper secrets](#platform-helper-secrets)
|
|
28
|
+
- [platform-helper secrets create](#platform-helper-secrets-create)
|
|
28
29
|
- [platform-helper secrets copy](#platform-helper-secrets-copy)
|
|
29
30
|
- [platform-helper secrets list](#platform-helper-secrets-list)
|
|
30
31
|
- [platform-helper notify](#platform-helper-notify)
|
|
@@ -63,6 +64,7 @@ platform-helper <command> [--version]
|
|
|
63
64
|
- [`database` ↪](#platform-helper-database)
|
|
64
65
|
- [`environment` ↪](#platform-helper-environment)
|
|
65
66
|
- [`generate` ↪](#platform-helper-generate)
|
|
67
|
+
- [`internal` ↪](#platform-helper-internal)
|
|
66
68
|
- [`notify` ↪](#platform-helper-notify)
|
|
67
69
|
- [`pipeline` ↪](#platform-helper-pipeline)
|
|
68
70
|
- [`secrets` ↪](#platform-helper-secrets)
|
|
@@ -600,7 +602,7 @@ This is generated from the terraform/environments-pipeline/<aws_account>/main.tf
|
|
|
600
602
|
## Usage
|
|
601
603
|
|
|
602
604
|
```
|
|
603
|
-
platform-helper secrets (copy|list)
|
|
605
|
+
platform-helper secrets (create|copy|list)
|
|
604
606
|
```
|
|
605
607
|
|
|
606
608
|
## Options
|
|
@@ -611,30 +613,53 @@ platform-helper secrets (copy|list)
|
|
|
611
613
|
## Commands
|
|
612
614
|
|
|
613
615
|
- [`copy` ↪](#platform-helper-secrets-copy)
|
|
616
|
+
- [`create` ↪](#platform-helper-secrets-create)
|
|
614
617
|
- [`list` ↪](#platform-helper-secrets-list)
|
|
615
618
|
|
|
616
|
-
# platform-helper secrets
|
|
619
|
+
# platform-helper secrets create
|
|
617
620
|
|
|
618
621
|
[↩ Parent](#platform-helper-secrets)
|
|
619
622
|
|
|
620
|
-
|
|
623
|
+
Create a Parameter Store secret for all environments of an
|
|
624
|
+
application.
|
|
621
625
|
|
|
622
626
|
## Usage
|
|
623
627
|
|
|
624
628
|
```
|
|
625
|
-
platform-helper secrets
|
|
626
|
-
--project-profile <project_profile>
|
|
629
|
+
platform-helper secrets create --app <application> --name <name> [--overwrite]
|
|
627
630
|
```
|
|
628
631
|
|
|
629
|
-
##
|
|
632
|
+
## Options
|
|
630
633
|
|
|
631
|
-
-
|
|
632
|
-
-
|
|
634
|
+
- `--app <text>`
|
|
635
|
+
- Application name.
|
|
636
|
+
- `--name <text>`
|
|
637
|
+
- Secret name (automatically uppercased).
|
|
638
|
+
- `--overwrite <boolean>` _Defaults to False._
|
|
639
|
+
- Allows overwriting the value of secrets if they already exist.
|
|
640
|
+
- `--help <boolean>` _Defaults to False._
|
|
641
|
+
- Show this message and exit.
|
|
642
|
+
|
|
643
|
+
# platform-helper secrets copy
|
|
644
|
+
|
|
645
|
+
[↩ Parent](#platform-helper-secrets)
|
|
646
|
+
|
|
647
|
+
Copy secrets from one environment to another.
|
|
648
|
+
|
|
649
|
+
## Usage
|
|
650
|
+
|
|
651
|
+
```
|
|
652
|
+
platform-helper secrets copy --app <application> --source <source> --target <target>
|
|
653
|
+
```
|
|
633
654
|
|
|
634
655
|
## Options
|
|
635
656
|
|
|
636
|
-
- `--
|
|
637
|
-
-
|
|
657
|
+
- `--app <text>`
|
|
658
|
+
- Application name.
|
|
659
|
+
- `--source <text>`
|
|
660
|
+
- Source environment where to copy secrets from.
|
|
661
|
+
- `--target <text>`
|
|
662
|
+
- Destination environment where to copy secrets to.
|
|
638
663
|
- `--help <boolean>` _Defaults to False._
|
|
639
664
|
- Show this message and exit.
|
|
640
665
|
|
|
@@ -642,7 +667,7 @@ platform-helper secrets copy <source_environment> <target_environment>
|
|
|
642
667
|
|
|
643
668
|
[↩ Parent](#platform-helper-secrets)
|
|
644
669
|
|
|
645
|
-
List secret names and values for an environment.
|
|
670
|
+
[DELETED] List secret names and values for an environment.
|
|
646
671
|
|
|
647
672
|
## Usage
|
|
648
673
|
|
|
@@ -281,12 +281,13 @@ def task_stats(env, app, disk, storage, network):
|
|
|
281
281
|
memory,
|
|
282
282
|
dsk,
|
|
283
283
|
) in cpu_response["results"]:
|
|
284
|
+
cpu_value = f"{float(cpu['value']):.1f}" + "%"
|
|
284
285
|
values = [
|
|
285
286
|
f"{YELLOW}{taskdef['value'].split('-')[-1]}",
|
|
286
287
|
f"{YELLOW}{task['value']}",
|
|
287
288
|
f"{YELLOW}{task_def_revision['value']}",
|
|
288
289
|
f"{YELLOW}{status['value']}",
|
|
289
|
-
f"{YELLOW}{
|
|
290
|
+
f"{YELLOW}{cpu_value}",
|
|
290
291
|
f"{YELLOW}{memory['value'] + 'M'}",
|
|
291
292
|
]
|
|
292
293
|
|
|
@@ -27,7 +27,7 @@ def conduit(addon_name: str, app: str, env: str, access: str):
|
|
|
27
27
|
"""Opens a shell for a given addon_name create a conduit connection to
|
|
28
28
|
interact with postgres, opensearch or redis."""
|
|
29
29
|
PlatformHelperVersioning().check_if_needs_update()
|
|
30
|
-
application = load_application(app)
|
|
30
|
+
application = load_application(app=app, env=env)
|
|
31
31
|
|
|
32
32
|
try:
|
|
33
33
|
secrets_provider: Secrets = Secrets(
|
|
@@ -8,7 +8,11 @@ from dbt_platform_helper.platform_exception import PlatformException
|
|
|
8
8
|
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
9
9
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
10
10
|
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
11
|
+
from dbt_platform_helper.providers.environment_variable import (
|
|
12
|
+
EnvironmentVariableProvider,
|
|
13
|
+
)
|
|
11
14
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
15
|
+
from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
|
|
12
16
|
from dbt_platform_helper.providers.vpc import VpcProvider
|
|
13
17
|
from dbt_platform_helper.utils.application import load_application
|
|
14
18
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
@@ -91,7 +95,14 @@ def generate_terraform(name):
|
|
|
91
95
|
try:
|
|
92
96
|
session = get_aws_session_or_abort()
|
|
93
97
|
config_provider = ConfigProvider(ConfigValidator(session=session))
|
|
94
|
-
|
|
98
|
+
platform_helper_versioning = PlatformHelperVersioning(
|
|
99
|
+
click_io,
|
|
100
|
+
config_provider,
|
|
101
|
+
EnvironmentVariableProvider(),
|
|
102
|
+
)
|
|
103
|
+
TerraformEnvironment(
|
|
104
|
+
config_provider, TerraformManifestProvider(), click_io, platform_helper_versioning
|
|
105
|
+
).generate(name)
|
|
95
106
|
|
|
96
107
|
except PlatformException as err:
|
|
97
108
|
click_io.abort_with_error(str(err))
|
|
@@ -3,7 +3,6 @@ import click
|
|
|
3
3
|
|
|
4
4
|
from dbt_platform_helper.commands.copilot import make_addons
|
|
5
5
|
from dbt_platform_helper.commands.pipeline import generate as pipeline_generate
|
|
6
|
-
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
7
6
|
from dbt_platform_helper.utils.click import ClickDocOptCommand
|
|
8
7
|
|
|
9
8
|
|
|
@@ -16,6 +15,5 @@ def generate(ctx: click.Context):
|
|
|
16
15
|
|
|
17
16
|
Wraps pipeline generate and make-addons.
|
|
18
17
|
"""
|
|
19
|
-
PlatformHelperVersioning().check_platform_helper_version_mismatch()
|
|
20
18
|
ctx.invoke(pipeline_generate)
|
|
21
19
|
ctx.invoke(make_addons)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from dbt_platform_helper.domain.service import ServiceManager
|
|
4
|
+
from dbt_platform_helper.domain.update_alb_rules import UpdateALBRules
|
|
5
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
6
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
7
|
+
from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
|
|
8
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
9
|
+
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
10
|
+
from dbt_platform_helper.providers.ecs import ECS
|
|
11
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
12
|
+
from dbt_platform_helper.providers.logs import LogsProvider
|
|
13
|
+
from dbt_platform_helper.providers.s3 import S3Provider
|
|
14
|
+
from dbt_platform_helper.utils.application import load_application
|
|
15
|
+
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
16
|
+
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@click.group(cls=ClickDocOptGroup)
|
|
20
|
+
def internal():
|
|
21
|
+
"""Internal commands for use within pipelines or by Platform Team."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@internal.command()
|
|
25
|
+
def migrate_service_manifests():
|
|
26
|
+
"""Migrate copilot manifests to service manifests."""
|
|
27
|
+
click_io = ClickIOProvider()
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
service_manager = ServiceManager()
|
|
31
|
+
service_manager.migrate_copilot_manifests()
|
|
32
|
+
except PlatformException as error:
|
|
33
|
+
click_io.abort_with_error(str(error))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@internal.group(cls=ClickDocOptGroup)
|
|
37
|
+
def service():
|
|
38
|
+
"""Subgroup for 'internal service' commands."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@service.command(help="Trigger an ECS deployment.")
|
|
42
|
+
@click.option("--name", required=True, help="The name of the ECS service to create or update.")
|
|
43
|
+
@click.option(
|
|
44
|
+
"--env",
|
|
45
|
+
required=True,
|
|
46
|
+
help="The name of the environment where the ECS service will be created or updated.",
|
|
47
|
+
)
|
|
48
|
+
@click.option(
|
|
49
|
+
"--image-tag",
|
|
50
|
+
required=True,
|
|
51
|
+
help="Image tag to deploy for the service(s). Takes precedence over the $IMAGE_TAG environment variable.",
|
|
52
|
+
)
|
|
53
|
+
def deploy(name, env, image_tag):
|
|
54
|
+
"""Register a new ECS task definition from an S3 JSON template, update the
|
|
55
|
+
ECS service, and tail CloudWatch logs until the ECS rollout is complete."""
|
|
56
|
+
click_io = ClickIOProvider()
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
|
|
60
|
+
config = ConfigProvider(ConfigValidator()).get_enriched_config()
|
|
61
|
+
application_name = config.get("application", "")
|
|
62
|
+
application = load_application(app=application_name, env=env)
|
|
63
|
+
|
|
64
|
+
ecs_client = application.environments[env].session.client("ecs")
|
|
65
|
+
ssm_client = application.environments[env].session.client("ssm")
|
|
66
|
+
s3_client = application.environments[env].session.client("s3")
|
|
67
|
+
logs_client = application.environments[env].session.client("logs")
|
|
68
|
+
autoscaling_client = application.environments[env].session.client("application-autoscaling")
|
|
69
|
+
|
|
70
|
+
ecs_provider = ECS(
|
|
71
|
+
ecs_client=ecs_client,
|
|
72
|
+
ssm_client=ssm_client,
|
|
73
|
+
application_name=application.name,
|
|
74
|
+
env=env,
|
|
75
|
+
)
|
|
76
|
+
s3_provider = S3Provider(client=s3_client)
|
|
77
|
+
logs_provider = LogsProvider(client=logs_client)
|
|
78
|
+
autoscaling_provider = AutoscalingProvider(client=autoscaling_client)
|
|
79
|
+
|
|
80
|
+
service_manager = ServiceManager(
|
|
81
|
+
ecs_provider=ecs_provider,
|
|
82
|
+
s3_provider=s3_provider,
|
|
83
|
+
logs_provider=logs_provider,
|
|
84
|
+
autoscaling_provider=autoscaling_provider,
|
|
85
|
+
)
|
|
86
|
+
service_manager.deploy(
|
|
87
|
+
service=name,
|
|
88
|
+
environment=env,
|
|
89
|
+
application=application.name,
|
|
90
|
+
image_tag=image_tag,
|
|
91
|
+
)
|
|
92
|
+
except PlatformException as error:
|
|
93
|
+
click_io.abort_with_error(str(error))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@service.command(help="Generate Terraform manifest for the specified service(s).")
|
|
97
|
+
@click.option(
|
|
98
|
+
"--name",
|
|
99
|
+
required=False,
|
|
100
|
+
help="The name of the service(s) to generate service manifest(s) for.",
|
|
101
|
+
multiple=True,
|
|
102
|
+
)
|
|
103
|
+
@click.option(
|
|
104
|
+
"--env",
|
|
105
|
+
required=True,
|
|
106
|
+
help="The name of the environment to generate service manifests for.",
|
|
107
|
+
)
|
|
108
|
+
def generate(name, env):
|
|
109
|
+
"""Validates the service-config.yml format, applies the environment-specific
|
|
110
|
+
overrides, and generates a Terraform manifest at
|
|
111
|
+
/terraform/services/<environment>/<service>/main.tf.json."""
|
|
112
|
+
|
|
113
|
+
services = list(name)
|
|
114
|
+
click_io = ClickIOProvider()
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
service_manager = ServiceManager()
|
|
118
|
+
service_manager.generate(environment=env, services=services)
|
|
119
|
+
|
|
120
|
+
except PlatformException as err:
|
|
121
|
+
click_io.abort_with_error(str(err))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@internal.group(cls=ClickDocOptGroup)
|
|
125
|
+
def alb():
|
|
126
|
+
"""Load Balancer related commands."""
|
|
127
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@alb.command()
|
|
131
|
+
@click.option("--env", type=str, required=True)
|
|
132
|
+
def update_rules(env: str):
|
|
133
|
+
"""Update alb rules based on service-deployment-mode for a given
|
|
134
|
+
environment."""
|
|
135
|
+
try:
|
|
136
|
+
session = get_aws_session_or_abort()
|
|
137
|
+
update_aws = UpdateALBRules(session)
|
|
138
|
+
update_aws.update_alb_rules(environment=env)
|
|
139
|
+
except PlatformException as err:
|
|
140
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
@@ -6,9 +6,12 @@ from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
|
6
6
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
7
7
|
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
8
8
|
from dbt_platform_helper.providers.ecr import ECRProvider
|
|
9
|
+
from dbt_platform_helper.providers.environment_variable import (
|
|
10
|
+
EnvironmentVariableProvider,
|
|
11
|
+
)
|
|
12
|
+
from dbt_platform_helper.providers.files import FileProvider
|
|
9
13
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
10
14
|
from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
|
|
11
|
-
from dbt_platform_helper.utils.aws import get_codestar_connection_arn
|
|
12
15
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
13
16
|
from dbt_platform_helper.utils.git import git_remote
|
|
14
17
|
|
|
@@ -43,15 +46,24 @@ def generate(deploy_branch: str):
|
|
|
43
46
|
The `terraform/codebase-pipelines/main.tf.json` file is generated using this configuration.
|
|
44
47
|
The `main.tf.json` file is then used to generate Terraform for creating a codebase pipeline resource.
|
|
45
48
|
"""
|
|
49
|
+
config_provider = ConfigProvider(ConfigValidator())
|
|
50
|
+
environment_variable_provider = EnvironmentVariableProvider()
|
|
46
51
|
io = ClickIOProvider()
|
|
52
|
+
platform_helper_versioning = PlatformHelperVersioning(
|
|
53
|
+
io,
|
|
54
|
+
config_provider,
|
|
55
|
+
environment_variable_provider,
|
|
56
|
+
)
|
|
57
|
+
|
|
47
58
|
try:
|
|
48
59
|
pipelines = Pipelines(
|
|
49
|
-
|
|
60
|
+
config_provider,
|
|
50
61
|
TerraformManifestProvider(),
|
|
51
62
|
ECRProvider(),
|
|
52
63
|
git_remote,
|
|
53
|
-
get_codestar_connection_arn,
|
|
54
64
|
io,
|
|
65
|
+
FileProvider(),
|
|
66
|
+
platform_helper_versioning,
|
|
55
67
|
)
|
|
56
68
|
pipelines.generate(deploy_branch)
|
|
57
69
|
except Exception as exc:
|
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
|
|
3
|
-
from pathlib import Path
|
|
4
3
|
|
|
5
4
|
import click
|
|
6
|
-
from botocore.exceptions import ClientError
|
|
7
|
-
from cloudfoundry_client.client import CloudFoundryClient
|
|
8
5
|
|
|
6
|
+
from dbt_platform_helper.domain.secrets import Secrets
|
|
9
7
|
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
10
|
-
from dbt_platform_helper.
|
|
11
|
-
from dbt_platform_helper.
|
|
12
|
-
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
13
|
-
from dbt_platform_helper.utils.aws import get_ssm_secrets
|
|
14
|
-
from dbt_platform_helper.utils.aws import set_ssm_param
|
|
8
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
9
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
15
10
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
16
11
|
|
|
17
12
|
|
|
@@ -19,100 +14,53 @@ def secret_should_be_skipped(secret_name):
|
|
|
19
14
|
return "AWS_" in secret_name
|
|
20
15
|
|
|
21
16
|
|
|
22
|
-
def get_paas_env_vars(client: CloudFoundryClient, paas: str) -> dict:
|
|
23
|
-
org, space, app = paas.split("/")
|
|
24
|
-
|
|
25
|
-
env_vars = None
|
|
26
|
-
|
|
27
|
-
for paas_org in client.v2.organizations:
|
|
28
|
-
if paas_org["entity"]["name"] == org:
|
|
29
|
-
for paas_space in paas_org.spaces():
|
|
30
|
-
if paas_space["entity"]["name"] == space:
|
|
31
|
-
for paas_app in paas_space.apps():
|
|
32
|
-
if paas_app["entity"]["name"] == app:
|
|
33
|
-
env_vars = paas_app["entity"]["environment_json"]
|
|
34
|
-
|
|
35
|
-
if not env_vars:
|
|
36
|
-
raise Exception(f"Application {paas} not found")
|
|
37
|
-
|
|
38
|
-
return dict(env_vars)
|
|
39
|
-
|
|
40
|
-
|
|
41
17
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
42
18
|
def secrets():
|
|
43
19
|
PlatformHelperVersioning().check_if_needs_update()
|
|
44
20
|
|
|
45
21
|
|
|
46
22
|
@secrets.command()
|
|
47
|
-
@click.
|
|
48
|
-
@click.
|
|
49
|
-
@click.option(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if secret_should_be_skipped(secret_name):
|
|
65
|
-
continue
|
|
66
|
-
|
|
67
|
-
click.echo(secret_name)
|
|
68
|
-
|
|
69
|
-
try:
|
|
70
|
-
set_ssm_param(
|
|
71
|
-
app_name,
|
|
72
|
-
target_environment,
|
|
73
|
-
secret_name,
|
|
74
|
-
secret[1],
|
|
75
|
-
False,
|
|
76
|
-
False,
|
|
77
|
-
f"Copied from {source_environment} environment.",
|
|
78
|
-
)
|
|
79
|
-
except ClientError as e:
|
|
80
|
-
if e.response["Error"]["Code"] == "ParameterAlreadyExists":
|
|
81
|
-
click.secho(
|
|
82
|
-
f"""The "{secret_name.split("/")[-1]}" parameter already exists for the "{target_environment}" environment.""",
|
|
83
|
-
fg="yellow",
|
|
84
|
-
)
|
|
85
|
-
else:
|
|
86
|
-
raise e
|
|
23
|
+
@click.option("--app", help="Application name.", required=True)
|
|
24
|
+
@click.option("--name", help="Secret name (automatically uppercased).", required=True)
|
|
25
|
+
@click.option(
|
|
26
|
+
"--overwrite",
|
|
27
|
+
is_flag=True,
|
|
28
|
+
default=False,
|
|
29
|
+
help="Allows overwriting the value of secrets if they already exist.",
|
|
30
|
+
)
|
|
31
|
+
def create(app: str, name: str, overwrite: bool):
|
|
32
|
+
"""Create a Parameter Store secret for all environments of an
|
|
33
|
+
application."""
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
Secrets().create(app, name, overwrite)
|
|
37
|
+
except PlatformException as err:
|
|
38
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
87
39
|
|
|
88
40
|
|
|
89
41
|
@secrets.command()
|
|
90
|
-
@click.
|
|
91
|
-
@click.
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
session = get_aws_session_or_abort()
|
|
96
|
-
client = session.client("ssm")
|
|
97
|
-
|
|
98
|
-
path = SSM_BASE_PATH.format(app=app, env=env)
|
|
99
|
-
|
|
100
|
-
params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
|
|
101
|
-
secrets = []
|
|
42
|
+
@click.option("--app", help="Application name.", required=True)
|
|
43
|
+
@click.option("--source", help="Source environment where to copy secrets from.", required=True)
|
|
44
|
+
@click.option("--target", help="Destination environment where to copy secrets to.", required=True)
|
|
45
|
+
def copy(app, source, target):
|
|
46
|
+
"""Copy secrets from one environment to another."""
|
|
102
47
|
|
|
103
|
-
|
|
104
|
-
|
|
48
|
+
try:
|
|
49
|
+
Secrets().copy(app, source, target)
|
|
50
|
+
except PlatformException as err:
|
|
51
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
105
52
|
|
|
106
|
-
for secret in response["Parameters"]:
|
|
107
|
-
secrets.append(f"{secret['Name']:<8}: {secret['Value']:<15}")
|
|
108
53
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
54
|
+
@secrets.command()
|
|
55
|
+
@click.argument("app", type=str, required=True)
|
|
56
|
+
@click.argument("env", type=str, required=True)
|
|
57
|
+
def list(app, env):
|
|
58
|
+
"""[DELETED] List secret names and values for an environment."""
|
|
113
59
|
|
|
114
|
-
|
|
115
|
-
|
|
60
|
+
click.secho(
|
|
61
|
+
message="\nThis command has been removed to prevent accidental exposure of secret values in local terminals and logs. To view secrets, log into your AWS account and head over to AWS Parameter Store https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/\n",
|
|
62
|
+
fg="magenta",
|
|
63
|
+
)
|
|
116
64
|
|
|
117
65
|
|
|
118
66
|
if __name__ == "__main__":
|
|
@@ -25,7 +25,8 @@ def get_platform_helper_for_project(pipeline):
|
|
|
25
25
|
|
|
26
26
|
Version preferences removed, requires specifying platform-helper version in platform-config.yml
|
|
27
27
|
"""
|
|
28
|
+
io = ClickIOProvider()
|
|
28
29
|
try:
|
|
29
|
-
PlatformHelperVersioning().get_required_version()
|
|
30
|
+
io.info(PlatformHelperVersioning().get_required_version())
|
|
30
31
|
except PlatformException as err:
|
|
31
|
-
|
|
32
|
+
io.abort_with_error(str(err))
|
dbt_platform_helper/constants.py
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# TODO: DBTP-1888: Move to Config provider
|
|
2
|
+
REPO_ORG = "uktrade"
|
|
2
3
|
PLATFORM_CONFIG_FILE = "platform-config.yml"
|
|
4
|
+
SERVICE_DIRECTORY = "services"
|
|
5
|
+
SERVICE_CONFIG_FILE = "service-config.yml"
|
|
6
|
+
ENVIRONMENT_PIPELINE_MODULE_PATH = (
|
|
7
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref="
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
CODEBASE_PIPELINE_MODULE_PATH = (
|
|
11
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref="
|
|
12
|
+
)
|
|
13
|
+
EXTENSIONS_MODULE_PATH = (
|
|
14
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/extensions?depth=1&ref="
|
|
15
|
+
)
|
|
3
16
|
PLATFORM_HELPER_VERSION_OVERRIDE_KEY = "PLATFORM_HELPER_VERSION_OVERRIDE"
|
|
4
17
|
TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR = "TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE"
|
|
5
18
|
TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
@@ -8,13 +21,19 @@ TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
|
8
21
|
TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
9
22
|
"TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE"
|
|
10
23
|
)
|
|
24
|
+
TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
25
|
+
"TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE"
|
|
26
|
+
)
|
|
27
|
+
TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR = "TERRAFORM_MODULE_SOURCE_TYPE" # "LOCAL", "SSH", "OVERRIDE"
|
|
28
|
+
IMAGE_TAG_ENV_VAR = "IMAGE_TAG"
|
|
11
29
|
PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
|
|
12
|
-
SUPPORTED_TERRAFORM_VERSION = "~> 1.
|
|
13
|
-
SUPPORTED_AWS_PROVIDER_VERSION = "~>
|
|
30
|
+
SUPPORTED_TERRAFORM_VERSION = "~> 1.11"
|
|
31
|
+
SUPPORTED_AWS_PROVIDER_VERSION = "~> 6"
|
|
14
32
|
FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR_VERSION = 13
|
|
15
33
|
|
|
16
34
|
MERGED_TPM_PLATFORM_HELPER_VERSION = 14
|
|
17
35
|
PLATFORM_CONFIG_SCHEMA_VERSION = 1
|
|
36
|
+
SERVICE_CONFIG_SCHEMA_VERSION = 1
|
|
18
37
|
|
|
19
38
|
# Keys
|
|
20
39
|
CODEBASE_PIPELINES_KEY = "codebase_pipelines"
|
|
@@ -37,3 +56,20 @@ SERVICE_NAME_SUFFIX = f"Service-{COPILOT_IDENTIFIER}"
|
|
|
37
56
|
REFRESH_TOKEN_MESSAGE = (
|
|
38
57
|
"To refresh this SSO session run `aws sso login` with the corresponding profile"
|
|
39
58
|
)
|
|
59
|
+
COPILOT_RULE_PRIORITY = 48000
|
|
60
|
+
PLATFORM_RULE_STARTING_PRIORITY = 10000
|
|
61
|
+
RULE_PRIORITY_INCREMENT = 100
|
|
62
|
+
DUMMY_RULE_REASON = "DummyRule"
|
|
63
|
+
MAINTENANCE_PAGE_TAGS = ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]
|
|
64
|
+
MAINTENANCE_PAGE_REASON = "MaintenancePage"
|
|
65
|
+
MANAGED_BY_PLATFORM = "DBT Platform"
|
|
66
|
+
MANAGED_BY_SERVICE_TERRAFORM = "DBT Platform - Service Terraform"
|
|
67
|
+
MANAGED_BY_PLATFORM_TERRAFORM = "DBT Platform - Terraform"
|
|
68
|
+
STANDARD_PLATFORM_SSO_ROLES = [
|
|
69
|
+
"AdministratorAccess",
|
|
70
|
+
"DBTPlatformDeveloperWrite",
|
|
71
|
+
"DBTPlatformDeveloperRead",
|
|
72
|
+
]
|
|
73
|
+
ROUTED_TO_PLATFORM_MODES = ["platform", "dual-deploy-platform-traffic"]
|
|
74
|
+
ROUTED_TO_COPILOT_MODES = ["copilot", "dual-deploy-copilot-traffic"]
|
|
75
|
+
HTTP_SERVICE_TYPES = ["Load Balanced Web Service"]
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from abc import ABC
|
|
2
3
|
from abc import abstractmethod
|
|
3
4
|
from typing import Callable
|
|
@@ -56,10 +57,16 @@ class TerraformConduitStrategy(ConduitECSStrategy):
|
|
|
56
57
|
|
|
57
58
|
def get_data(self):
|
|
58
59
|
self.io.info("Starting conduit in Terraform mode.")
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
try:
|
|
61
|
+
cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
|
|
62
|
+
f"{self.application.name}-{self.env}-cluster"
|
|
63
|
+
)
|
|
64
|
+
except:
|
|
65
|
+
cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
|
|
61
66
|
f"{self.application.name}-{self.env}"
|
|
62
|
-
)
|
|
67
|
+
)
|
|
68
|
+
return {
|
|
69
|
+
"cluster_arn": cluster_arn,
|
|
63
70
|
"task_def_family": self._generate_container_name(),
|
|
64
71
|
"vpc_name": self._resolve_vpc_name(),
|
|
65
72
|
"addon_type": self.addon_type,
|
|
@@ -94,8 +101,10 @@ class TerraformConduitStrategy(ConduitECSStrategy):
|
|
|
94
101
|
},
|
|
95
102
|
]
|
|
96
103
|
|
|
104
|
+
cluster_name = data_context["cluster_arn"].split("/")[-1]
|
|
105
|
+
|
|
97
106
|
self.ecs_provider.start_ecs_task(
|
|
98
|
-
|
|
107
|
+
cluster_name,
|
|
99
108
|
self._generate_container_name(),
|
|
100
109
|
data_context["task_def_family"],
|
|
101
110
|
vpc_config,
|
|
@@ -110,14 +119,18 @@ class TerraformConduitStrategy(ConduitECSStrategy):
|
|
|
110
119
|
|
|
111
120
|
def _resolve_vpc_name(self):
|
|
112
121
|
ssm_client = self.clients["ssm"]
|
|
113
|
-
parameter_key = f"/
|
|
122
|
+
parameter_key = f"/platform/applications/{self.application.name}/environments/{self.env}"
|
|
114
123
|
|
|
115
124
|
try:
|
|
116
|
-
response = ssm_client.get_parameter(Name=parameter_key)
|
|
117
|
-
return response["
|
|
125
|
+
response = ssm_client.get_parameter(Name=parameter_key)["Parameter"]["Value"]
|
|
126
|
+
return json.loads(response)["vpc_name"]
|
|
118
127
|
except ssm_client.exceptions.ParameterNotFound:
|
|
119
128
|
self.io.abort_with_error(
|
|
120
|
-
f"Could not find
|
|
129
|
+
f"Could not find AWS SSM parameter {parameter_key}. Please ensure your environment Terraform is up to date."
|
|
130
|
+
)
|
|
131
|
+
except KeyError:
|
|
132
|
+
self.io.abort_with_error(
|
|
133
|
+
f"The parameter {parameter_key} exists but does not contain the 'vpc_name' field. Please ensure your environment Terraform is up to date."
|
|
121
134
|
)
|
|
122
135
|
|
|
123
136
|
|
|
@@ -326,7 +339,7 @@ class Conduit:
|
|
|
326
339
|
data_context = strategy.get_data()
|
|
327
340
|
|
|
328
341
|
data_context["task_arns"] = self.ecs_provider.get_ecs_task_arns(
|
|
329
|
-
data_context["cluster_arn"], data_context["task_def_family"]
|
|
342
|
+
cluster=data_context["cluster_arn"], task_def_family=data_context["task_def_family"]
|
|
330
343
|
)
|
|
331
344
|
|
|
332
345
|
info_log = (
|