dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +107 -27
- dbt_platform_helper/commands/application.py +5 -6
- dbt_platform_helper/commands/codebase.py +31 -10
- dbt_platform_helper/commands/conduit.py +3 -5
- dbt_platform_helper/commands/config.py +20 -311
- dbt_platform_helper/commands/copilot.py +18 -391
- dbt_platform_helper/commands/database.py +17 -9
- dbt_platform_helper/commands/environment.py +20 -14
- dbt_platform_helper/commands/generate.py +0 -3
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/notify.py +58 -78
- dbt_platform_helper/commands/pipeline.py +23 -19
- dbt_platform_helper/commands/secrets.py +39 -93
- dbt_platform_helper/commands/version.py +7 -12
- dbt_platform_helper/constants.py +52 -7
- dbt_platform_helper/domain/codebase.py +89 -39
- dbt_platform_helper/domain/conduit.py +335 -76
- dbt_platform_helper/domain/config.py +381 -0
- dbt_platform_helper/domain/copilot.py +398 -0
- dbt_platform_helper/domain/copilot_environment.py +8 -8
- dbt_platform_helper/domain/database_copy.py +2 -2
- dbt_platform_helper/domain/maintenance_page.py +254 -430
- dbt_platform_helper/domain/notify.py +64 -0
- dbt_platform_helper/domain/pipelines.py +43 -35
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +14 -13
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +249 -0
- dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
- dbt_platform_helper/entities/semantic_version.py +83 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/__init__.py +0 -0
- dbt_platform_helper/providers/aws/exceptions.py +70 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/aws/sso_auth.py +75 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/cloudformation.py +1 -1
- dbt_platform_helper/providers/config.py +137 -19
- dbt_platform_helper/providers/config_validator.py +112 -51
- dbt_platform_helper/providers/copilot.py +24 -16
- dbt_platform_helper/providers/ecr.py +89 -7
- dbt_platform_helper/providers/ecs.py +228 -36
- dbt_platform_helper/providers/environment_variable.py +24 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +36 -4
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +402 -42
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +134 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
- dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
- dbt_platform_helper/providers/schema_migrator.py +77 -0
- dbt_platform_helper/providers/secrets.py +5 -5
- dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
- dbt_platform_helper/providers/terraform_manifest.py +121 -19
- dbt_platform_helper/providers/version.py +106 -23
- dbt_platform_helper/providers/version_status.py +27 -0
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +58 -2
- dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utilities/decorators.py +103 -0
- dbt_platform_helper/utils/application.py +119 -22
- dbt_platform_helper/utils/aws.py +39 -150
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -14
- dbt_platform_helper/utils/validation.py +1 -1
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
- dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +3 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- dbt_platform_helper/providers/aws.py +0 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/providers/semantic_version.py +0 -126
- dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
- dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
- dbt_platform_helper/utils/cloudfoundry.py +0 -14
- dbt_platform_helper/utils/files.py +0 -53
- dbt_platform_helper/utils/manifests.py +0 -18
- dbt_platform_helper/utils/versioning.py +0 -238
- dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from dbt_platform_helper.domain.service import ServiceManager
|
|
4
|
+
from dbt_platform_helper.domain.update_alb_rules import UpdateALBRules
|
|
5
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
6
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
7
|
+
from dbt_platform_helper.providers.autoscaling import AutoscalingProvider
|
|
8
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
9
|
+
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
10
|
+
from dbt_platform_helper.providers.ecs import ECS
|
|
11
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
12
|
+
from dbt_platform_helper.providers.logs import LogsProvider
|
|
13
|
+
from dbt_platform_helper.providers.s3 import S3Provider
|
|
14
|
+
from dbt_platform_helper.utils.application import load_application
|
|
15
|
+
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
16
|
+
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@click.group(cls=ClickDocOptGroup)
|
|
20
|
+
def internal():
|
|
21
|
+
"""Internal commands for use within pipelines or by Platform Team."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@internal.command()
|
|
25
|
+
def migrate_service_manifests():
|
|
26
|
+
"""Migrate copilot manifests to service manifests."""
|
|
27
|
+
click_io = ClickIOProvider()
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
service_manager = ServiceManager()
|
|
31
|
+
service_manager.migrate_copilot_manifests()
|
|
32
|
+
except PlatformException as error:
|
|
33
|
+
click_io.abort_with_error(str(error))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@internal.group(cls=ClickDocOptGroup)
|
|
37
|
+
def service():
|
|
38
|
+
"""Subgroup for 'internal service' commands."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@service.command(help="Trigger an ECS deployment.")
|
|
42
|
+
@click.option("--name", required=True, help="The name of the ECS service to create or update.")
|
|
43
|
+
@click.option(
|
|
44
|
+
"--env",
|
|
45
|
+
required=True,
|
|
46
|
+
help="The name of the environment where the ECS service will be created or updated.",
|
|
47
|
+
)
|
|
48
|
+
@click.option(
|
|
49
|
+
"--image-tag",
|
|
50
|
+
required=True,
|
|
51
|
+
help="Image tag to deploy for the service(s). Takes precedence over the $IMAGE_TAG environment variable.",
|
|
52
|
+
)
|
|
53
|
+
def deploy(name, env, image_tag):
|
|
54
|
+
"""Register a new ECS task definition from an S3 JSON template, update the
|
|
55
|
+
ECS service, and tail CloudWatch logs until the ECS rollout is complete."""
|
|
56
|
+
click_io = ClickIOProvider()
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
|
|
60
|
+
config = ConfigProvider(ConfigValidator()).get_enriched_config()
|
|
61
|
+
application_name = config.get("application", "")
|
|
62
|
+
application = load_application(app=application_name, env=env)
|
|
63
|
+
|
|
64
|
+
ecs_client = application.environments[env].session.client("ecs")
|
|
65
|
+
ssm_client = application.environments[env].session.client("ssm")
|
|
66
|
+
s3_client = application.environments[env].session.client("s3")
|
|
67
|
+
logs_client = application.environments[env].session.client("logs")
|
|
68
|
+
autoscaling_client = application.environments[env].session.client("application-autoscaling")
|
|
69
|
+
|
|
70
|
+
ecs_provider = ECS(
|
|
71
|
+
ecs_client=ecs_client,
|
|
72
|
+
ssm_client=ssm_client,
|
|
73
|
+
application_name=application.name,
|
|
74
|
+
env=env,
|
|
75
|
+
)
|
|
76
|
+
s3_provider = S3Provider(client=s3_client)
|
|
77
|
+
logs_provider = LogsProvider(client=logs_client)
|
|
78
|
+
autoscaling_provider = AutoscalingProvider(client=autoscaling_client)
|
|
79
|
+
|
|
80
|
+
service_manager = ServiceManager(
|
|
81
|
+
ecs_provider=ecs_provider,
|
|
82
|
+
s3_provider=s3_provider,
|
|
83
|
+
logs_provider=logs_provider,
|
|
84
|
+
autoscaling_provider=autoscaling_provider,
|
|
85
|
+
)
|
|
86
|
+
service_manager.deploy(
|
|
87
|
+
service=name,
|
|
88
|
+
environment=env,
|
|
89
|
+
application=application.name,
|
|
90
|
+
image_tag=image_tag,
|
|
91
|
+
)
|
|
92
|
+
except PlatformException as error:
|
|
93
|
+
click_io.abort_with_error(str(error))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@service.command(help="Generate Terraform manifest for the specified service(s).")
|
|
97
|
+
@click.option(
|
|
98
|
+
"--name",
|
|
99
|
+
required=False,
|
|
100
|
+
help="The name of the service(s) to generate service manifest(s) for.",
|
|
101
|
+
multiple=True,
|
|
102
|
+
)
|
|
103
|
+
@click.option(
|
|
104
|
+
"--env",
|
|
105
|
+
required=True,
|
|
106
|
+
help="The name of the environment to generate service manifests for.",
|
|
107
|
+
)
|
|
108
|
+
def generate(name, env):
|
|
109
|
+
"""Validates the service-config.yml format, applies the environment-specific
|
|
110
|
+
overrides, and generates a Terraform manifest at
|
|
111
|
+
/terraform/services/<environment>/<service>/main.tf.json."""
|
|
112
|
+
|
|
113
|
+
services = list(name)
|
|
114
|
+
click_io = ClickIOProvider()
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
service_manager = ServiceManager()
|
|
118
|
+
service_manager.generate(environment=env, services=services)
|
|
119
|
+
|
|
120
|
+
except PlatformException as err:
|
|
121
|
+
click_io.abort_with_error(str(err))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@internal.group(cls=ClickDocOptGroup)
|
|
125
|
+
def alb():
|
|
126
|
+
"""Load Balancer related commands."""
|
|
127
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@alb.command()
|
|
131
|
+
@click.option("--env", type=str, required=True)
|
|
132
|
+
def update_rules(env: str):
|
|
133
|
+
"""Update alb rules based on service-deployment-mode for a given
|
|
134
|
+
environment."""
|
|
135
|
+
try:
|
|
136
|
+
session = get_aws_session_or_abort()
|
|
137
|
+
update_aws = UpdateALBRules(session)
|
|
138
|
+
update_aws.update_alb_rules(environment=env)
|
|
139
|
+
except PlatformException as err:
|
|
140
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import click
|
|
2
|
-
from slack_sdk import WebClient
|
|
3
|
-
from slack_sdk.models import blocks
|
|
4
2
|
|
|
5
|
-
from dbt_platform_helper.
|
|
3
|
+
from dbt_platform_helper.domain.notify import Notify
|
|
4
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
5
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
6
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
7
|
+
from dbt_platform_helper.providers.slack_channel_notifier import SlackChannelNotifier
|
|
6
8
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
7
|
-
from dbt_platform_helper.utils.versioning import (
|
|
8
|
-
check_platform_helper_version_needs_update,
|
|
9
|
-
)
|
|
10
9
|
|
|
11
10
|
|
|
12
11
|
@click.group(cls=ClickDocOptGroup, help="Send Slack notifications")
|
|
13
12
|
def notify():
|
|
14
|
-
|
|
13
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
@notify.command(
|
|
18
|
-
help="Send environment progress notifications. This creates (or updates if --slack-ref is provided) the top level message to the channel."
|
|
17
|
+
help="Send environment progress notifications. This creates (or updates if --slack-ref is provided) the top level message to the channel.",
|
|
18
|
+
deprecated=True,
|
|
19
19
|
)
|
|
20
20
|
@click.argument("slack-channel-id")
|
|
21
21
|
@click.argument("slack-token")
|
|
@@ -24,7 +24,9 @@ def notify():
|
|
|
24
24
|
@click.option("--repository")
|
|
25
25
|
@click.option("--commit-sha")
|
|
26
26
|
@click.option("--slack-ref", help="Slack message reference of the message to update")
|
|
27
|
+
@click.pass_context
|
|
27
28
|
def environment_progress(
|
|
29
|
+
ctx,
|
|
28
30
|
slack_channel_id: str,
|
|
29
31
|
slack_token: str,
|
|
30
32
|
message: str,
|
|
@@ -33,56 +35,52 @@ def environment_progress(
|
|
|
33
35
|
commit_sha: str,
|
|
34
36
|
slack_ref: str,
|
|
35
37
|
):
|
|
36
|
-
args = _get_slack_args(build_arn, commit_sha, message, repository, slack_channel_id)
|
|
37
|
-
slack = _get_slack_client(slack_token)
|
|
38
|
-
|
|
39
|
-
if slack_ref:
|
|
40
|
-
response = slack.chat_update(ts=slack_ref, **args)
|
|
41
|
-
else:
|
|
42
|
-
response = slack.chat_postMessage(ts=slack_ref, **args)
|
|
43
38
|
|
|
44
|
-
|
|
39
|
+
ctx.invoke(
|
|
40
|
+
post_message,
|
|
41
|
+
slack_channel_id=slack_channel_id,
|
|
42
|
+
slack_token=slack_token,
|
|
43
|
+
message=message,
|
|
44
|
+
build_arn=build_arn,
|
|
45
|
+
repository=repository,
|
|
46
|
+
commit_sha=commit_sha,
|
|
47
|
+
slack_ref=slack_ref,
|
|
48
|
+
)
|
|
45
49
|
|
|
46
50
|
|
|
47
|
-
|
|
48
|
-
|
|
51
|
+
@notify.command(
|
|
52
|
+
help="Send Slack notifications. This creates (or updates if --slack-ref is provided) the top level message to the channel."
|
|
53
|
+
)
|
|
54
|
+
@click.argument("slack-channel-id")
|
|
55
|
+
@click.argument("slack-token")
|
|
56
|
+
@click.argument("message")
|
|
57
|
+
@click.option("--build-arn")
|
|
58
|
+
@click.option("--repository")
|
|
59
|
+
@click.option("--commit-sha")
|
|
60
|
+
@click.option("--slack-ref", help="Slack message reference of the message to update")
|
|
61
|
+
def post_message(
|
|
62
|
+
slack_channel_id: str,
|
|
63
|
+
slack_token: str,
|
|
64
|
+
message: str,
|
|
65
|
+
build_arn: str,
|
|
66
|
+
repository: str,
|
|
67
|
+
commit_sha: str,
|
|
68
|
+
slack_ref: str,
|
|
49
69
|
):
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
message_blocks = [
|
|
60
|
-
blocks.SectionBlock(
|
|
61
|
-
text=blocks.TextObject(type="mrkdwn", text=message),
|
|
62
|
-
),
|
|
63
|
-
]
|
|
64
|
-
|
|
65
|
-
if context_elements:
|
|
66
|
-
message_blocks.append(
|
|
67
|
-
blocks.ContextBlock(
|
|
68
|
-
elements=[
|
|
69
|
-
blocks.TextObject(type="mrkdwn", text=element) for element in context_elements
|
|
70
|
-
]
|
|
71
|
-
)
|
|
70
|
+
try:
|
|
71
|
+
io = ClickIOProvider()
|
|
72
|
+
slack_notifier = SlackChannelNotifier(slack_token, slack_channel_id)
|
|
73
|
+
result = Notify(slack_notifier).post_message(
|
|
74
|
+
original_message_ref=slack_ref,
|
|
75
|
+
message=message,
|
|
76
|
+
build_arn=build_arn,
|
|
77
|
+
repository=repository,
|
|
78
|
+
commit_sha=commit_sha,
|
|
72
79
|
)
|
|
73
80
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
"text": message,
|
|
78
|
-
"unfurl_links": False,
|
|
79
|
-
"unfurl_media": False,
|
|
80
|
-
}
|
|
81
|
-
return args
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def _get_slack_client(token: str):
|
|
85
|
-
return WebClient(token=token)
|
|
81
|
+
io.info(result)
|
|
82
|
+
except PlatformException as err:
|
|
83
|
+
io.abort_with_error(str(err))
|
|
86
84
|
|
|
87
85
|
|
|
88
86
|
@notify.command(help="Add a comment to an existing Slack message")
|
|
@@ -100,31 +98,13 @@ def add_comment(
|
|
|
100
98
|
title: str,
|
|
101
99
|
send_to_main_channel: bool,
|
|
102
100
|
):
|
|
103
|
-
slack = _get_slack_client(slack_token)
|
|
104
|
-
|
|
105
|
-
slack.chat_postMessage(
|
|
106
|
-
channel=slack_channel_id,
|
|
107
|
-
blocks=[blocks.SectionBlock(text=blocks.TextObject(type="mrkdwn", text=message))],
|
|
108
|
-
text=title if title else message,
|
|
109
|
-
reply_broadcast=send_to_main_channel,
|
|
110
|
-
unfurl_links=False,
|
|
111
|
-
unfurl_media=False,
|
|
112
|
-
thread_ts=slack_ref,
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
def get_build_url(build_arn: str):
|
|
117
101
|
try:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
region=arn.region,
|
|
125
|
-
account=arn.account_id,
|
|
126
|
-
project=arn.project.replace("build/", ""),
|
|
127
|
-
build_id=arn.build_id,
|
|
102
|
+
slack_notifier = SlackChannelNotifier(slack_token, slack_channel_id)
|
|
103
|
+
Notify(slack_notifier).add_comment(
|
|
104
|
+
message=message,
|
|
105
|
+
title=title,
|
|
106
|
+
reply_broadcast=send_to_main_channel,
|
|
107
|
+
original_message_ref=slack_ref,
|
|
128
108
|
)
|
|
129
|
-
except
|
|
130
|
-
|
|
109
|
+
except PlatformException as err:
|
|
110
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
@@ -1,35 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
import click
|
|
3
3
|
|
|
4
|
-
from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
|
|
5
4
|
from dbt_platform_helper.domain.pipelines import Pipelines
|
|
5
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
6
6
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
7
7
|
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
8
8
|
from dbt_platform_helper.providers.ecr import ECRProvider
|
|
9
|
+
from dbt_platform_helper.providers.environment_variable import (
|
|
10
|
+
EnvironmentVariableProvider,
|
|
11
|
+
)
|
|
12
|
+
from dbt_platform_helper.providers.files import FileProvider
|
|
9
13
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
10
14
|
from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
|
|
11
|
-
from dbt_platform_helper.utils.aws import get_codestar_connection_arn
|
|
12
15
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
13
16
|
from dbt_platform_helper.utils.git import git_remote
|
|
14
|
-
from dbt_platform_helper.utils.versioning import (
|
|
15
|
-
check_platform_helper_version_needs_update,
|
|
16
|
-
)
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
20
20
|
def pipeline():
|
|
21
21
|
"""Pipeline commands."""
|
|
22
|
-
|
|
22
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
@pipeline.command()
|
|
26
|
-
@click.option(
|
|
27
|
-
"--terraform-platform-modules-version",
|
|
28
|
-
help=f"""Override the default version of terraform-platform-modules with a specific version or branch.
|
|
29
|
-
Precedence of version used is version supplied via CLI, then the version found in
|
|
30
|
-
platform-config.yml/default_versions/terraform-platform-modules.
|
|
31
|
-
In absence of these inputs, defaults to version '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}'.""",
|
|
32
|
-
)
|
|
33
26
|
@click.option(
|
|
34
27
|
"--deploy-branch",
|
|
35
28
|
help="""Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource.
|
|
@@ -38,29 +31,40 @@ def pipeline():
|
|
|
38
31
|
<application>-deploy/platform-config.yml/environment_pipelines/<environment-pipeline>/branch).""",
|
|
39
32
|
default=None,
|
|
40
33
|
)
|
|
41
|
-
def generate(
|
|
34
|
+
def generate(deploy_branch: str):
|
|
42
35
|
"""
|
|
43
36
|
Given a platform-config.yml file, generate environment and service
|
|
44
37
|
deployment pipelines.
|
|
45
38
|
|
|
46
39
|
This command does the following in relation to the environment pipelines:
|
|
47
|
-
- Reads contents of `platform-config.yml/
|
|
40
|
+
- Reads contents of `platform-config.yml/environment_pipelines` configuration.
|
|
48
41
|
The `terraform/environment-pipelines/<aws_account>/main.tf` file is generated using this configuration.
|
|
49
42
|
The `main.tf` file is then used to generate Terraform for creating an environment pipeline resource.
|
|
50
43
|
|
|
51
44
|
This command does the following in relation to the codebase pipelines:
|
|
52
|
-
-
|
|
45
|
+
- Reads contents of `platform-config.yml/codebase_pipelines` configuration.
|
|
46
|
+
The `terraform/codebase-pipelines/main.tf.json` file is generated using this configuration.
|
|
47
|
+
The `main.tf.json` file is then used to generate Terraform for creating a codebase pipeline resource.
|
|
53
48
|
"""
|
|
49
|
+
config_provider = ConfigProvider(ConfigValidator())
|
|
50
|
+
environment_variable_provider = EnvironmentVariableProvider()
|
|
54
51
|
io = ClickIOProvider()
|
|
52
|
+
platform_helper_versioning = PlatformHelperVersioning(
|
|
53
|
+
io,
|
|
54
|
+
config_provider,
|
|
55
|
+
environment_variable_provider,
|
|
56
|
+
)
|
|
57
|
+
|
|
55
58
|
try:
|
|
56
59
|
pipelines = Pipelines(
|
|
57
|
-
|
|
60
|
+
config_provider,
|
|
58
61
|
TerraformManifestProvider(),
|
|
59
62
|
ECRProvider(),
|
|
60
63
|
git_remote,
|
|
61
|
-
get_codestar_connection_arn,
|
|
62
64
|
io,
|
|
65
|
+
FileProvider(),
|
|
66
|
+
platform_helper_versioning,
|
|
63
67
|
)
|
|
64
|
-
pipelines.generate(
|
|
68
|
+
pipelines.generate(deploy_branch)
|
|
65
69
|
except Exception as exc:
|
|
66
70
|
io.abort_with_error(str(exc))
|
|
@@ -1,120 +1,66 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
|
|
3
|
-
from pathlib import Path
|
|
4
3
|
|
|
5
4
|
import click
|
|
6
|
-
|
|
7
|
-
from
|
|
8
|
-
|
|
9
|
-
from dbt_platform_helper.
|
|
10
|
-
from dbt_platform_helper.
|
|
11
|
-
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
12
|
-
from dbt_platform_helper.utils.aws import get_ssm_secrets
|
|
13
|
-
from dbt_platform_helper.utils.aws import set_ssm_param
|
|
5
|
+
|
|
6
|
+
from dbt_platform_helper.domain.secrets import Secrets
|
|
7
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
8
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
9
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
14
10
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
15
|
-
from dbt_platform_helper.utils.versioning import (
|
|
16
|
-
check_platform_helper_version_needs_update,
|
|
17
|
-
)
|
|
18
11
|
|
|
19
12
|
|
|
20
13
|
def secret_should_be_skipped(secret_name):
|
|
21
14
|
return "AWS_" in secret_name
|
|
22
15
|
|
|
23
16
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
env_vars = None
|
|
28
|
-
|
|
29
|
-
for paas_org in client.v2.organizations:
|
|
30
|
-
if paas_org["entity"]["name"] == org:
|
|
31
|
-
for paas_space in paas_org.spaces():
|
|
32
|
-
if paas_space["entity"]["name"] == space:
|
|
33
|
-
for paas_app in paas_space.apps():
|
|
34
|
-
if paas_app["entity"]["name"] == app:
|
|
35
|
-
env_vars = paas_app["entity"]["environment_json"]
|
|
36
|
-
|
|
37
|
-
if not env_vars:
|
|
38
|
-
raise Exception(f"Application {paas} not found")
|
|
17
|
+
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
18
|
+
def secrets():
|
|
19
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
39
20
|
|
|
40
|
-
return dict(env_vars)
|
|
41
21
|
|
|
22
|
+
@secrets.command()
|
|
23
|
+
@click.option("--app", help="Application name.", required=True)
|
|
24
|
+
@click.option("--name", help="Secret name (automatically uppercased).", required=True)
|
|
25
|
+
@click.option(
|
|
26
|
+
"--overwrite",
|
|
27
|
+
is_flag=True,
|
|
28
|
+
default=False,
|
|
29
|
+
help="Allows overwriting the value of secrets if they already exist.",
|
|
30
|
+
)
|
|
31
|
+
def create(app: str, name: str, overwrite: bool):
|
|
32
|
+
"""Create a Parameter Store secret for all environments of an
|
|
33
|
+
application."""
|
|
42
34
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
35
|
+
try:
|
|
36
|
+
Secrets().create(app, name, overwrite)
|
|
37
|
+
except PlatformException as err:
|
|
38
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
46
39
|
|
|
47
40
|
|
|
48
41
|
@secrets.command()
|
|
49
|
-
@click.
|
|
50
|
-
@click.
|
|
51
|
-
@click.option("--
|
|
52
|
-
def copy(
|
|
53
|
-
"""Copy secrets from one environment to
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
app_name = get_application_name()
|
|
61
|
-
secrets = get_ssm_secrets(app_name, source_environment)
|
|
62
|
-
|
|
63
|
-
for secret in secrets:
|
|
64
|
-
secret_name = secret[0].replace(f"/{source_environment}/", f"/{target_environment}/")
|
|
65
|
-
|
|
66
|
-
if secret_should_be_skipped(secret_name):
|
|
67
|
-
continue
|
|
68
|
-
|
|
69
|
-
click.echo(secret_name)
|
|
70
|
-
|
|
71
|
-
try:
|
|
72
|
-
set_ssm_param(
|
|
73
|
-
app_name,
|
|
74
|
-
target_environment,
|
|
75
|
-
secret_name,
|
|
76
|
-
secret[1],
|
|
77
|
-
False,
|
|
78
|
-
False,
|
|
79
|
-
f"Copied from {source_environment} environment.",
|
|
80
|
-
)
|
|
81
|
-
except ClientError as e:
|
|
82
|
-
if e.response["Error"]["Code"] == "ParameterAlreadyExists":
|
|
83
|
-
click.secho(
|
|
84
|
-
f"""The "{secret_name.split("/")[-1]}" parameter already exists for the "{target_environment}" environment.""",
|
|
85
|
-
fg="yellow",
|
|
86
|
-
)
|
|
87
|
-
else:
|
|
88
|
-
raise e
|
|
42
|
+
@click.option("--app", help="Application name.", required=True)
|
|
43
|
+
@click.option("--source", help="Source environment where to copy secrets from.", required=True)
|
|
44
|
+
@click.option("--target", help="Destination environment where to copy secrets to.", required=True)
|
|
45
|
+
def copy(app, source, target):
|
|
46
|
+
"""Copy secrets from one environment to another."""
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
Secrets().copy(app, source, target)
|
|
50
|
+
except PlatformException as err:
|
|
51
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
89
52
|
|
|
90
53
|
|
|
91
54
|
@secrets.command()
|
|
92
55
|
@click.argument("app", type=str, required=True)
|
|
93
56
|
@click.argument("env", type=str, required=True)
|
|
94
57
|
def list(app, env):
|
|
95
|
-
"""List secret names and values for an environment."""
|
|
96
|
-
|
|
97
|
-
session = get_aws_session_or_abort()
|
|
98
|
-
client = session.client("ssm")
|
|
99
|
-
|
|
100
|
-
path = SSM_BASE_PATH.format(app=app, env=env)
|
|
101
|
-
|
|
102
|
-
params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
|
|
103
|
-
secrets = []
|
|
104
|
-
|
|
105
|
-
while True:
|
|
106
|
-
response = client.get_parameters_by_path(**params)
|
|
107
|
-
|
|
108
|
-
for secret in response["Parameters"]:
|
|
109
|
-
secrets.append(f"{secret['Name']:<8}: {secret['Value']:<15}")
|
|
110
|
-
|
|
111
|
-
if "NextToken" in response:
|
|
112
|
-
params["NextToken"] = response["NextToken"]
|
|
113
|
-
else:
|
|
114
|
-
break
|
|
58
|
+
"""[DELETED] List secret names and values for an environment."""
|
|
115
59
|
|
|
116
|
-
|
|
117
|
-
|
|
60
|
+
click.secho(
|
|
61
|
+
message="\nThis command has been removed to prevent accidental exposure of secret values in local terminals and logs. To view secrets, log into your AWS account and head over to AWS Parameter Store https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/\n",
|
|
62
|
+
fg="magenta",
|
|
63
|
+
)
|
|
118
64
|
|
|
119
65
|
|
|
120
66
|
if __name__ == "__main__":
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
3
4
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
4
5
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
5
6
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
6
|
-
from dbt_platform_helper.utils.versioning import RequiredVersion
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
@@ -21,17 +21,12 @@ def version():
|
|
|
21
21
|
)
|
|
22
22
|
def get_platform_helper_for_project(pipeline):
|
|
23
23
|
"""
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
<pipeline>:
|
|
28
|
-
...
|
|
29
|
-
versions:
|
|
30
|
-
platform-helper
|
|
31
|
-
- The version from default_versions/platform-helper in 'platform-config.yml'
|
|
32
|
-
- Fall back on the version in the deprecated '.platform-helper-version' file
|
|
24
|
+
Deprecated: Required in pipelines for projects running platform-helper < 14.0.0.
|
|
25
|
+
|
|
26
|
+
Version preferences removed, requires specifying platform-helper version in platform-config.yml
|
|
33
27
|
"""
|
|
28
|
+
io = ClickIOProvider()
|
|
34
29
|
try:
|
|
35
|
-
|
|
30
|
+
io.info(PlatformHelperVersioning().get_required_version())
|
|
36
31
|
except PlatformException as err:
|
|
37
|
-
|
|
32
|
+
io.abort_with_error(str(err))
|
dbt_platform_helper/constants.py
CHANGED
|
@@ -1,11 +1,39 @@
|
|
|
1
|
-
#
|
|
1
|
+
# TODO: DBTP-1888: Move to Config provider
|
|
2
|
+
REPO_ORG = "uktrade"
|
|
2
3
|
PLATFORM_CONFIG_FILE = "platform-config.yml"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
SERVICE_DIRECTORY = "services"
|
|
5
|
+
SERVICE_CONFIG_FILE = "service-config.yml"
|
|
6
|
+
ENVIRONMENT_PIPELINE_MODULE_PATH = (
|
|
7
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref="
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
CODEBASE_PIPELINE_MODULE_PATH = (
|
|
11
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref="
|
|
12
|
+
)
|
|
13
|
+
EXTENSIONS_MODULE_PATH = (
|
|
14
|
+
"git::git@github.com:uktrade/platform-tools.git//terraform/extensions?depth=1&ref="
|
|
15
|
+
)
|
|
16
|
+
PLATFORM_HELPER_VERSION_OVERRIDE_KEY = "PLATFORM_HELPER_VERSION_OVERRIDE"
|
|
17
|
+
TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE_ENV_VAR = "TERRAFORM_EXTENSIONS_MODULE_SOURCE_OVERRIDE"
|
|
18
|
+
TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
19
|
+
"TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE"
|
|
20
|
+
)
|
|
21
|
+
TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
22
|
+
"TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE"
|
|
23
|
+
)
|
|
24
|
+
TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
|
|
25
|
+
"TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE"
|
|
26
|
+
)
|
|
27
|
+
TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR = "TERRAFORM_MODULE_SOURCE_TYPE" # "LOCAL", "SSH", "OVERRIDE"
|
|
28
|
+
IMAGE_TAG_ENV_VAR = "IMAGE_TAG"
|
|
29
|
+
PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
|
|
30
|
+
SUPPORTED_TERRAFORM_VERSION = "~> 1.11"
|
|
31
|
+
SUPPORTED_AWS_PROVIDER_VERSION = "~> 6"
|
|
32
|
+
FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR_VERSION = 13
|
|
33
|
+
|
|
34
|
+
MERGED_TPM_PLATFORM_HELPER_VERSION = 14
|
|
35
|
+
PLATFORM_CONFIG_SCHEMA_VERSION = 1
|
|
36
|
+
SERVICE_CONFIG_SCHEMA_VERSION = 1
|
|
9
37
|
|
|
10
38
|
# Keys
|
|
11
39
|
CODEBASE_PIPELINES_KEY = "codebase_pipelines"
|
|
@@ -28,3 +56,20 @@ SERVICE_NAME_SUFFIX = f"Service-{COPILOT_IDENTIFIER}"
|
|
|
28
56
|
REFRESH_TOKEN_MESSAGE = (
|
|
29
57
|
"To refresh this SSO session run `aws sso login` with the corresponding profile"
|
|
30
58
|
)
|
|
59
|
+
COPILOT_RULE_PRIORITY = 48000
|
|
60
|
+
PLATFORM_RULE_STARTING_PRIORITY = 10000
|
|
61
|
+
RULE_PRIORITY_INCREMENT = 100
|
|
62
|
+
DUMMY_RULE_REASON = "DummyRule"
|
|
63
|
+
MAINTENANCE_PAGE_TAGS = ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]
|
|
64
|
+
MAINTENANCE_PAGE_REASON = "MaintenancePage"
|
|
65
|
+
MANAGED_BY_PLATFORM = "DBT Platform"
|
|
66
|
+
MANAGED_BY_SERVICE_TERRAFORM = "DBT Platform - Service Terraform"
|
|
67
|
+
MANAGED_BY_PLATFORM_TERRAFORM = "DBT Platform - Terraform"
|
|
68
|
+
STANDARD_PLATFORM_SSO_ROLES = [
|
|
69
|
+
"AdministratorAccess",
|
|
70
|
+
"DBTPlatformDeveloperWrite",
|
|
71
|
+
"DBTPlatformDeveloperRead",
|
|
72
|
+
]
|
|
73
|
+
ROUTED_TO_PLATFORM_MODES = ["platform", "dual-deploy-platform-traffic"]
|
|
74
|
+
ROUTED_TO_COPILOT_MODES = ["copilot", "dual-deploy-copilot-traffic"]
|
|
75
|
+
HTTP_SERVICE_TYPES = ["Load Balanced Web Service"]
|