dbt-platform-helper 13.1.1__py3-none-any.whl → 13.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/commands/application.py +3 -5
- dbt_platform_helper/commands/codebase.py +2 -4
- dbt_platform_helper/commands/conduit.py +2 -4
- dbt_platform_helper/commands/config.py +19 -17
- dbt_platform_helper/commands/copilot.py +13 -390
- dbt_platform_helper/commands/environment.py +6 -6
- dbt_platform_helper/commands/generate.py +2 -3
- dbt_platform_helper/commands/notify.py +2 -4
- dbt_platform_helper/commands/pipeline.py +2 -4
- dbt_platform_helper/commands/secrets.py +2 -4
- dbt_platform_helper/commands/version.py +2 -2
- dbt_platform_helper/domain/codebase.py +14 -11
- dbt_platform_helper/domain/copilot.py +397 -0
- dbt_platform_helper/domain/copilot_environment.py +6 -6
- dbt_platform_helper/domain/maintenance_page.py +227 -431
- dbt_platform_helper/domain/pipelines.py +1 -1
- dbt_platform_helper/domain/terraform_environment.py +1 -1
- dbt_platform_helper/domain/versioning.py +157 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/config_validator.py +15 -14
- dbt_platform_helper/providers/copilot.py +1 -1
- dbt_platform_helper/providers/io.py +17 -0
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +269 -43
- dbt_platform_helper/providers/semantic_version.py +33 -10
- dbt_platform_helper/providers/version.py +42 -0
- dbt_platform_helper/providers/yaml_file.py +0 -1
- dbt_platform_helper/utils/application.py +14 -0
- dbt_platform_helper/utils/aws.py +27 -4
- dbt_platform_helper/utils/tool_versioning.py +96 -0
- {dbt_platform_helper-13.1.1.dist-info → dbt_platform_helper-13.2.0.dist-info}/METADATA +3 -4
- {dbt_platform_helper-13.1.1.dist-info → dbt_platform_helper-13.2.0.dist-info}/RECORD +39 -35
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/utils/versioning.py +0 -238
- /dbt_platform_helper/providers/{aws.py → aws/exceptions.py} +0 -0
- {dbt_platform_helper-13.1.1.dist-info → dbt_platform_helper-13.2.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-13.1.1.dist-info → dbt_platform_helper-13.2.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-13.1.1.dist-info → dbt_platform_helper-13.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -10,11 +10,9 @@ from datetime import timedelta
|
|
|
10
10
|
import click
|
|
11
11
|
from prettytable import PrettyTable
|
|
12
12
|
|
|
13
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
13
14
|
from dbt_platform_helper.utils.application import load_application
|
|
14
15
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
15
|
-
from dbt_platform_helper.utils.versioning import (
|
|
16
|
-
check_platform_helper_version_needs_update,
|
|
17
|
-
)
|
|
18
16
|
|
|
19
17
|
YELLOW = "\033[93m"
|
|
20
18
|
CYAN = "\033[96m"
|
|
@@ -97,7 +95,7 @@ def get_query_results(env, app, query_string, timeout):
|
|
|
97
95
|
@click.group(chain=True, cls=ClickDocOptGroup, deprecated=True)
|
|
98
96
|
def application():
|
|
99
97
|
"""[DEPRECATED] Application metrics."""
|
|
100
|
-
|
|
98
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
101
99
|
|
|
102
100
|
|
|
103
101
|
@application.command(deprecated=True)
|
|
@@ -141,7 +139,7 @@ def container_stats(env, app, storage, network):
|
|
|
141
139
|
storage_write = field[6]["value"]
|
|
142
140
|
network_read = field[7]["value"]
|
|
143
141
|
network_write = field[8]["value"]
|
|
144
|
-
cpu = "
|
|
142
|
+
cpu = f"{float(field[9]['value']):.1f}%"
|
|
145
143
|
memory = f"{field[10]['value']}M"
|
|
146
144
|
|
|
147
145
|
# Nothing to compare to at start.
|
|
@@ -1,18 +1,16 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from dbt_platform_helper.domain.codebase import Codebase
|
|
4
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
4
5
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
6
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
6
7
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
7
|
-
from dbt_platform_helper.utils.versioning import (
|
|
8
|
-
check_platform_helper_version_needs_update,
|
|
9
|
-
)
|
|
10
8
|
|
|
11
9
|
|
|
12
10
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
13
11
|
def codebase():
|
|
14
12
|
"""Codebase commands."""
|
|
15
|
-
|
|
13
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
16
14
|
|
|
17
15
|
|
|
18
16
|
@codebase.command()
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from dbt_platform_helper.domain.conduit import Conduit
|
|
4
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
4
5
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
6
|
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
6
7
|
from dbt_platform_helper.providers.ecs import ECS
|
|
@@ -8,9 +9,6 @@ from dbt_platform_helper.providers.io import ClickIOProvider
|
|
|
8
9
|
from dbt_platform_helper.providers.secrets import Secrets
|
|
9
10
|
from dbt_platform_helper.utils.application import load_application
|
|
10
11
|
from dbt_platform_helper.utils.click import ClickDocOptCommand
|
|
11
|
-
from dbt_platform_helper.utils.versioning import (
|
|
12
|
-
check_platform_helper_version_needs_update,
|
|
13
|
-
)
|
|
14
12
|
|
|
15
13
|
CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
|
|
16
14
|
|
|
@@ -28,7 +26,7 @@ CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
|
|
|
28
26
|
def conduit(addon_name: str, app: str, env: str, access: str):
|
|
29
27
|
"""Opens a shell for a given addon_name create a conduit connection to
|
|
30
28
|
interact with postgres, opensearch or redis."""
|
|
31
|
-
|
|
29
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
32
30
|
application = load_application(app)
|
|
33
31
|
|
|
34
32
|
try:
|
|
@@ -9,13 +9,14 @@ import click
|
|
|
9
9
|
from prettytable import PrettyTable
|
|
10
10
|
|
|
11
11
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
12
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
12
13
|
from dbt_platform_helper.providers.semantic_version import (
|
|
13
14
|
IncompatibleMajorVersionException,
|
|
14
15
|
)
|
|
15
16
|
from dbt_platform_helper.providers.validation import ValidationException
|
|
16
|
-
from dbt_platform_helper.utils import
|
|
17
|
+
from dbt_platform_helper.utils import tool_versioning
|
|
17
18
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
18
|
-
from dbt_platform_helper.utils.
|
|
19
|
+
from dbt_platform_helper.utils.tool_versioning import get_platform_helper_version_status
|
|
19
20
|
|
|
20
21
|
yes = "\033[92m✔\033[0m"
|
|
21
22
|
no = "\033[91m✖\033[0m"
|
|
@@ -75,14 +76,15 @@ def deployment():
|
|
|
75
76
|
click.secho()
|
|
76
77
|
|
|
77
78
|
compatible = True
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
platform_helper_version_status = get_platform_helper_version_status()
|
|
80
|
+
ClickIOProvider().process_messages(platform_helper_version_status.validate())
|
|
81
|
+
copilot_versions = tool_versioning.get_copilot_versions()
|
|
82
|
+
aws_versions = tool_versioning.get_aws_versions()
|
|
83
|
+
_check_tool_versions(platform_helper_version_status, copilot_versions, aws_versions)
|
|
82
84
|
click.secho("Checking addons templates versions...", fg="blue")
|
|
83
85
|
|
|
84
|
-
local_version =
|
|
85
|
-
latest_release =
|
|
86
|
+
local_version = platform_helper_version_status.installed
|
|
87
|
+
latest_release = platform_helper_version_status.latest
|
|
86
88
|
addons_templates_table = PrettyTable()
|
|
87
89
|
addons_templates_table.field_names = [
|
|
88
90
|
"Addons Template File",
|
|
@@ -108,10 +110,10 @@ def deployment():
|
|
|
108
110
|
latest_compatible_symbol = yes
|
|
109
111
|
|
|
110
112
|
try:
|
|
111
|
-
generated_with_version =
|
|
113
|
+
generated_with_version = tool_versioning.get_template_generated_with_version(
|
|
112
114
|
str(template_file.resolve())
|
|
113
115
|
)
|
|
114
|
-
|
|
116
|
+
tool_versioning.validate_template_version(local_version, str(template_file.resolve()))
|
|
115
117
|
except IncompatibleMajorVersionException:
|
|
116
118
|
local_compatible_symbol = no
|
|
117
119
|
compatible = False
|
|
@@ -132,10 +134,10 @@ def deployment():
|
|
|
132
134
|
]
|
|
133
135
|
|
|
134
136
|
try:
|
|
135
|
-
generated_with_version =
|
|
137
|
+
generated_with_version = tool_versioning.get_template_generated_with_version(
|
|
136
138
|
str(template_file.resolve())
|
|
137
139
|
)
|
|
138
|
-
|
|
140
|
+
tool_versioning.validate_template_version(latest_release, str(template_file.resolve()))
|
|
139
141
|
except IncompatibleMajorVersionException:
|
|
140
142
|
latest_compatible_symbol = no
|
|
141
143
|
compatible = False
|
|
@@ -162,14 +164,14 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
|
|
|
162
164
|
click.secho("Checking tooling versions...", fg="blue")
|
|
163
165
|
recommendations = {}
|
|
164
166
|
|
|
165
|
-
local_copilot_version = copilot_versions.
|
|
167
|
+
local_copilot_version = copilot_versions.installed
|
|
166
168
|
copilot_latest_release = copilot_versions.latest
|
|
167
169
|
if local_copilot_version is None:
|
|
168
170
|
recommendations["install-copilot"] = (
|
|
169
171
|
"Install AWS Copilot https://aws.github.io/copilot-cli/"
|
|
170
172
|
)
|
|
171
173
|
|
|
172
|
-
if aws_versions.
|
|
174
|
+
if aws_versions.installed is None:
|
|
173
175
|
recommendations["install-aws"] = "Install AWS CLI https://aws.amazon.com/cli/"
|
|
174
176
|
|
|
175
177
|
tool_versions_table = PrettyTable()
|
|
@@ -184,7 +186,7 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
|
|
|
184
186
|
tool_versions_table.add_row(
|
|
185
187
|
[
|
|
186
188
|
"aws",
|
|
187
|
-
str(aws_versions.
|
|
189
|
+
str(aws_versions.installed),
|
|
188
190
|
str(aws_versions.latest),
|
|
189
191
|
no if aws_versions.is_outdated() else yes,
|
|
190
192
|
]
|
|
@@ -192,7 +194,7 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
|
|
|
192
194
|
tool_versions_table.add_row(
|
|
193
195
|
[
|
|
194
196
|
"copilot",
|
|
195
|
-
str(copilot_versions.
|
|
197
|
+
str(copilot_versions.installed),
|
|
196
198
|
str(copilot_versions.latest),
|
|
197
199
|
no if copilot_versions.is_outdated() else yes,
|
|
198
200
|
]
|
|
@@ -200,7 +202,7 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
|
|
|
200
202
|
tool_versions_table.add_row(
|
|
201
203
|
[
|
|
202
204
|
"dbt-platform-helper",
|
|
203
|
-
str(platform_helper_versions.
|
|
205
|
+
str(platform_helper_versions.installed),
|
|
204
206
|
str(platform_helper_versions.latest),
|
|
205
207
|
no if platform_helper_versions.is_outdated() else yes,
|
|
206
208
|
]
|
|
@@ -1,412 +1,35 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
|
|
3
|
-
import copy
|
|
4
|
-
import json
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from pathlib import PosixPath
|
|
7
|
-
|
|
8
3
|
import click
|
|
9
|
-
import yaml
|
|
10
|
-
from schema import SchemaError
|
|
11
4
|
|
|
12
|
-
from dbt_platform_helper.
|
|
5
|
+
from dbt_platform_helper.domain.copilot import Copilot
|
|
13
6
|
from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
|
|
7
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
14
8
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
15
9
|
from dbt_platform_helper.providers.config_validator import ConfigValidator
|
|
16
10
|
from dbt_platform_helper.providers.files import FileProvider
|
|
17
|
-
from dbt_platform_helper.
|
|
18
|
-
from dbt_platform_helper.
|
|
11
|
+
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
12
|
+
from dbt_platform_helper.providers.kms import KMSProvider
|
|
19
13
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
20
14
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
21
|
-
from dbt_platform_helper.utils.files import generate_override_files
|
|
22
|
-
from dbt_platform_helper.utils.messages import abort_with_error
|
|
23
|
-
from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
|
|
24
|
-
from dbt_platform_helper.utils.template import camel_case
|
|
25
|
-
from dbt_platform_helper.utils.template import setup_templates
|
|
26
|
-
from dbt_platform_helper.utils.validation import validate_addons
|
|
27
|
-
from dbt_platform_helper.utils.versioning import (
|
|
28
|
-
check_platform_helper_version_needs_update,
|
|
29
|
-
)
|
|
30
|
-
|
|
31
|
-
PACKAGE_DIR = Path(__file__).resolve().parent.parent
|
|
32
|
-
|
|
33
|
-
WAF_ACL_ARN_KEY = "waf-acl-arn"
|
|
34
|
-
|
|
35
|
-
SERVICE_TYPES = [
|
|
36
|
-
"Load Balanced Web Service",
|
|
37
|
-
"Backend Service",
|
|
38
|
-
"Request-Driven Web Service",
|
|
39
|
-
"Static Site",
|
|
40
|
-
"Worker Service",
|
|
41
|
-
]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def list_copilot_local_environments():
|
|
45
|
-
return [
|
|
46
|
-
path.parent.parts[-1] for path in Path("./copilot/environments/").glob("*/manifest.yml")
|
|
47
|
-
]
|
|
48
15
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
data = yaml.safe_load(manifest_file)
|
|
53
|
-
if not data or not data.get("type"):
|
|
54
|
-
click.echo(
|
|
55
|
-
click.style(f"No type defined in manifest file {str(path)}; exiting", fg="red")
|
|
56
|
-
)
|
|
57
|
-
exit(1)
|
|
58
|
-
|
|
59
|
-
return data.get("type") in SERVICE_TYPES
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def list_copilot_local_services():
|
|
63
|
-
return [
|
|
64
|
-
path.parent.parts[-1]
|
|
65
|
-
for path in Path("./copilot/").glob("*/manifest.yml")
|
|
66
|
-
if is_service(path)
|
|
67
|
-
]
|
|
16
|
+
# TODOs
|
|
17
|
+
# Figure out a pattern for copilot templating and the new copilot domain - probably a lot of overlap here that really belongs in the copilottemplating domain instead (atleast whatever is concerned with "templating")
|
|
18
|
+
# Check for E2E test coverage.
|
|
68
19
|
|
|
69
20
|
|
|
70
21
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
71
22
|
def copilot():
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def _validate_and_normalise_extensions_config(config_file, key_in_config_file=None):
|
|
76
|
-
"""Load a config file, validate it against the extensions schemas and return
|
|
77
|
-
the normalised config dict."""
|
|
78
|
-
|
|
79
|
-
def _lookup_plan(addon_type, env_conf):
|
|
80
|
-
plan = env_conf.pop("plan", None)
|
|
81
|
-
conf = addon_plans[addon_type][plan] if plan else {}
|
|
82
|
-
|
|
83
|
-
# Make a copy of the addon plan config so subsequent
|
|
84
|
-
# calls do not override the root object
|
|
85
|
-
conf = conf.copy()
|
|
86
|
-
|
|
87
|
-
conf.update(env_conf)
|
|
88
|
-
|
|
89
|
-
return conf
|
|
90
|
-
|
|
91
|
-
def _normalise_keys(source: dict):
|
|
92
|
-
return {k.replace("-", "_"): v for k, v in source.items()}
|
|
93
|
-
|
|
94
|
-
with open(PACKAGE_DIR / "addon-plans.yml", "r") as fd:
|
|
95
|
-
addon_plans = yaml.safe_load(fd)
|
|
96
|
-
|
|
97
|
-
# load and validate config
|
|
98
|
-
with open(config_file, "r") as fd:
|
|
99
|
-
config = yaml.safe_load(fd)
|
|
100
|
-
|
|
101
|
-
if config and key_in_config_file:
|
|
102
|
-
config = config[key_in_config_file]
|
|
103
|
-
|
|
104
|
-
# empty file
|
|
105
|
-
if not config:
|
|
106
|
-
return {}
|
|
107
|
-
|
|
108
|
-
errors = validate_addons(config)
|
|
109
|
-
|
|
110
|
-
if errors:
|
|
111
|
-
click.echo(click.style(f"Errors found in {config_file}:", fg="red"))
|
|
112
|
-
for addon, error in errors.items():
|
|
113
|
-
click.echo(click.style(f"Addon '{addon}': {error}", fg="red"))
|
|
114
|
-
exit(1)
|
|
115
|
-
|
|
116
|
-
env_names = list_copilot_local_environments()
|
|
117
|
-
svc_names = list_copilot_local_services()
|
|
118
|
-
|
|
119
|
-
if not env_names:
|
|
120
|
-
click.echo(
|
|
121
|
-
click.style(f"No environments found in ./copilot/environments; exiting", fg="red")
|
|
122
|
-
)
|
|
123
|
-
exit(1)
|
|
124
|
-
|
|
125
|
-
if not svc_names:
|
|
126
|
-
click.echo(click.style(f"No services found in ./copilot/; exiting", fg="red"))
|
|
127
|
-
exit(1)
|
|
128
|
-
|
|
129
|
-
normalised_config = {}
|
|
130
|
-
config_has_errors = False
|
|
131
|
-
for addon_name, addon_config in config.items():
|
|
132
|
-
addon_type = addon_config["type"]
|
|
133
|
-
normalised_config[addon_name] = copy.deepcopy(addon_config)
|
|
134
|
-
|
|
135
|
-
if "services" in normalised_config[addon_name]:
|
|
136
|
-
if normalised_config[addon_name]["services"] == "__all__":
|
|
137
|
-
normalised_config[addon_name]["services"] = svc_names
|
|
138
|
-
|
|
139
|
-
if not set(normalised_config[addon_name]["services"]).issubset(set(svc_names)):
|
|
140
|
-
click.echo(
|
|
141
|
-
click.style(
|
|
142
|
-
f"Services listed in {addon_name}.services do not exist in ./copilot/",
|
|
143
|
-
fg="red",
|
|
144
|
-
),
|
|
145
|
-
)
|
|
146
|
-
config_has_errors = True
|
|
147
|
-
|
|
148
|
-
environments = normalised_config[addon_name].pop("environments", {})
|
|
149
|
-
default = environments.pop("*", environments.pop("default", {}))
|
|
150
|
-
|
|
151
|
-
initial = _lookup_plan(addon_type, default)
|
|
152
|
-
|
|
153
|
-
missing_envs = set(environments.keys()) - set(env_names)
|
|
154
|
-
if missing_envs:
|
|
155
|
-
click.echo(
|
|
156
|
-
click.style(
|
|
157
|
-
f"Environment keys listed in {addon_name} do not match those defined in ./copilot/environments.",
|
|
158
|
-
fg="red",
|
|
159
|
-
)
|
|
160
|
-
),
|
|
161
|
-
click.echo(
|
|
162
|
-
click.style(
|
|
163
|
-
f" Missing environments: {', '.join(sorted(missing_envs))}",
|
|
164
|
-
fg="white",
|
|
165
|
-
),
|
|
166
|
-
)
|
|
167
|
-
config_has_errors = True
|
|
168
|
-
|
|
169
|
-
if config_has_errors:
|
|
170
|
-
continue
|
|
171
|
-
|
|
172
|
-
normalised_environments = {}
|
|
173
|
-
|
|
174
|
-
for env in env_names:
|
|
175
|
-
normalised_environments[env] = _normalise_keys(initial)
|
|
176
|
-
|
|
177
|
-
for env_name, env_config in environments.items():
|
|
178
|
-
if env_config is None:
|
|
179
|
-
env_config = {}
|
|
180
|
-
normalised_environments[env_name].update(
|
|
181
|
-
_lookup_plan(addon_type, _normalise_keys(env_config))
|
|
182
|
-
)
|
|
183
|
-
|
|
184
|
-
normalised_config[addon_name]["environments"] = normalised_environments
|
|
185
|
-
|
|
186
|
-
if config_has_errors:
|
|
187
|
-
exit(1)
|
|
188
|
-
|
|
189
|
-
return normalised_config
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
def get_log_destination_arn():
|
|
193
|
-
"""Get destination arns stored in param store in projects aws account."""
|
|
194
|
-
session = get_aws_session_or_abort()
|
|
195
|
-
client = session.client("ssm", region_name="eu-west-2")
|
|
196
|
-
response = client.get_parameters(Names=["/copilot/tools/central_log_groups"])
|
|
197
|
-
|
|
198
|
-
if not response["Parameters"]:
|
|
199
|
-
click.echo(
|
|
200
|
-
click.style(
|
|
201
|
-
"No aws central log group defined in Parameter Store at location /copilot/tools/central_log_groups; exiting",
|
|
202
|
-
fg="red",
|
|
203
|
-
)
|
|
204
|
-
)
|
|
205
|
-
exit(1)
|
|
206
|
-
|
|
207
|
-
destination_arns = json.loads(response["Parameters"][0]["Value"])
|
|
208
|
-
return destination_arns
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
def _generate_svc_overrides(base_path, templates, name):
|
|
212
|
-
click.echo(f"\n>>> Generating service overrides for {name}\n")
|
|
213
|
-
overrides_path = base_path.joinpath(f"copilot/{name}/overrides")
|
|
214
|
-
overrides_path.mkdir(parents=True, exist_ok=True)
|
|
215
|
-
overrides_file = overrides_path.joinpath("cfn.patches.yml")
|
|
216
|
-
overrides_file.write_text(templates.get_template("svc/overrides/cfn.patches.yml").render())
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def _get_s3_kms_alias_arns(session, application_name, config):
|
|
220
|
-
application = load_application(application_name, session)
|
|
221
|
-
arns = {}
|
|
222
|
-
|
|
223
|
-
for environment_name in application.environments:
|
|
224
|
-
if environment_name not in config:
|
|
225
|
-
continue
|
|
226
|
-
|
|
227
|
-
if "bucket_name" not in config[environment_name]:
|
|
228
|
-
continue
|
|
229
|
-
|
|
230
|
-
bucket_name = config[environment_name]["bucket_name"]
|
|
231
|
-
kms_client = application.environments[environment_name].session.client("kms")
|
|
232
|
-
alias_name = f"alias/{application_name}-{environment_name}-{bucket_name}-key"
|
|
233
|
-
|
|
234
|
-
try:
|
|
235
|
-
response = kms_client.describe_key(KeyId=alias_name)
|
|
236
|
-
except kms_client.exceptions.NotFoundException:
|
|
237
|
-
pass
|
|
238
|
-
else:
|
|
239
|
-
arns[environment_name] = response["KeyMetadata"]["Arn"]
|
|
240
|
-
|
|
241
|
-
return arns
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
def copilot_provider():
|
|
245
|
-
return CopilotTemplating()
|
|
23
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
246
24
|
|
|
247
25
|
|
|
248
26
|
@copilot.command()
|
|
249
27
|
def make_addons():
|
|
250
28
|
"""Generate addons CloudFormation for each environment."""
|
|
251
29
|
try:
|
|
30
|
+
session = get_aws_session_or_abort()
|
|
252
31
|
config_provider = ConfigProvider(ConfigValidator())
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
def _make_addons(config_provider: ConfigProvider):
|
|
259
|
-
config_provider.config_file_check()
|
|
260
|
-
try:
|
|
261
|
-
config = config_provider.load_and_validate_platform_config()
|
|
262
|
-
except SchemaError as ex:
|
|
263
|
-
click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
|
|
264
|
-
raise click.Abort
|
|
265
|
-
|
|
266
|
-
templates = setup_templates()
|
|
267
|
-
extensions = _get_extensions()
|
|
268
|
-
session = get_aws_session_or_abort()
|
|
269
|
-
|
|
270
|
-
application_name = get_application_name()
|
|
271
|
-
|
|
272
|
-
click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
|
|
273
|
-
|
|
274
|
-
output_dir = Path(".").absolute()
|
|
275
|
-
env_path = Path(f"copilot/environments/")
|
|
276
|
-
env_addons_path = env_path / "addons"
|
|
277
|
-
env_overrides_path = env_path / "overrides"
|
|
278
|
-
|
|
279
|
-
_cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
|
|
280
|
-
_generate_env_overrides(output_dir)
|
|
281
|
-
|
|
282
|
-
svc_names = list_copilot_local_services()
|
|
283
|
-
base_path = Path(".")
|
|
284
|
-
for svc_name in svc_names:
|
|
285
|
-
_generate_svc_overrides(base_path, templates, svc_name)
|
|
286
|
-
|
|
287
|
-
services = []
|
|
288
|
-
for ext_name, ext_data in extensions.items():
|
|
289
|
-
extension = {**ext_data}
|
|
290
|
-
addon_type = extension.pop("type")
|
|
291
|
-
environments = extension.pop("environments")
|
|
292
|
-
environment_addon_config = {
|
|
293
|
-
"addon_type": addon_type,
|
|
294
|
-
"environments": environments,
|
|
295
|
-
"name": extension.get("name", None) or ext_name,
|
|
296
|
-
"prefix": camel_case(ext_name),
|
|
297
|
-
"secret_name": ext_name.upper().replace("-", "_"),
|
|
298
|
-
**extension,
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
services.append(environment_addon_config)
|
|
302
|
-
|
|
303
|
-
service_addon_config = {
|
|
304
|
-
"application_name": application_name,
|
|
305
|
-
"name": extension.get("name", None) or ext_name,
|
|
306
|
-
"prefix": camel_case(ext_name),
|
|
307
|
-
"environments": environments,
|
|
308
|
-
**extension,
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
log_destination_arns = get_log_destination_arn()
|
|
312
|
-
|
|
313
|
-
if addon_type in ["s3", "s3-policy"]:
|
|
314
|
-
if extensions[ext_name].get("serve_static_content"):
|
|
315
|
-
continue
|
|
316
|
-
|
|
317
|
-
s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
|
|
318
|
-
for environment_name in environments:
|
|
319
|
-
environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
|
|
320
|
-
environment_name, "kms-key-not-found"
|
|
321
|
-
)
|
|
322
|
-
|
|
323
|
-
_generate_service_addons(
|
|
324
|
-
extension,
|
|
325
|
-
ext_name,
|
|
326
|
-
addon_type,
|
|
327
|
-
output_dir,
|
|
328
|
-
service_addon_config,
|
|
329
|
-
templates,
|
|
330
|
-
log_destination_arns,
|
|
331
|
-
)
|
|
332
|
-
|
|
333
|
-
environments = config_provider.apply_environment_defaults(config)["environments"]
|
|
334
|
-
|
|
335
|
-
provider = copilot_provider()
|
|
336
|
-
provider.generate_cross_account_s3_policies(environments, extensions)
|
|
337
|
-
|
|
338
|
-
click.echo(templates.get_template("addon-instructions.txt").render(services=services))
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
def _get_extensions():
|
|
342
|
-
config = _validate_and_normalise_extensions_config(PACKAGE_DIR / "default-extensions.yml")
|
|
343
|
-
project_config = _validate_and_normalise_extensions_config(PLATFORM_CONFIG_FILE, "extensions")
|
|
344
|
-
config.update(project_config)
|
|
345
|
-
return config
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
def _generate_env_overrides(output_dir):
|
|
349
|
-
path = "templates/env/terraform-overrides"
|
|
350
|
-
click.echo("\n>>> Generating Environment overrides\n")
|
|
351
|
-
overrides_path = output_dir.joinpath(f"copilot/environments/overrides")
|
|
352
|
-
overrides_path.mkdir(parents=True, exist_ok=True)
|
|
353
|
-
template_overrides_path = Path(__file__).parent.parent.joinpath(path)
|
|
354
|
-
generate_override_files(Path("."), template_overrides_path, overrides_path)
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
def _generate_service_addons(
|
|
358
|
-
addon_config,
|
|
359
|
-
addon_name,
|
|
360
|
-
addon_type,
|
|
361
|
-
output_dir,
|
|
362
|
-
service_addon_config,
|
|
363
|
-
templates,
|
|
364
|
-
log_destination_arns,
|
|
365
|
-
):
|
|
366
|
-
# generate svc addons
|
|
367
|
-
for addon_template in ADDON_TEMPLATE_MAP.get(addon_type, []):
|
|
368
|
-
template = templates.get_template(addon_template)
|
|
369
|
-
|
|
370
|
-
for svc in addon_config.get("services", []):
|
|
371
|
-
service_path = Path(f"copilot/{svc}/addons/")
|
|
372
|
-
|
|
373
|
-
contents = template.render(
|
|
374
|
-
{
|
|
375
|
-
"addon_config": service_addon_config,
|
|
376
|
-
"log_destination": log_destination_arns,
|
|
377
|
-
}
|
|
378
|
-
)
|
|
379
|
-
|
|
380
|
-
(output_dir / service_path).mkdir(parents=True, exist_ok=True)
|
|
381
|
-
click.echo(
|
|
382
|
-
FileProvider.mkfile(
|
|
383
|
-
output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
|
|
384
|
-
)
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
def _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path):
|
|
389
|
-
def _rmdir(path):
|
|
390
|
-
if not path.exists():
|
|
391
|
-
return
|
|
392
|
-
for f in path.iterdir():
|
|
393
|
-
if f.is_file():
|
|
394
|
-
f.unlink()
|
|
395
|
-
if f.is_dir():
|
|
396
|
-
_rmdir(f)
|
|
397
|
-
f.rmdir()
|
|
398
|
-
|
|
399
|
-
_rmdir(output_dir / env_addons_path)
|
|
400
|
-
_rmdir(output_dir / env_overrides_path)
|
|
401
|
-
|
|
402
|
-
all_services = set()
|
|
403
|
-
for services in [v["services"] for v in config.values() if "services" in v]:
|
|
404
|
-
all_services.update(services)
|
|
405
|
-
|
|
406
|
-
for service in all_services:
|
|
407
|
-
svc_addons_dir = Path(output_dir, "copilot", service, "addons")
|
|
408
|
-
if not svc_addons_dir.exists():
|
|
409
|
-
continue
|
|
410
|
-
for f in svc_addons_dir.iterdir():
|
|
411
|
-
if f.is_file():
|
|
412
|
-
f.unlink()
|
|
32
|
+
kms_provider = KMSProvider(session.client("kms"))
|
|
33
|
+
Copilot(config_provider, FileProvider(), CopilotTemplating(), kms_provider).make_addons()
|
|
34
|
+
except Exception as err:
|
|
35
|
+
ClickIOProvider().abort_with_error(str(err))
|
|
@@ -4,6 +4,7 @@ from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VER
|
|
|
4
4
|
from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
|
|
5
5
|
from dbt_platform_helper.domain.maintenance_page import MaintenancePage
|
|
6
6
|
from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
|
|
7
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
7
8
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
8
9
|
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
9
10
|
from dbt_platform_helper.providers.config import ConfigProvider
|
|
@@ -13,9 +14,6 @@ from dbt_platform_helper.providers.vpc import VpcProvider
|
|
|
13
14
|
from dbt_platform_helper.utils.application import load_application
|
|
14
15
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
15
16
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
16
|
-
from dbt_platform_helper.utils.versioning import (
|
|
17
|
-
check_platform_helper_version_needs_update,
|
|
18
|
-
)
|
|
19
17
|
|
|
20
18
|
AVAILABLE_TEMPLATES = ["default", "migration", "dmas-migration"]
|
|
21
19
|
|
|
@@ -23,7 +21,7 @@ AVAILABLE_TEMPLATES = ["default", "migration", "dmas-migration"]
|
|
|
23
21
|
@click.group(cls=ClickDocOptGroup)
|
|
24
22
|
def environment():
|
|
25
23
|
"""Commands affecting environments."""
|
|
26
|
-
|
|
24
|
+
PlatformHelperVersioning().check_if_needs_update()
|
|
27
25
|
|
|
28
26
|
|
|
29
27
|
@environment.command()
|
|
@@ -94,9 +92,11 @@ def generate(name):
|
|
|
94
92
|
help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
|
|
95
93
|
)
|
|
96
94
|
def generate_terraform(name, terraform_platform_modules_version):
|
|
97
|
-
|
|
95
|
+
click_io = ClickIOProvider()
|
|
98
96
|
try:
|
|
97
|
+
# TODO = pass the session to ConfigValidator
|
|
98
|
+
get_aws_session_or_abort()
|
|
99
99
|
config_provider = ConfigProvider(ConfigValidator())
|
|
100
100
|
TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
|
|
101
101
|
except PlatformException as err:
|
|
102
|
-
|
|
102
|
+
click_io.abort_with_error(str(err))
|
|
@@ -3,8 +3,8 @@ import click
|
|
|
3
3
|
|
|
4
4
|
from dbt_platform_helper.commands.copilot import make_addons
|
|
5
5
|
from dbt_platform_helper.commands.pipeline import generate as pipeline_generate
|
|
6
|
+
from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
|
|
6
7
|
from dbt_platform_helper.utils.click import ClickDocOptCommand
|
|
7
|
-
from dbt_platform_helper.utils.versioning import RequiredVersion
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
@click.command(cls=ClickDocOptCommand)
|
|
@@ -16,7 +16,6 @@ def generate(ctx: click.Context):
|
|
|
16
16
|
|
|
17
17
|
Wraps pipeline generate and make-addons.
|
|
18
18
|
"""
|
|
19
|
-
|
|
20
|
-
RequiredVersion().check_platform_helper_version_mismatch()
|
|
19
|
+
PlatformHelperVersioning().check_platform_helper_version_mismatch()
|
|
21
20
|
ctx.invoke(pipeline_generate)
|
|
22
21
|
ctx.invoke(make_addons)
|