dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (38) hide show
  1. dbt_platform_helper/COMMANDS.md +1 -6
  2. dbt_platform_helper/commands/config.py +2 -2
  3. dbt_platform_helper/commands/copilot.py +51 -30
  4. dbt_platform_helper/commands/environment.py +25 -185
  5. dbt_platform_helper/commands/pipeline.py +10 -173
  6. dbt_platform_helper/constants.py +10 -0
  7. dbt_platform_helper/domain/codebase.py +8 -4
  8. dbt_platform_helper/domain/config_validator.py +242 -0
  9. dbt_platform_helper/domain/copilot_environment.py +204 -0
  10. dbt_platform_helper/domain/database_copy.py +16 -12
  11. dbt_platform_helper/domain/maintenance_page.py +44 -20
  12. dbt_platform_helper/domain/pipelines.py +213 -0
  13. dbt_platform_helper/domain/terraform_environment.py +86 -0
  14. dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py +100 -0
  15. dbt_platform_helper/jinja2_tags.py +1 -1
  16. dbt_platform_helper/providers/cache.py +14 -21
  17. dbt_platform_helper/providers/cloudformation.py +0 -1
  18. dbt_platform_helper/providers/config.py +100 -0
  19. dbt_platform_helper/providers/copilot.py +2 -0
  20. dbt_platform_helper/providers/files.py +26 -0
  21. dbt_platform_helper/providers/opensearch.py +36 -0
  22. dbt_platform_helper/providers/platform_config_schema.py +589 -527
  23. dbt_platform_helper/providers/redis.py +34 -0
  24. dbt_platform_helper/providers/vpc.py +57 -0
  25. dbt_platform_helper/providers/yaml_file.py +72 -0
  26. dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml +67 -0
  27. dbt_platform_helper/utils/application.py +32 -34
  28. dbt_platform_helper/utils/aws.py +1 -107
  29. dbt_platform_helper/utils/files.py +8 -59
  30. dbt_platform_helper/utils/platform_config.py +0 -7
  31. dbt_platform_helper/utils/template.py +10 -0
  32. dbt_platform_helper/utils/validation.py +5 -327
  33. dbt_platform_helper/utils/versioning.py +12 -0
  34. {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/METADATA +2 -2
  35. {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/RECORD +38 -26
  36. {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/WHEEL +1 -1
  37. {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/LICENSE +0 -0
  38. {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/entry_points.txt +0 -0
@@ -460,13 +460,11 @@ platform-helper environment online --app <application> --env <environment>
460
460
  ## Usage
461
461
 
462
462
  ```
463
- platform-helper environment generate --name <name> [--vpc-name <vpc_name>]
463
+ platform-helper environment generate --name <name>
464
464
  ```
465
465
 
466
466
  ## Options
467
467
 
468
- - `--vpc-name <text>`
469
-
470
468
  - `--name
471
469
  -n <text>`
472
470
 
@@ -551,9 +549,6 @@ platform-helper pipeline generate
551
549
  This command does the following in relation to the codebase pipelines:
552
550
  - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
553
551
 
554
- (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
555
- - Generates the copilot manifest.yml for copilot/environments/<environment>
556
-
557
552
  ## Usage
558
553
 
559
554
  ```
@@ -8,11 +8,11 @@ import botocore
8
8
  import click
9
9
  from prettytable import PrettyTable
10
10
 
11
+ from dbt_platform_helper.providers.config import ConfigProvider
11
12
  from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
12
13
  from dbt_platform_helper.providers.validation import ValidationException
13
14
  from dbt_platform_helper.utils import versioning
14
15
  from dbt_platform_helper.utils.click import ClickDocOptGroup
15
- from dbt_platform_helper.utils.validation import config_file_check
16
16
  from dbt_platform_helper.utils.versioning import get_platform_helper_versions
17
17
 
18
18
  yes = "\033[92m✔\033[0m"
@@ -98,7 +98,7 @@ def deployment():
98
98
 
99
99
  recommendations = {}
100
100
 
101
- config_file_check()
101
+ ConfigProvider.config_file_check()
102
102
 
103
103
  for template_file in addons_templates:
104
104
  generated_with_version = maybe
@@ -7,17 +7,22 @@ from pathlib import PosixPath
7
7
 
8
8
  import click
9
9
  import yaml
10
+ from schema import SchemaError
10
11
 
11
12
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
13
+ from dbt_platform_helper.domain.config_validator import ConfigValidator
14
+ from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
15
+ from dbt_platform_helper.providers.config import ConfigProvider
16
+ from dbt_platform_helper.providers.files import FileProvider
12
17
  from dbt_platform_helper.utils.application import get_application_name
13
18
  from dbt_platform_helper.utils.application import load_application
14
19
  from dbt_platform_helper.utils.aws import get_aws_session_or_abort
15
20
  from dbt_platform_helper.utils.click import ClickDocOptGroup
16
21
  from dbt_platform_helper.utils.files import generate_override_files
17
- from dbt_platform_helper.utils.files import mkfile
22
+ from dbt_platform_helper.utils.messages import abort_with_error
23
+ from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
18
24
  from dbt_platform_helper.utils.template import camel_case
19
25
  from dbt_platform_helper.utils.template import setup_templates
20
- from dbt_platform_helper.utils.validation import config_file_check
21
26
  from dbt_platform_helper.utils.validation import validate_addons
22
27
  from dbt_platform_helper.utils.versioning import (
23
28
  check_platform_helper_version_needs_update,
@@ -35,14 +40,6 @@ SERVICE_TYPES = [
35
40
  "Worker Service",
36
41
  ]
37
42
 
38
- ADDON_TEMPLATE_MAP = {
39
- "s3": ["addons/svc/s3-policy.yml"],
40
- "s3-policy": ["addons/svc/s3-policy.yml"],
41
- "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
42
- "subscription-filter": ["addons/svc/subscription-filter.yml"],
43
- "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
44
- }
45
-
46
43
 
47
44
  def list_copilot_local_environments():
48
45
  return [
@@ -244,25 +241,42 @@ def _get_s3_kms_alias_arns(session, application_name, config):
244
241
  return arns
245
242
 
246
243
 
244
+ def copilot_provider():
245
+ return CopilotTemplating()
246
+
247
+
247
248
  @copilot.command()
248
249
  def make_addons():
249
250
  """Generate addons CloudFormation for each environment."""
250
- output_dir = Path(".").absolute()
251
- config_file_check()
251
+ try:
252
+ config_provider = ConfigProvider(ConfigValidator())
253
+ _make_addons(config_provider)
254
+ except Exception as exc:
255
+ abort_with_error(exc)
256
+
257
+
258
+ def _make_addons(config_provider: ConfigProvider):
259
+ config_provider.config_file_check()
260
+ try:
261
+ config = config_provider.load_and_validate_platform_config()
262
+ except SchemaError as ex:
263
+ click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
264
+ raise click.Abort
252
265
 
253
266
  templates = setup_templates()
254
- config = _get_config()
267
+ extensions = _get_extensions()
255
268
  session = get_aws_session_or_abort()
256
269
 
257
270
  application_name = get_application_name()
258
271
 
259
272
  click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
260
273
 
274
+ output_dir = Path(".").absolute()
261
275
  env_path = Path(f"copilot/environments/")
262
276
  env_addons_path = env_path / "addons"
263
277
  env_overrides_path = env_path / "overrides"
264
278
 
265
- _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path)
279
+ _cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
266
280
  _generate_env_overrides(output_dir)
267
281
 
268
282
  svc_names = list_copilot_local_services()
@@ -271,33 +285,33 @@ def make_addons():
271
285
  _generate_svc_overrides(base_path, templates, svc_name)
272
286
 
273
287
  services = []
274
- for addon_name, addon_config in config.items():
275
- print(f">>>>>>>>> {addon_name}")
276
- addon_type = addon_config.pop("type")
277
- environments = addon_config.pop("environments")
288
+ for ext_name, ext_data in extensions.items():
289
+ extension = {**ext_data}
290
+ addon_type = extension.pop("type")
291
+ environments = extension.pop("environments")
278
292
  environment_addon_config = {
279
293
  "addon_type": addon_type,
280
294
  "environments": environments,
281
- "name": addon_config.get("name", None) or addon_name,
282
- "prefix": camel_case(addon_name),
283
- "secret_name": addon_name.upper().replace("-", "_"),
284
- **addon_config,
295
+ "name": extension.get("name", None) or ext_name,
296
+ "prefix": camel_case(ext_name),
297
+ "secret_name": ext_name.upper().replace("-", "_"),
298
+ **extension,
285
299
  }
286
300
 
287
301
  services.append(environment_addon_config)
288
302
 
289
303
  service_addon_config = {
290
304
  "application_name": application_name,
291
- "name": addon_config.get("name", None) or addon_name,
292
- "prefix": camel_case(addon_name),
305
+ "name": extension.get("name", None) or ext_name,
306
+ "prefix": camel_case(ext_name),
293
307
  "environments": environments,
294
- **addon_config,
308
+ **extension,
295
309
  }
296
310
 
297
311
  log_destination_arns = get_log_destination_arn()
298
312
 
299
313
  if addon_type in ["s3", "s3-policy"]:
300
- if config[addon_name].get("serve_static_content"):
314
+ if extensions[ext_name].get("serve_static_content"):
301
315
  continue
302
316
 
303
317
  s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
@@ -307,8 +321,8 @@ def make_addons():
307
321
  )
308
322
 
309
323
  _generate_service_addons(
310
- addon_config,
311
- addon_name,
324
+ extension,
325
+ ext_name,
312
326
  addon_type,
313
327
  output_dir,
314
328
  service_addon_config,
@@ -316,10 +330,15 @@ def make_addons():
316
330
  log_destination_arns,
317
331
  )
318
332
 
333
+ environments = config_provider.apply_environment_defaults(config)["environments"]
334
+
335
+ provider = copilot_provider()
336
+ provider.generate_cross_account_s3_policies(environments, extensions)
337
+
319
338
  click.echo(templates.get_template("addon-instructions.txt").render(services=services))
320
339
 
321
340
 
322
- def _get_config():
341
+ def _get_extensions():
323
342
  config = _validate_and_normalise_extensions_config(PACKAGE_DIR / "default-extensions.yml")
324
343
  project_config = _validate_and_normalise_extensions_config(PLATFORM_CONFIG_FILE, "extensions")
325
344
  config.update(project_config)
@@ -360,7 +379,9 @@ def _generate_service_addons(
360
379
 
361
380
  (output_dir / service_path).mkdir(parents=True, exist_ok=True)
362
381
  click.echo(
363
- mkfile(output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True)
382
+ FileProvider.mkfile(
383
+ output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
384
+ )
364
385
  )
365
386
 
366
387
 
@@ -1,18 +1,15 @@
1
- import boto3
2
1
  import click
3
2
  from schema import SchemaError
4
3
 
5
4
  from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
6
5
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
7
- from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
6
+ from dbt_platform_helper.domain.config_validator import ConfigValidator
7
+ from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
8
+ from dbt_platform_helper.domain.maintenance_page import MaintenancePage
9
+ from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
8
10
  from dbt_platform_helper.platform_exception import PlatformException
9
- from dbt_platform_helper.providers.load_balancers import find_https_listener
10
- from dbt_platform_helper.utils.aws import get_aws_session_or_abort
11
+ from dbt_platform_helper.providers.config import ConfigProvider
11
12
  from dbt_platform_helper.utils.click import ClickDocOptGroup
12
- from dbt_platform_helper.utils.files import apply_environment_defaults
13
- from dbt_platform_helper.utils.files import mkfile
14
- from dbt_platform_helper.utils.template import setup_templates
15
- from dbt_platform_helper.utils.validation import load_and_validate_platform_config
16
13
  from dbt_platform_helper.utils.versioning import (
17
14
  check_platform_helper_version_needs_update,
18
15
  )
@@ -39,8 +36,11 @@ def environment():
39
36
  @click.option("--vpc", type=str)
40
37
  def offline(app, env, svc, template, vpc):
41
38
  """Take load-balanced web services offline with a maintenance page."""
42
- maintenance_page = MaintenancePageProvider()
43
- maintenance_page.activate(app, env, svc, template, vpc)
39
+ try:
40
+ MaintenancePage().activate(app, env, svc, template, vpc)
41
+ except PlatformException as err:
42
+ click.secho(str(err), fg="red")
43
+ raise click.Abort
44
44
 
45
45
 
46
46
  @environment.command()
@@ -48,112 +48,26 @@ def offline(app, env, svc, template, vpc):
48
48
  @click.option("--env", type=str, required=True)
49
49
  def online(app, env):
50
50
  """Remove a maintenance page from an environment."""
51
- maintenance_page = MaintenancePageProvider()
52
- maintenance_page.deactivate(app, env)
53
-
54
-
55
- def get_vpc_id(session, env_name, vpc_name=None):
56
- if not vpc_name:
57
- vpc_name = f"{session.profile_name}-{env_name}"
58
-
59
- filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
60
- vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
61
-
62
- if not vpcs:
63
- filters[0]["Values"] = [session.profile_name]
64
- vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
65
-
66
- if not vpcs:
67
- click.secho(
68
- f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
69
- )
70
- raise click.Abort
71
-
72
- return vpcs[0]["VpcId"]
73
-
74
-
75
- def get_subnet_ids(session, vpc_id, environment_name):
76
- subnets = session.client("ec2").describe_subnets(
77
- Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
78
- )["Subnets"]
79
-
80
- if not subnets:
81
- click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
82
- raise click.Abort
83
-
84
- public_tag = {"Key": "subnet_type", "Value": "public"}
85
- public_subnets = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
86
- private_tag = {"Key": "subnet_type", "Value": "private"}
87
- private_subnets = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
88
-
89
- # This call and the method declaration can be removed when we stop using AWS Copilot to deploy the services
90
- public_subnets, private_subnets = _match_subnet_id_order_to_cloudformation_exports(
91
- session,
92
- environment_name,
93
- public_subnets,
94
- private_subnets,
95
- )
96
-
97
- return public_subnets, private_subnets
98
-
99
-
100
- def _match_subnet_id_order_to_cloudformation_exports(
101
- session, environment_name, public_subnets, private_subnets
102
- ):
103
- public_subnet_exports = []
104
- private_subnet_exports = []
105
- for page in session.client("cloudformation").get_paginator("list_exports").paginate():
106
- for export in page["Exports"]:
107
- if f"-{environment_name}-" in export["Name"]:
108
- if export["Name"].endswith("-PublicSubnets"):
109
- public_subnet_exports = export["Value"].split(",")
110
- if export["Name"].endswith("-PrivateSubnets"):
111
- private_subnet_exports = export["Value"].split(",")
112
-
113
- # If the elements match, regardless of order, use the list from the CloudFormation exports
114
- if set(public_subnets) == set(public_subnet_exports):
115
- public_subnets = public_subnet_exports
116
- if set(private_subnets) == set(private_subnet_exports):
117
- private_subnets = private_subnet_exports
118
-
119
- return public_subnets, private_subnets
120
-
121
-
122
- def get_cert_arn(session, application, env_name):
123
51
  try:
124
- arn = find_https_certificate(session, application, env_name)
125
- except:
126
- click.secho(
127
- f"No certificate found with domain name matching environment {env_name}.", fg="red"
128
- )
52
+ MaintenancePage().deactivate(app, env)
53
+ except PlatformException as err:
54
+ click.secho(str(err), fg="red")
129
55
  raise click.Abort
130
56
 
131
- return arn
132
-
133
57
 
134
58
  @environment.command()
135
- @click.option("--vpc-name", hidden=True)
136
59
  @click.option("--name", "-n", required=True)
137
- def generate(name, vpc_name):
138
- if vpc_name:
139
- click.secho(
140
- f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
141
- fg="red",
142
- )
143
- raise click.Abort
144
-
60
+ def generate(name):
145
61
  try:
146
- conf = load_and_validate_platform_config()
62
+ config_provider = ConfigProvider(ConfigValidator())
63
+ CopilotEnvironment(config_provider).generate(name)
64
+ # TODO this exception will never be caught as the config provider catches schema errors and aborts
147
65
  except SchemaError as ex:
148
66
  click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
149
67
  raise click.Abort
150
-
151
- env_config = apply_environment_defaults(conf)["environments"][name]
152
- profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
153
- click.secho(f"Using {profile_for_environment} for this AWS session")
154
- session = get_aws_session_or_abort(profile_for_environment)
155
-
156
- _generate_copilot_environment_manifests(name, conf["application"], env_config, session)
68
+ except PlatformException as err:
69
+ click.secho(str(err), fg="red")
70
+ raise click.Abort
157
71
 
158
72
 
159
73
  @environment.command(help="Generate terraform manifest for the specified environment.")
@@ -165,84 +79,10 @@ def generate(name, vpc_name):
165
79
  help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
166
80
  )
167
81
  def generate_terraform(name, terraform_platform_modules_version):
168
- conf = load_and_validate_platform_config()
169
-
170
- env_config = apply_environment_defaults(conf)["environments"][name]
171
- _generate_terraform_environment_manifests(
172
- conf["application"], name, env_config, terraform_platform_modules_version
173
- )
174
-
175
-
176
- def _generate_copilot_environment_manifests(environment_name, application, env_config, session):
177
- env_template = setup_templates().get_template("env/manifest.yml")
178
- vpc_name = env_config.get("vpc", None)
179
- vpc_id = get_vpc_id(session, environment_name, vpc_name)
180
- pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id, environment_name)
181
- cert_arn = get_cert_arn(session, application, environment_name)
182
- contents = env_template.render(
183
- {
184
- "name": environment_name,
185
- "vpc_id": vpc_id,
186
- "pub_subnet_ids": pub_subnet_ids,
187
- "priv_subnet_ids": priv_subnet_ids,
188
- "certificate_arn": cert_arn,
189
- }
190
- )
191
- click.echo(
192
- mkfile(
193
- ".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
194
- )
195
- )
196
-
197
-
198
- def _generate_terraform_environment_manifests(
199
- application, env, env_config, cli_terraform_platform_modules_version
200
- ):
201
- env_template = setup_templates().get_template("environments/main.tf")
202
-
203
- terraform_platform_modules_version = _determine_terraform_platform_modules_version(
204
- env_config, cli_terraform_platform_modules_version
205
- )
206
-
207
- contents = env_template.render(
208
- {
209
- "application": application,
210
- "environment": env,
211
- "config": env_config,
212
- "terraform_platform_modules_version": terraform_platform_modules_version,
213
- }
214
- )
215
-
216
- click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
217
-
218
-
219
- def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
220
- cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
221
- env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
222
- "terraform-platform-modules"
223
- )
224
- version_preference_order = [
225
- cli_terraform_platform_modules_version,
226
- env_conf_terraform_platform_modules_version,
227
- DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
228
- ]
229
- return [version for version in version_preference_order if version][0]
230
-
231
-
232
- def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
233
- listener_arn = find_https_listener(session, app, env)
234
- cert_client = session.client("elbv2")
235
- certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
236
- "Certificates"
237
- ]
238
82
 
239
83
  try:
240
- certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
241
- except StopIteration:
242
- raise CertificateNotFoundException()
243
-
244
- return certificate_arn
245
-
246
-
247
- class CertificateNotFoundException(PlatformException):
248
- pass
84
+ config_provider = ConfigProvider(ConfigValidator())
85
+ TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
86
+ except PlatformException as err:
87
+ click.secho(str(err), fg="red")
88
+ raise click.Abort
@@ -1,30 +1,19 @@
1
1
  #!/usr/bin/env python
2
- from os import makedirs
3
- from pathlib import Path
4
- from shutil import rmtree
5
2
 
6
3
  import click
7
4
 
8
5
  from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
9
- from dbt_platform_helper.utils.application import get_application_name
10
- from dbt_platform_helper.utils.aws import get_account_details
6
+ from dbt_platform_helper.domain.config_validator import ConfigValidator
7
+ from dbt_platform_helper.domain.pipelines import Pipelines
8
+ from dbt_platform_helper.providers.config import ConfigProvider
11
9
  from dbt_platform_helper.utils.aws import get_codestar_connection_arn
12
- from dbt_platform_helper.utils.aws import get_public_repository_arn
13
10
  from dbt_platform_helper.utils.click import ClickDocOptGroup
14
- from dbt_platform_helper.utils.files import generate_override_files_from_template
15
- from dbt_platform_helper.utils.files import mkfile
16
11
  from dbt_platform_helper.utils.git import git_remote
17
12
  from dbt_platform_helper.utils.messages import abort_with_error
18
- from dbt_platform_helper.utils.template import setup_templates
19
- from dbt_platform_helper.utils.validation import load_and_validate_platform_config
20
13
  from dbt_platform_helper.utils.versioning import (
21
14
  check_platform_helper_version_needs_update,
22
15
  )
23
16
 
24
- CODEBASE_PIPELINES_KEY = "codebase_pipelines"
25
- ENVIRONMENTS_KEY = "environments"
26
- ENVIRONMENT_PIPELINES_KEY = "environment_pipelines"
27
-
28
17
 
29
18
  @click.group(chain=True, cls=ClickDocOptGroup)
30
19
  def pipeline():
@@ -60,164 +49,12 @@ def generate(terraform_platform_modules_version, deploy_branch):
60
49
 
61
50
  This command does the following in relation to the codebase pipelines:
62
51
  - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
63
-
64
- (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
65
- - Generates the copilot manifest.yml for copilot/environments/<environment>
66
52
  """
67
- pipeline_config = load_and_validate_platform_config()
68
-
69
- has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
70
- has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
71
-
72
- if not (has_codebase_pipelines or has_environment_pipelines):
73
- click.secho("No pipelines defined: nothing to do.", err=True, fg="yellow")
74
- return
75
-
76
- platform_config_terraform_modules_default_version = pipeline_config.get(
77
- "default_versions", {}
78
- ).get("terraform-platform-modules", "")
79
-
80
- templates = setup_templates()
81
- app_name = get_application_name()
82
-
83
- git_repo = git_remote()
84
- if not git_repo:
85
- abort_with_error("The current directory is not a git repository")
86
-
87
- codestar_connection_arn = get_codestar_connection_arn(app_name)
88
- if codestar_connection_arn is None:
89
- abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
90
-
91
- base_path = Path(".")
92
- copilot_pipelines_dir = base_path / f"copilot/pipelines"
93
-
94
- _clean_pipeline_config(copilot_pipelines_dir)
95
-
96
- if has_environment_pipelines:
97
- environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
98
-
99
- for config in environment_pipelines.values():
100
- aws_account = config.get("account")
101
- _generate_terraform_environment_pipeline_manifest(
102
- pipeline_config["application"],
103
- aws_account,
104
- terraform_platform_modules_version,
105
- platform_config_terraform_modules_default_version,
106
- deploy_branch,
107
- )
108
-
109
- if has_codebase_pipelines:
110
- account_id, _ = get_account_details()
111
-
112
- for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
113
- _generate_codebase_pipeline(
114
- account_id,
115
- app_name,
116
- codestar_connection_arn,
117
- git_repo,
118
- codebase,
119
- base_path,
120
- copilot_pipelines_dir,
121
- templates,
122
- )
123
-
124
-
125
- def _clean_pipeline_config(pipelines_dir):
126
- if pipelines_dir.exists():
127
- click.echo("Deleting copilot/pipelines directory.")
128
- rmtree(pipelines_dir)
129
-
130
-
131
- def _generate_codebase_pipeline(
132
- account_id,
133
- app_name,
134
- codestar_connection_arn,
135
- git_repo,
136
- codebase,
137
- base_path,
138
- pipelines_dir,
139
- templates,
140
- ):
141
- makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
142
- environments = []
143
- for pipelines in codebase["pipelines"]:
144
- environments += pipelines[ENVIRONMENTS_KEY]
145
-
146
- additional_ecr = codebase.get("additional_ecr_repository", None)
147
- add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
148
- additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
149
-
150
- template_data = {
151
- "account_id": account_id,
152
- "app_name": app_name,
153
- "deploy_repo": git_repo,
154
- "codebase": codebase,
155
- ENVIRONMENTS_KEY: environments,
156
- "codestar_connection_arn": codestar_connection_arn,
157
- "codestar_connection_id": codestar_connection_arn.split("/")[-1],
158
- "additional_ecr_arn": additional_ecr_arn,
159
- }
160
-
161
- _create_file_from_template(
162
- base_path,
163
- f"{codebase['name']}/manifest.yml",
164
- pipelines_dir,
165
- template_data,
166
- templates,
167
- "codebase/manifest.yml",
168
- )
169
-
170
- overrides_path = Path(__file__).parent.parent.joinpath("templates/pipelines/codebase/overrides")
171
- generate_override_files_from_template(
172
- base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
173
- )
174
-
175
-
176
- def _create_file_from_template(
177
- base_path, file_name, pipelines_dir, template_data, templates, template_name=None
178
- ):
179
- contents = templates.get_template(
180
- f"pipelines/{file_name if template_name is None else template_name}"
181
- ).render(template_data)
182
- message = mkfile(base_path, pipelines_dir / file_name, contents, overwrite=True)
183
- click.echo(message)
184
-
185
-
186
- def _generate_terraform_environment_pipeline_manifest(
187
- application,
188
- aws_account,
189
- cli_terraform_platform_modules_version,
190
- platform_config_terraform_modules_default_version,
191
- deploy_branch,
192
- ):
193
- env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
194
-
195
- terraform_platform_modules_version = _determine_terraform_platform_modules_version(
196
- cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
53
+ pipelines = Pipelines(
54
+ ConfigProvider(ConfigValidator()),
55
+ click.secho,
56
+ abort_with_error,
57
+ git_remote,
58
+ get_codestar_connection_arn,
197
59
  )
198
-
199
- contents = env_pipeline_template.render(
200
- {
201
- "application": application,
202
- "aws_account": aws_account,
203
- "terraform_platform_modules_version": terraform_platform_modules_version,
204
- "deploy_branch": deploy_branch,
205
- }
206
- )
207
-
208
- dir_path = f"terraform/environment-pipelines/{aws_account}"
209
- makedirs(dir_path, exist_ok=True)
210
-
211
- click.echo(mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
212
-
213
-
214
- def _determine_terraform_platform_modules_version(
215
- cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
216
- ):
217
-
218
- version_preference_order = [
219
- cli_terraform_platform_modules_version,
220
- platform_config_terraform_modules_default_version,
221
- DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
222
- ]
223
- return [version for version in version_preference_order if version][0]
60
+ pipelines.generate(terraform_platform_modules_version, deploy_branch)