dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. dbt_platform_helper/COMMANDS.md +107 -27
  2. dbt_platform_helper/commands/application.py +5 -6
  3. dbt_platform_helper/commands/codebase.py +31 -10
  4. dbt_platform_helper/commands/conduit.py +3 -5
  5. dbt_platform_helper/commands/config.py +20 -311
  6. dbt_platform_helper/commands/copilot.py +18 -391
  7. dbt_platform_helper/commands/database.py +17 -9
  8. dbt_platform_helper/commands/environment.py +20 -14
  9. dbt_platform_helper/commands/generate.py +0 -3
  10. dbt_platform_helper/commands/internal.py +140 -0
  11. dbt_platform_helper/commands/notify.py +58 -78
  12. dbt_platform_helper/commands/pipeline.py +23 -19
  13. dbt_platform_helper/commands/secrets.py +39 -93
  14. dbt_platform_helper/commands/version.py +7 -12
  15. dbt_platform_helper/constants.py +52 -7
  16. dbt_platform_helper/domain/codebase.py +89 -39
  17. dbt_platform_helper/domain/conduit.py +335 -76
  18. dbt_platform_helper/domain/config.py +381 -0
  19. dbt_platform_helper/domain/copilot.py +398 -0
  20. dbt_platform_helper/domain/copilot_environment.py +8 -8
  21. dbt_platform_helper/domain/database_copy.py +2 -2
  22. dbt_platform_helper/domain/maintenance_page.py +254 -430
  23. dbt_platform_helper/domain/notify.py +64 -0
  24. dbt_platform_helper/domain/pipelines.py +43 -35
  25. dbt_platform_helper/domain/plans.py +41 -0
  26. dbt_platform_helper/domain/secrets.py +279 -0
  27. dbt_platform_helper/domain/service.py +570 -0
  28. dbt_platform_helper/domain/terraform_environment.py +14 -13
  29. dbt_platform_helper/domain/update_alb_rules.py +412 -0
  30. dbt_platform_helper/domain/versioning.py +249 -0
  31. dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
  32. dbt_platform_helper/entities/semantic_version.py +83 -0
  33. dbt_platform_helper/entities/service.py +339 -0
  34. dbt_platform_helper/platform_exception.py +4 -0
  35. dbt_platform_helper/providers/autoscaling.py +24 -0
  36. dbt_platform_helper/providers/aws/__init__.py +0 -0
  37. dbt_platform_helper/providers/aws/exceptions.py +70 -0
  38. dbt_platform_helper/providers/aws/interfaces.py +13 -0
  39. dbt_platform_helper/providers/aws/opensearch.py +23 -0
  40. dbt_platform_helper/providers/aws/redis.py +21 -0
  41. dbt_platform_helper/providers/aws/sso_auth.py +75 -0
  42. dbt_platform_helper/providers/cache.py +40 -4
  43. dbt_platform_helper/providers/cloudformation.py +1 -1
  44. dbt_platform_helper/providers/config.py +137 -19
  45. dbt_platform_helper/providers/config_validator.py +112 -51
  46. dbt_platform_helper/providers/copilot.py +24 -16
  47. dbt_platform_helper/providers/ecr.py +89 -7
  48. dbt_platform_helper/providers/ecs.py +228 -36
  49. dbt_platform_helper/providers/environment_variable.py +24 -0
  50. dbt_platform_helper/providers/files.py +1 -1
  51. dbt_platform_helper/providers/io.py +36 -4
  52. dbt_platform_helper/providers/kms.py +22 -0
  53. dbt_platform_helper/providers/load_balancers.py +402 -42
  54. dbt_platform_helper/providers/logs.py +72 -0
  55. dbt_platform_helper/providers/parameter_store.py +134 -0
  56. dbt_platform_helper/providers/s3.py +21 -0
  57. dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
  58. dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
  59. dbt_platform_helper/providers/schema_migrator.py +77 -0
  60. dbt_platform_helper/providers/secrets.py +5 -5
  61. dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
  62. dbt_platform_helper/providers/terraform_manifest.py +121 -19
  63. dbt_platform_helper/providers/version.py +106 -23
  64. dbt_platform_helper/providers/version_status.py +27 -0
  65. dbt_platform_helper/providers/vpc.py +36 -5
  66. dbt_platform_helper/providers/yaml_file.py +58 -2
  67. dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
  68. dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
  69. dbt_platform_helper/utilities/decorators.py +103 -0
  70. dbt_platform_helper/utils/application.py +119 -22
  71. dbt_platform_helper/utils/aws.py +39 -150
  72. dbt_platform_helper/utils/deep_merge.py +10 -0
  73. dbt_platform_helper/utils/git.py +1 -14
  74. dbt_platform_helper/utils/validation.py +1 -1
  75. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
  76. dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
  77. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
  78. platform_helper.py +3 -1
  79. terraform/elasticache-redis/plans.yml +85 -0
  80. terraform/opensearch/plans.yml +71 -0
  81. terraform/postgres/plans.yml +128 -0
  82. dbt_platform_helper/addon-plans.yml +0 -224
  83. dbt_platform_helper/providers/aws.py +0 -37
  84. dbt_platform_helper/providers/opensearch.py +0 -36
  85. dbt_platform_helper/providers/redis.py +0 -34
  86. dbt_platform_helper/providers/semantic_version.py +0 -126
  87. dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
  88. dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
  89. dbt_platform_helper/utils/cloudfoundry.py +0 -14
  90. dbt_platform_helper/utils/files.py +0 -53
  91. dbt_platform_helper/utils/manifests.py +0 -18
  92. dbt_platform_helper/utils/versioning.py +0 -238
  93. dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
  94. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
  95. {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
@@ -1,55 +1,11 @@
1
- import os
2
- import webbrowser
3
- from pathlib import Path
4
- from typing import Dict
5
-
6
- import boto3
7
- import botocore
8
1
  import click
9
- from prettytable import PrettyTable
10
2
 
11
- from dbt_platform_helper.providers.config import ConfigProvider
12
- from dbt_platform_helper.providers.semantic_version import (
13
- IncompatibleMajorVersionException,
14
- )
15
- from dbt_platform_helper.providers.validation import ValidationException
16
- from dbt_platform_helper.utils import versioning
3
+ from dbt_platform_helper.domain.config import Config
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
+ from dbt_platform_helper.providers.aws.sso_auth import SSOAuthProvider
6
+ from dbt_platform_helper.providers.io import ClickIOProvider
7
+ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
17
8
  from dbt_platform_helper.utils.click import ClickDocOptGroup
18
- from dbt_platform_helper.utils.versioning import get_platform_helper_versions
19
-
20
- yes = "\033[92m✔\033[0m"
21
- no = "\033[91m✖\033[0m"
22
- maybe = "\033[93m?\033[0m"
23
-
24
- RECOMMENDATIONS = {
25
- "dbt-platform-helper-upgrade": (
26
- "Upgrade dbt-platform-helper to version {version} `pip install "
27
- "--upgrade dbt-platform-helper=={version}`."
28
- ),
29
- "dbt-platform-helper-upgrade-note": (
30
- "Post upgrade, run `platform-helper copilot make-addons` to " "update your addon templates."
31
- ),
32
- "generic-tool-upgrade": "Upgrade {tool} to version {version}.",
33
- }
34
-
35
- SSO_START_URL = "https://uktrade.awsapps.com/start"
36
-
37
- AWS_CONFIG = """
38
- #
39
- # uktrade
40
- #
41
-
42
- [sso-session uktrade]
43
- sso_start_url = https://uktrade.awsapps.com/start#/
44
- sso_region = eu-west-2
45
- sso_registration_scopes = sso:account:access
46
-
47
- [default]
48
- sso_session = uktrade
49
- region = eu-west-2
50
- output = json
51
-
52
- """
53
9
 
54
10
 
55
11
  @click.group(cls=ClickDocOptGroup)
@@ -60,189 +16,19 @@ def config():
60
16
  @config.command()
61
17
  def validate():
62
18
  """Validate deployment or application configuration."""
63
- ran_checks = False
64
- if Path("copilot").exists():
65
- click.secho("\nDetected a deployment repository", fg="blue")
66
- deployment()
67
- ran_checks = True
68
-
69
- if not ran_checks:
70
- click.secho("Could not find a deployment repository, no checks to run.", fg="red")
71
- exit(1)
72
-
73
-
74
- def deployment():
75
- click.secho()
76
-
77
- compatible = True
78
- platform_helper_versions = get_platform_helper_versions()
79
- copilot_versions = versioning.get_copilot_versions()
80
- aws_versions = versioning.get_aws_versions()
81
- _check_tool_versions(platform_helper_versions, copilot_versions, aws_versions)
82
- click.secho("Checking addons templates versions...", fg="blue")
83
-
84
- local_version = platform_helper_versions.local
85
- latest_release = platform_helper_versions.latest
86
- addons_templates_table = PrettyTable()
87
- addons_templates_table.field_names = [
88
- "Addons Template File",
89
- "Generated with",
90
- "Compatible with local?",
91
- "Compatible with latest?",
92
- ]
93
- addons_templates_table.align["Addons Template File"] = "l"
94
-
95
- addons_templates = list(Path("./copilot").glob("**/addons/*"))
96
- # Sort by template file path
97
- addons_templates.sort(key=lambda e: str(e))
98
- # Bring environment addons to the top
99
- addons_templates.sort(key=lambda e: "environments/" not in str(e))
100
-
101
- recommendations = {}
102
-
103
- ConfigProvider().config_file_check()
104
-
105
- for template_file in addons_templates:
106
- generated_with_version = maybe
107
- local_compatible_symbol = yes
108
- latest_compatible_symbol = yes
109
-
110
- try:
111
- generated_with_version = versioning.get_template_generated_with_version(
112
- str(template_file.resolve())
113
- )
114
- versioning.validate_template_version(local_version, str(template_file.resolve()))
115
- except IncompatibleMajorVersionException:
116
- local_compatible_symbol = no
117
- compatible = False
118
- recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
119
- "dbt-platform-helper-upgrade"
120
- ].format(version=latest_release)
121
- recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
122
- "dbt-platform-helper-upgrade-note"
123
- ]
124
- except ValidationException:
125
- local_compatible_symbol = maybe
126
- compatible = False
127
- recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
128
- "dbt-platform-helper-upgrade"
129
- ].format(version=latest_release)
130
- recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
131
- "dbt-platform-helper-upgrade-note"
132
- ]
133
-
134
- try:
135
- generated_with_version = versioning.get_template_generated_with_version(
136
- str(template_file.resolve())
137
- )
138
- versioning.validate_template_version(latest_release, str(template_file.resolve()))
139
- except IncompatibleMajorVersionException:
140
- latest_compatible_symbol = no
141
- compatible = False
142
- except ValidationException:
143
- latest_compatible_symbol = maybe
144
- compatible = False
145
-
146
- addons_templates_table.add_row(
147
- [
148
- template_file.relative_to("."),
149
- (maybe if latest_compatible_symbol is maybe else str(generated_with_version)),
150
- local_compatible_symbol,
151
- latest_compatible_symbol,
152
- ]
153
- )
154
-
155
- click.secho(addons_templates_table)
156
- render_recommendations(recommendations)
157
-
158
- exit(0 if compatible else 1)
159
-
160
-
161
- def _check_tool_versions(platform_helper_versions, copilot_versions, aws_versions):
162
- click.secho("Checking tooling versions...", fg="blue")
163
- recommendations = {}
164
-
165
- local_copilot_version = copilot_versions.local
166
- copilot_latest_release = copilot_versions.latest
167
- if local_copilot_version is None:
168
- recommendations["install-copilot"] = (
169
- "Install AWS Copilot https://aws.github.io/copilot-cli/"
170
- )
171
-
172
- if aws_versions.local is None:
173
- recommendations["install-aws"] = "Install AWS CLI https://aws.amazon.com/cli/"
174
-
175
- tool_versions_table = PrettyTable()
176
- tool_versions_table.field_names = [
177
- "Tool",
178
- "Local version",
179
- "Released version",
180
- "Running latest?",
181
- ]
182
- tool_versions_table.align["Tool"] = "l"
183
-
184
- tool_versions_table.add_row(
185
- [
186
- "aws",
187
- str(aws_versions.local),
188
- str(aws_versions.latest),
189
- no if aws_versions.is_outdated() else yes,
190
- ]
191
- )
192
- tool_versions_table.add_row(
193
- [
194
- "copilot",
195
- str(copilot_versions.local),
196
- str(copilot_versions.latest),
197
- no if copilot_versions.is_outdated() else yes,
198
- ]
199
- )
200
- tool_versions_table.add_row(
201
- [
202
- "dbt-platform-helper",
203
- str(platform_helper_versions.local),
204
- str(platform_helper_versions.latest),
205
- no if platform_helper_versions.is_outdated() else yes,
206
- ]
207
- )
208
-
209
- click.secho(tool_versions_table)
210
-
211
- if aws_versions.is_outdated() and "install-aws" not in recommendations:
212
- recommendations["aws-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
213
- tool="AWS CLI",
214
- version=str(aws_versions.latest),
215
- )
216
-
217
- if copilot_versions.is_outdated() and "install-copilot" not in recommendations:
218
- recommendations["copilot-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
219
- tool="AWS Copilot",
220
- version=str(copilot_latest_release),
221
- )
222
-
223
- if platform_helper_versions.is_outdated():
224
- recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
225
- "dbt-platform-helper-upgrade"
226
- ].format(version=str(platform_helper_versions.latest))
227
- recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
228
- "dbt-platform-helper-upgrade-note"
229
- ]
230
-
231
- render_recommendations(recommendations)
232
-
233
-
234
- def render_recommendations(recommendations: Dict[str, str]):
235
- if recommendations:
236
- click.secho("\nRecommendations:\n", bold=True)
19
+ try:
20
+ Config().validate()
21
+ except PlatformException as err:
22
+ ClickIOProvider().abort_with_error(str(err))
237
23
 
238
- for name, recommendation in recommendations.items():
239
- if name.endswith("-note"):
240
- continue
241
- click.secho(f" - {recommendation}")
242
- if recommendations.get(f"{name}-note", False):
243
- click.secho(f" {recommendations.get(f'{name}-note')}")
244
24
 
245
- click.secho()
25
+ @config.command()
26
+ def migrate():
27
+ """Update configuration to match current schema."""
28
+ try:
29
+ Config().migrate()
30
+ except PlatformException as err:
31
+ ClickIOProvider().abort_with_error(str(err))
246
32
 
247
33
 
248
34
  @config.command()
@@ -254,85 +40,8 @@ def aws(file_path):
254
40
 
255
41
  If no `--file-path` is specified, defaults to `~/.aws/config`.
256
42
  """
257
- sso_oidc_client = boto3.client("sso-oidc", region_name="eu-west-2")
258
- sso_client = boto3.client("sso", region_name="eu-west-2")
259
- oidc_app = create_oidc_application(sso_oidc_client)
260
- verification_url, device_code = get_device_code(sso_oidc_client, oidc_app, SSO_START_URL)
261
-
262
- if click.confirm(
263
- "You are about to be redirected to a verification page. You will need to complete sign-in before returning to the command line. Do you want to continue?",
264
- abort=True,
265
- ):
266
- webbrowser.open(verification_url)
267
-
268
- if click.confirm("Have you completed the sign-in process in your browser?", abort=True):
269
- access_token = get_access_token(device_code, sso_oidc_client, oidc_app)
270
-
271
- aws_config_path = os.path.expanduser(file_path)
272
-
273
- if click.confirm(
274
- f"This command is destructive and will overwrite file contents at {file_path}. Are you sure you want to continue?",
275
- abort=True,
276
- ):
277
- with open(aws_config_path, "w") as config_file:
278
- config_file.write(AWS_CONFIG)
279
-
280
- for account in retrieve_aws_accounts(sso_client, access_token):
281
- config_file.write(f"[profile {account['accountName']}]\n")
282
- config_file.write("sso_session = uktrade\n")
283
- config_file.write(f"sso_account_id = {account['accountId']}\n")
284
- config_file.write("sso_role_name = AdministratorAccess\n")
285
- config_file.write("region = eu-west-2\n")
286
- config_file.write("output = json\n")
287
- config_file.write("\n")
288
-
289
-
290
- def create_oidc_application(sso_oidc_client):
291
- print("Creating temporary AWS SSO OIDC application")
292
- client = sso_oidc_client.register_client(
293
- clientName="platform-helper",
294
- clientType="public",
295
- )
296
- client_id = client.get("clientId")
297
- client_secret = client.get("clientSecret")
298
-
299
- return client_id, client_secret
300
-
301
-
302
- def get_device_code(sso_oidc_client, oidc_application, start_url):
303
- print("Initiating device code flow")
304
- authz = sso_oidc_client.start_device_authorization(
305
- clientId=oidc_application[0],
306
- clientSecret=oidc_application[1],
307
- startUrl=start_url,
308
- )
309
- url = authz.get("verificationUriComplete")
310
- deviceCode = authz.get("deviceCode")
311
-
312
- return url, deviceCode
313
-
314
-
315
- def retrieve_aws_accounts(sso_client, aws_sso_token):
316
- aws_accounts_response = sso_client.list_accounts(
317
- accessToken=aws_sso_token,
318
- maxResults=100,
319
- )
320
- if len(aws_accounts_response.get("accountList", [])) == 0:
321
- raise RuntimeError("Unable to retrieve AWS SSO account list\n")
322
- return aws_accounts_response.get("accountList")
323
-
324
-
325
- def get_access_token(device_code, sso_oidc_client, oidc_app):
326
43
  try:
327
- token_response = sso_oidc_client.create_token(
328
- clientId=oidc_app[0],
329
- clientSecret=oidc_app[1],
330
- grantType="urn:ietf:params:oauth:grant-type:device_code",
331
- deviceCode=device_code,
332
- )
333
-
334
- return token_response.get("accessToken")
335
-
336
- except botocore.exceptions.ClientError as e:
337
- if e.response["Error"]["Code"] != "AuthorizationPendingException":
338
- raise e
44
+ session = get_aws_session_or_abort()
45
+ Config(sso=SSOAuthProvider(session)).generate_aws(file_path)
46
+ except PlatformException as err:
47
+ ClickIOProvider().abort_with_error(str(err))