dbt-platform-helper 11.0.0__py3-none-any.whl → 11.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +60 -19
- dbt_platform_helper/commands/database.py +97 -18
- dbt_platform_helper/commands/environment.py +6 -518
- dbt_platform_helper/domain/__init__.py +0 -0
- dbt_platform_helper/domain/database_copy.py +220 -0
- dbt_platform_helper/domain/maintenance_page.py +459 -0
- dbt_platform_helper/providers/load_balancers.py +51 -0
- dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +5 -4
- dbt_platform_helper/utils/aws.py +47 -37
- dbt_platform_helper/utils/validation.py +29 -0
- {dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/METADATA +3 -2
- {dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/RECORD +15 -12
- dbt_platform_helper/commands/database_helpers.py +0 -145
- {dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
|
|
5
|
+
lb_client = session.client("elbv2")
|
|
6
|
+
|
|
7
|
+
describe_response = lb_client.describe_load_balancers()
|
|
8
|
+
load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
|
|
9
|
+
|
|
10
|
+
load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
|
|
11
|
+
|
|
12
|
+
load_balancer_arn = None
|
|
13
|
+
for lb in load_balancers:
|
|
14
|
+
tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
|
|
15
|
+
if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
|
|
16
|
+
load_balancer_arn = lb["ResourceArn"]
|
|
17
|
+
|
|
18
|
+
if not load_balancer_arn:
|
|
19
|
+
raise LoadBalancerNotFoundError()
|
|
20
|
+
|
|
21
|
+
return load_balancer_arn
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
|
|
25
|
+
load_balancer_arn = find_load_balancer(session, app, env)
|
|
26
|
+
lb_client = session.client("elbv2")
|
|
27
|
+
listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
|
|
28
|
+
|
|
29
|
+
listener_arn = None
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
|
|
33
|
+
except StopIteration:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
if not listener_arn:
|
|
37
|
+
raise ListenerNotFoundError()
|
|
38
|
+
|
|
39
|
+
return listener_arn
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class LoadBalancerNotFoundError(Exception):
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ListenerNotFoundError(Exception):
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ListenerRuleNotFoundError(Exception):
|
|
51
|
+
pass
|
|
@@ -3321,12 +3321,13 @@
|
|
|
3321
3321
|
"dev": true
|
|
3322
3322
|
},
|
|
3323
3323
|
"node_modules/micromatch": {
|
|
3324
|
-
"version": "4.0.
|
|
3325
|
-
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.
|
|
3326
|
-
"integrity": "sha512-
|
|
3324
|
+
"version": "4.0.8",
|
|
3325
|
+
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
|
|
3326
|
+
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
|
|
3327
3327
|
"dev": true,
|
|
3328
|
+
"license": "MIT",
|
|
3328
3329
|
"dependencies": {
|
|
3329
|
-
"braces": "^3.0.
|
|
3330
|
+
"braces": "^3.0.3",
|
|
3330
3331
|
"picomatch": "^2.3.1"
|
|
3331
3332
|
},
|
|
3332
3333
|
"engines": {
|
dbt_platform_helper/utils/aws.py
CHANGED
|
@@ -7,6 +7,7 @@ from typing import Tuple
|
|
|
7
7
|
|
|
8
8
|
import boto3
|
|
9
9
|
import botocore
|
|
10
|
+
import botocore.exceptions
|
|
10
11
|
import click
|
|
11
12
|
import yaml
|
|
12
13
|
from boto3 import Session
|
|
@@ -20,62 +21,71 @@ AWS_SESSION_CACHE = {}
|
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
|
|
23
|
-
|
|
24
|
+
REFRESH_TOKEN_MESSAGE = (
|
|
25
|
+
"To refresh this SSO session run `aws sso login` with the corresponding profile"
|
|
26
|
+
)
|
|
27
|
+
aws_profile = aws_profile or os.getenv("AWS_PROFILE")
|
|
24
28
|
if aws_profile in AWS_SESSION_CACHE:
|
|
25
29
|
return AWS_SESSION_CACHE[aws_profile]
|
|
26
30
|
|
|
27
|
-
|
|
28
|
-
click.secho(f"""Checking AWS connection for profile "{aws_profile}"...""", fg="cyan")
|
|
31
|
+
click.secho(f'Checking AWS connection for profile "{aws_profile}"...', fg="cyan")
|
|
29
32
|
|
|
30
33
|
try:
|
|
31
34
|
session = boto3.session.Session(profile_name=aws_profile)
|
|
35
|
+
sts = session.client("sts")
|
|
36
|
+
account_id, user_id = get_account_details(sts)
|
|
37
|
+
click.secho("Credentials are valid.", fg="green")
|
|
38
|
+
|
|
32
39
|
except botocore.exceptions.ProfileNotFound:
|
|
33
|
-
|
|
34
|
-
exit(1)
|
|
40
|
+
_handle_error(f'AWS profile "{aws_profile}" is not configured.')
|
|
35
41
|
except botocore.exceptions.ClientError as e:
|
|
36
42
|
if e.response["Error"]["Code"] == "ExpiredToken":
|
|
37
|
-
|
|
38
|
-
f"Credentials are NOT valid. \nPlease login with: aws sso login --profile {aws_profile}"
|
|
39
|
-
fg="red",
|
|
43
|
+
_handle_error(
|
|
44
|
+
f"Credentials are NOT valid. \nPlease login with: aws sso login --profile {aws_profile}"
|
|
40
45
|
)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
except
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
):
|
|
52
|
-
click.secho(
|
|
53
|
-
"The SSO session associated with this profile has expired or is otherwise invalid."
|
|
54
|
-
"To refresh this SSO session run `aws sso login` with the corresponding profile",
|
|
55
|
-
fg="red",
|
|
46
|
+
except botocore.exceptions.NoCredentialsError:
|
|
47
|
+
_handle_error("There are no credentials set for this session.", REFRESH_TOKEN_MESSAGE)
|
|
48
|
+
except botocore.exceptions.UnauthorizedSSOTokenError:
|
|
49
|
+
_handle_error("The SSO Token used for this session is unauthorised.", REFRESH_TOKEN_MESSAGE)
|
|
50
|
+
except botocore.exceptions.TokenRetrievalError:
|
|
51
|
+
_handle_error("Unable to retrieve the Token for this session.", REFRESH_TOKEN_MESSAGE)
|
|
52
|
+
except botocore.exceptions.SSOTokenLoadError:
|
|
53
|
+
_handle_error(
|
|
54
|
+
"The SSO session associated with this profile has expired, is not set or is otherwise invalid.",
|
|
55
|
+
REFRESH_TOKEN_MESSAGE,
|
|
56
56
|
)
|
|
57
|
-
exit(1)
|
|
58
57
|
|
|
59
58
|
alias_client = session.client("iam")
|
|
60
|
-
account_name = alias_client.list_account_aliases()
|
|
59
|
+
account_name = alias_client.list_account_aliases().get("AccountAliases", [])
|
|
60
|
+
|
|
61
|
+
_log_account_info(account_name, account_id)
|
|
62
|
+
|
|
63
|
+
click.echo(
|
|
64
|
+
click.style("User: ", fg="yellow")
|
|
65
|
+
+ click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True)
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
AWS_SESSION_CACHE[aws_profile] = session
|
|
69
|
+
return session
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _handle_error(message: str, refresh_token_message: str = None) -> None:
|
|
73
|
+
full_message = message + (" " + refresh_token_message if refresh_token_message else "")
|
|
74
|
+
click.secho(full_message, fg="red")
|
|
75
|
+
exit(1)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _log_account_info(account_name: list, account_id: str) -> None:
|
|
61
79
|
if account_name:
|
|
62
80
|
click.echo(
|
|
63
81
|
click.style("Logged in with AWS account: ", fg="yellow")
|
|
64
|
-
+ click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True)
|
|
82
|
+
+ click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True)
|
|
65
83
|
)
|
|
66
84
|
else:
|
|
67
85
|
click.echo(
|
|
68
86
|
click.style("Logged in with AWS account id: ", fg="yellow")
|
|
69
|
-
+ click.style(f"{account_id}", fg="white", bold=True)
|
|
87
|
+
+ click.style(f"{account_id}", fg="white", bold=True)
|
|
70
88
|
)
|
|
71
|
-
click.echo(
|
|
72
|
-
click.style("User: ", fg="yellow")
|
|
73
|
-
+ click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True),
|
|
74
|
-
)
|
|
75
|
-
|
|
76
|
-
AWS_SESSION_CACHE[aws_profile] = session
|
|
77
|
-
|
|
78
|
-
return session
|
|
79
89
|
|
|
80
90
|
|
|
81
91
|
class NoProfileForAccountIdError(Exception):
|
|
@@ -362,12 +372,12 @@ def get_connection_string(
|
|
|
362
372
|
|
|
363
373
|
|
|
364
374
|
class Vpc:
|
|
365
|
-
def __init__(self, subnets, security_groups):
|
|
375
|
+
def __init__(self, subnets: list[str], security_groups: list[str]):
|
|
366
376
|
self.subnets = subnets
|
|
367
377
|
self.security_groups = security_groups
|
|
368
378
|
|
|
369
379
|
|
|
370
|
-
def get_vpc_info_by_name(session, app, env, vpc_name):
|
|
380
|
+
def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) -> Vpc:
|
|
371
381
|
ec2_client = session.client("ec2")
|
|
372
382
|
vpc_response = ec2_client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [vpc_name]}])
|
|
373
383
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import ipaddress
|
|
2
|
+
import os
|
|
2
3
|
import re
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
@@ -11,6 +12,8 @@ from schema import Regex
|
|
|
11
12
|
from schema import Schema
|
|
12
13
|
from schema import SchemaError
|
|
13
14
|
from yaml.parser import ParserError
|
|
15
|
+
from yamllint import config
|
|
16
|
+
from yamllint import linter
|
|
14
17
|
|
|
15
18
|
from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
|
|
16
19
|
from dbt_platform_helper.constants import ENVIRONMENTS_KEY
|
|
@@ -677,6 +680,25 @@ def _validate_environment_pipelines_triggers(config):
|
|
|
677
680
|
abort_with_error(error_message + "\n ".join(errors))
|
|
678
681
|
|
|
679
682
|
|
|
683
|
+
def lint_yaml_for_duplicate_keys(file_path):
|
|
684
|
+
lint_yaml_config = """
|
|
685
|
+
rules:
|
|
686
|
+
key-duplicates: enable
|
|
687
|
+
"""
|
|
688
|
+
yaml_config = config.YamlLintConfig(lint_yaml_config)
|
|
689
|
+
|
|
690
|
+
with open(file_path, "r") as yaml_file:
|
|
691
|
+
file_contents = yaml_file.read()
|
|
692
|
+
results = linter.run(file_contents, yaml_config)
|
|
693
|
+
|
|
694
|
+
parsed_results = [
|
|
695
|
+
"\t" + f"Line {result.line}: {result.message}".replace(" in mapping (key-duplicates)", "")
|
|
696
|
+
for result in results
|
|
697
|
+
]
|
|
698
|
+
|
|
699
|
+
return parsed_results
|
|
700
|
+
|
|
701
|
+
|
|
680
702
|
def load_and_validate_platform_config(
|
|
681
703
|
path=PLATFORM_CONFIG_FILE, disable_aws_validation=False, disable_file_check=False
|
|
682
704
|
):
|
|
@@ -684,6 +706,13 @@ def load_and_validate_platform_config(
|
|
|
684
706
|
config_file_check(path)
|
|
685
707
|
try:
|
|
686
708
|
conf = yaml.safe_load(Path(path).read_text())
|
|
709
|
+
duplicate_keys = lint_yaml_for_duplicate_keys(path)
|
|
710
|
+
if duplicate_keys:
|
|
711
|
+
abort_with_error(
|
|
712
|
+
"Duplicate keys found in platform-config:"
|
|
713
|
+
+ os.linesep
|
|
714
|
+
+ os.linesep.join(duplicate_keys)
|
|
715
|
+
)
|
|
687
716
|
validate_platform_config(conf, disable_aws_validation)
|
|
688
717
|
return conf
|
|
689
718
|
except ParserError:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dbt-platform-helper
|
|
3
|
-
Version: 11.
|
|
3
|
+
Version: 11.1.0
|
|
4
4
|
Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Department for Business and Trade Platform Team
|
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
-
Requires-Dist: Jinja2 (
|
|
15
|
+
Requires-Dist: Jinja2 (==3.1.4)
|
|
16
16
|
Requires-Dist: PyYAML (==6.0.1)
|
|
17
17
|
Requires-Dist: aiohttp (>=3.8.4,<4.0.0)
|
|
18
18
|
Requires-Dist: boto3 (>=1.28.24,<2.0.0)
|
|
@@ -34,6 +34,7 @@ Requires-Dist: schema (==0.7.5)
|
|
|
34
34
|
Requires-Dist: semver (>=3.0.2,<4.0.0)
|
|
35
35
|
Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
|
|
36
36
|
Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
|
|
37
|
+
Requires-Dist: yamllint (>=1.35.1,<2.0.0)
|
|
37
38
|
Description-Content-Type: text/markdown
|
|
38
39
|
|
|
39
40
|
# DBT Platform Helper
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
dbt_platform_helper/COMMANDS.md,sha256=
|
|
1
|
+
dbt_platform_helper/COMMANDS.md,sha256=tS87wy3pq_jxScNedIUehkBAqavm9Sp3EhptCVQTeZs,25970
|
|
2
2
|
dbt_platform_helper/README.md,sha256=B0qN2_u_ASqqgkGDWY2iwNGZt_9tUgMb9XqtaTuzYjw,1530
|
|
3
3
|
dbt_platform_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
dbt_platform_helper/addon-plans.yml,sha256=O46a_ODsGG9KXmQY_1XbSGqrpSaHSLDe-SdROzHx8Go,4545
|
|
@@ -10,10 +10,9 @@ dbt_platform_helper/commands/codebase.py,sha256=NchJzH-yxv5mXCe2rPyXVNHmXGEvjFUv
|
|
|
10
10
|
dbt_platform_helper/commands/conduit.py,sha256=BC2cxIX14e7_BEdvuMiq8z85k_4OWL-RwX0QJFJrUak,15436
|
|
11
11
|
dbt_platform_helper/commands/config.py,sha256=NOHea7OAjrl6XHlW6HMLn0m0T5lFPyNH3HXoyCOWsJk,12070
|
|
12
12
|
dbt_platform_helper/commands/copilot.py,sha256=euid0FTlVtwKmBQ6vxt_HxtBdRYiVQvb-9CyrK1-MWc,16724
|
|
13
|
-
dbt_platform_helper/commands/database.py,sha256=
|
|
14
|
-
dbt_platform_helper/commands/database_helpers.py,sha256=7TE6ojEmrIcJ-XQf3MPKeRSuTO5GRBwzm8IoWYmjE3M,4967
|
|
13
|
+
dbt_platform_helper/commands/database.py,sha256=_HnuOxlfVIFGkDotGv0SGb6oWrnm517FSvLv0aGcLJQ,3542
|
|
15
14
|
dbt_platform_helper/commands/dns.py,sha256=o7PkvHktZo0jmqbx0krJTL0R4GtWSf1rF2KDEWor8Ts,35211
|
|
16
|
-
dbt_platform_helper/commands/environment.py,sha256=
|
|
15
|
+
dbt_platform_helper/commands/environment.py,sha256=QrG2ewchP2D1BmG067QpO-rR77F9lbNn2oVlM_zVS-g,7953
|
|
17
16
|
dbt_platform_helper/commands/generate.py,sha256=YLCPb-xcPapGcsLn-7d1Am7BpGp5l0iecIDTOdNGjHk,722
|
|
18
17
|
dbt_platform_helper/commands/notify.py,sha256=kVJ0s78QMiaEWPVKu_bbMko4DW2uJy2fu8-HNJsglyk,3748
|
|
19
18
|
dbt_platform_helper/commands/pipeline.py,sha256=jQGwCRpJ_hXK988XmLHzRBHDWmhFzZb37wa75KuSd0M,5945
|
|
@@ -23,8 +22,12 @@ dbt_platform_helper/constants.py,sha256=fzN2VZt81mspNfdYpNef5_eEjDVsh8GUYmhBMTIf
|
|
|
23
22
|
dbt_platform_helper/custom_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
23
|
dbt_platform_helper/custom_resources/s3_object.py,sha256=0mhLuKD0-vwuN1qmnLCrLo2qL58FvtCjNNjH34kac6Y,2526
|
|
25
24
|
dbt_platform_helper/default-extensions.yml,sha256=SU1ZitskbuEBpvE7efc3s56eAUF11j70brhj_XrNMMo,493
|
|
25
|
+
dbt_platform_helper/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
+
dbt_platform_helper/domain/database_copy.py,sha256=OY7X5wmlb-s0kOn77X24-QWLD13ru27xi9_2gV-st2k,8586
|
|
27
|
+
dbt_platform_helper/domain/maintenance_page.py,sha256=g9rYOeV7-AOMGJL9J8EdB--1xhl8pihOGDcbNK2IKV8,15729
|
|
26
28
|
dbt_platform_helper/exceptions.py,sha256=dCQkqmlt5yycZgblwTtpkwGrP7ANQB_1MRORUWIYI-U,541
|
|
27
29
|
dbt_platform_helper/jinja2_tags.py,sha256=jFyN_Sxmko1GSfvrqRIGQ80CCW8EwlCV3su0ahJPfoE,541
|
|
30
|
+
dbt_platform_helper/providers/load_balancers.py,sha256=e1SPrWbBWq95paSVd3Y5yORIrHAZxcVabBYDjPyUTsU,1430
|
|
28
31
|
dbt_platform_helper/templates/.copilot/config.yml,sha256=J_bA9sCtBdCPBRImpCBRnYvhQd4vpLYIXIU-lq9vbkA,158
|
|
29
32
|
dbt_platform_helper/templates/.copilot/image_build_run.sh,sha256=adYucYXEB-kAgZNjTQo0T6EIAY8sh_xCEvVhWKKQ8mw,164
|
|
30
33
|
dbt_platform_helper/templates/.copilot/phases/build.sh,sha256=umKXePcRvx4XyrRY0fAWIyYFtNjqBI2L8vIJk-V7C60,121
|
|
@@ -67,7 +70,7 @@ dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts,sha25
|
|
|
67
70
|
dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml,sha256=neXXpwjCrNRPTOxec3m8nRIFZ0bI4zq2WaPHf5eSU_Y,1090
|
|
68
71
|
dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml,sha256=oHtRzH27IXJRyORWp7zvtjln-kTf3FgTdc9W_pBFBfU,1480
|
|
69
72
|
dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json,sha256=ZbvoQdcj_k9k1GAD9qHUQcDfQPbMcBPjJwt2mu_S6ho,339
|
|
70
|
-
dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json,sha256=
|
|
73
|
+
dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json,sha256=olH0o2L_csz-05gsjZ-GMKzNZqrkxciaJFUiAt7sYKc,152695
|
|
71
74
|
dbt_platform_helper/templates/pipelines/codebase/overrides/package.json,sha256=XB0Pf63NSsGyowkPGTl1Nki167nRDXJdnxLSN3S_lQg,536
|
|
72
75
|
dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts,sha256=v9m6EziRgFnrhF7inbr1KtuOh75FeC054vaWMoAi-qg,21500
|
|
73
76
|
dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json,sha256=k6KabP-WwhFNgA1AFHNuonTEAnES6eR74jUuYUJEGOM,651
|
|
@@ -84,7 +87,7 @@ dbt_platform_helper/templates/svc/overrides/cfn.patches.yml,sha256=W7-d017akuUq9
|
|
|
84
87
|
dbt_platform_helper/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
85
88
|
dbt_platform_helper/utils/application.py,sha256=FzEoyTUlRJ7bt01cZAg-JHRQM4CqosVLOojdfwaDl6w,4113
|
|
86
89
|
dbt_platform_helper/utils/arn_parser.py,sha256=1jY0elpAe4YL3ulrrCf1YiKmjI-7YXz4gJASqkIFHTc,1294
|
|
87
|
-
dbt_platform_helper/utils/aws.py,sha256=
|
|
90
|
+
dbt_platform_helper/utils/aws.py,sha256=_NM76HOT2EvPwPe0ltuZ2_WH0NYdNLPWEnw7HgVQ6K8,14332
|
|
88
91
|
dbt_platform_helper/utils/click.py,sha256=Fx4y4bbve1zypvog_sgK7tJtCocmzheoEFLBRv1lfdM,2943
|
|
89
92
|
dbt_platform_helper/utils/cloudformation.py,sha256=A1z12IYpapCOngAvZUCc_Ig7S7tuSP3ceeVzVR-5BGY,1053
|
|
90
93
|
dbt_platform_helper/utils/cloudfoundry.py,sha256=GnQ4fVLnDfOdNSrsJjI6ElZHqpgwINeoPn77cUH2UFY,484
|
|
@@ -94,11 +97,11 @@ dbt_platform_helper/utils/manifests.py,sha256=ji3UYHCxq9tTpkm4MlRa2y0-JOYYqq1pWZ
|
|
|
94
97
|
dbt_platform_helper/utils/messages.py,sha256=aLx6s9utt__IqlDdeIYq4n82ERwludu2Zfqy0Q2t-x8,115
|
|
95
98
|
dbt_platform_helper/utils/platform_config.py,sha256=zJYCIsgUk5kNerocTzD7Q2XIrkYhiCu65sx8KQ-d-o8,833
|
|
96
99
|
dbt_platform_helper/utils/template.py,sha256=raRx4QUCVJtKfvJK08Egg6gwWcs3r3V4nPWcJW4xNhA,574
|
|
97
|
-
dbt_platform_helper/utils/validation.py,sha256=
|
|
100
|
+
dbt_platform_helper/utils/validation.py,sha256=5gjq6x_RWGcUcu_te7qGiSBsI563QmBG4e7NLOWN6Go,27545
|
|
98
101
|
dbt_platform_helper/utils/versioning.py,sha256=IBxdocJ8ZyJib38d1ja87tTuFE0iJ4npaDcAHQAKQ58,10825
|
|
99
102
|
platform_helper.py,sha256=1lvPwynKODyi2U-ePKzJyFwRdKPs6_6zAYUPDYzDKMo,2300
|
|
100
|
-
dbt_platform_helper-11.
|
|
101
|
-
dbt_platform_helper-11.
|
|
102
|
-
dbt_platform_helper-11.
|
|
103
|
-
dbt_platform_helper-11.
|
|
104
|
-
dbt_platform_helper-11.
|
|
103
|
+
dbt_platform_helper-11.1.0.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
|
|
104
|
+
dbt_platform_helper-11.1.0.dist-info/METADATA,sha256=jc5BhsQgS20nrQjP0Orgqsu_rHM7UJFriAHin3UFMAQ,3161
|
|
105
|
+
dbt_platform_helper-11.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
106
|
+
dbt_platform_helper-11.1.0.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
|
|
107
|
+
dbt_platform_helper-11.1.0.dist-info/RECORD,,
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
import boto3
|
|
2
|
-
import click
|
|
3
|
-
|
|
4
|
-
from dbt_platform_helper.utils.aws import Vpc
|
|
5
|
-
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
6
|
-
from dbt_platform_helper.utils.aws import get_connection_string
|
|
7
|
-
from dbt_platform_helper.utils.aws import get_vpc_info_by_name
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def run_database_copy_task(
|
|
11
|
-
session: boto3.session.Session,
|
|
12
|
-
account_id: str,
|
|
13
|
-
app: str,
|
|
14
|
-
env: str,
|
|
15
|
-
database: str,
|
|
16
|
-
vpc_config: Vpc,
|
|
17
|
-
is_dump: bool,
|
|
18
|
-
db_connection_string: str,
|
|
19
|
-
):
|
|
20
|
-
client = session.client("ecs")
|
|
21
|
-
action = "dump" if is_dump else "load"
|
|
22
|
-
response = client.run_task(
|
|
23
|
-
taskDefinition=f"arn:aws:ecs:eu-west-2:{account_id}:task-definition/{app}-{env}-{database}-{action}",
|
|
24
|
-
cluster=f"{app}-{env}",
|
|
25
|
-
capacityProviderStrategy=[
|
|
26
|
-
{"capacityProvider": "FARGATE", "weight": 1, "base": 0},
|
|
27
|
-
],
|
|
28
|
-
networkConfiguration={
|
|
29
|
-
"awsvpcConfiguration": {
|
|
30
|
-
"subnets": vpc_config.subnets,
|
|
31
|
-
"securityGroups": vpc_config.security_groups,
|
|
32
|
-
"assignPublicIp": "DISABLED",
|
|
33
|
-
}
|
|
34
|
-
},
|
|
35
|
-
overrides={
|
|
36
|
-
"containerOverrides": [
|
|
37
|
-
{
|
|
38
|
-
"name": f"{app}-{env}-{database}-{action}",
|
|
39
|
-
"environment": [
|
|
40
|
-
{"name": "DATA_COPY_OPERATION", "value": action.upper()},
|
|
41
|
-
{"name": "DB_CONNECTION_STRING", "value": db_connection_string},
|
|
42
|
-
],
|
|
43
|
-
}
|
|
44
|
-
]
|
|
45
|
-
},
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
return response.get("tasks", [{}])[0].get("taskArn")
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class DatabaseCopy:
|
|
52
|
-
def __init__(
|
|
53
|
-
self,
|
|
54
|
-
account_id,
|
|
55
|
-
app,
|
|
56
|
-
env,
|
|
57
|
-
database,
|
|
58
|
-
vpc_name,
|
|
59
|
-
get_session_fn=get_aws_session_or_abort,
|
|
60
|
-
run_database_copy_fn=run_database_copy_task,
|
|
61
|
-
vpc_config_fn=get_vpc_info_by_name,
|
|
62
|
-
db_connection_string_fn=get_connection_string,
|
|
63
|
-
input_fn=click.prompt,
|
|
64
|
-
echo_fn=click.secho,
|
|
65
|
-
):
|
|
66
|
-
self.account_id = account_id
|
|
67
|
-
self.app = app
|
|
68
|
-
self.env = env
|
|
69
|
-
self.database = database
|
|
70
|
-
self.vpc_name = vpc_name
|
|
71
|
-
self.get_session_fn = get_session_fn
|
|
72
|
-
self.run_database_copy_fn = run_database_copy_fn
|
|
73
|
-
self.vpc_config_fn = vpc_config_fn
|
|
74
|
-
self.db_connection_string_fn = db_connection_string_fn
|
|
75
|
-
self.input_fn = input_fn
|
|
76
|
-
self.echo_fn = echo_fn
|
|
77
|
-
|
|
78
|
-
def _execute_operation(self, is_dump):
|
|
79
|
-
session = self.get_session_fn()
|
|
80
|
-
vpc_config = self.vpc_config_fn(session, self.app, self.env, self.vpc_name)
|
|
81
|
-
database_identifier = f"{self.app}-{self.env}-{self.database}"
|
|
82
|
-
db_connection_string = self.db_connection_string_fn(
|
|
83
|
-
session, self.app, self.env, database_identifier
|
|
84
|
-
)
|
|
85
|
-
task_arn = self.run_database_copy_fn(
|
|
86
|
-
session,
|
|
87
|
-
self.account_id,
|
|
88
|
-
self.app,
|
|
89
|
-
self.env,
|
|
90
|
-
self.database,
|
|
91
|
-
vpc_config,
|
|
92
|
-
is_dump,
|
|
93
|
-
db_connection_string,
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
self.echo_fn(
|
|
97
|
-
f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
|
|
98
|
-
fg="green",
|
|
99
|
-
)
|
|
100
|
-
self.tail_logs(is_dump)
|
|
101
|
-
self.wait_for_task_to_stop(task_arn)
|
|
102
|
-
|
|
103
|
-
def dump(self):
|
|
104
|
-
self._execute_operation(True)
|
|
105
|
-
|
|
106
|
-
def load(self):
|
|
107
|
-
if self.is_confirmed_ready_to_load():
|
|
108
|
-
self._execute_operation(False)
|
|
109
|
-
|
|
110
|
-
def is_confirmed_ready_to_load(self):
|
|
111
|
-
user_input = self.input_fn(
|
|
112
|
-
f"Are all tasks using {self.database} in the {self.env} environment stopped? (y/n)"
|
|
113
|
-
)
|
|
114
|
-
return user_input.lower().strip() in ["y", "yes"]
|
|
115
|
-
|
|
116
|
-
def tail_logs(self, is_dump: bool):
|
|
117
|
-
action = "dump" if is_dump else "load"
|
|
118
|
-
log_group_name = f"/ecs/{self.app}-{self.env}-{self.database}-{action}"
|
|
119
|
-
log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id}:log-group:{log_group_name}"
|
|
120
|
-
self.echo_fn(f"Tailing logs for {log_group_name}", fg="yellow")
|
|
121
|
-
session = self.get_session_fn()
|
|
122
|
-
response = session.client("logs").start_live_tail(logGroupIdentifiers=[log_group_arn])
|
|
123
|
-
|
|
124
|
-
stopped = False
|
|
125
|
-
for data in response["responseStream"]:
|
|
126
|
-
if stopped:
|
|
127
|
-
break
|
|
128
|
-
results = data.get("sessionUpdate", {}).get("sessionResults", [])
|
|
129
|
-
for result in results:
|
|
130
|
-
message = result.get("message")
|
|
131
|
-
|
|
132
|
-
if message:
|
|
133
|
-
if message.startswith("Stopping data "):
|
|
134
|
-
stopped = True
|
|
135
|
-
self.echo_fn(message)
|
|
136
|
-
|
|
137
|
-
def wait_for_task_to_stop(self, task_arn):
|
|
138
|
-
self.echo_fn("Waiting for task to complete", fg="yellow")
|
|
139
|
-
client = self.get_session_fn().client("ecs")
|
|
140
|
-
waiter = client.get_waiter("tasks_stopped")
|
|
141
|
-
waiter.wait(
|
|
142
|
-
cluster=f"{self.app}-{self.env}",
|
|
143
|
-
tasks=[task_arn],
|
|
144
|
-
WaiterConfig={"Delay": 6, "MaxAttempts": 300},
|
|
145
|
-
)
|
|
File without changes
|
|
File without changes
|
{dbt_platform_helper-11.0.0.dist-info → dbt_platform_helper-11.1.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|