dbt-platform-helper 11.0.0__py3-none-any.whl → 11.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

@@ -0,0 +1,51 @@
1
+ import boto3
2
+
3
+
4
+ def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
5
+ lb_client = session.client("elbv2")
6
+
7
+ describe_response = lb_client.describe_load_balancers()
8
+ load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
9
+
10
+ load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
11
+
12
+ load_balancer_arn = None
13
+ for lb in load_balancers:
14
+ tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
15
+ if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
16
+ load_balancer_arn = lb["ResourceArn"]
17
+
18
+ if not load_balancer_arn:
19
+ raise LoadBalancerNotFoundError()
20
+
21
+ return load_balancer_arn
22
+
23
+
24
+ def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
25
+ load_balancer_arn = find_load_balancer(session, app, env)
26
+ lb_client = session.client("elbv2")
27
+ listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
28
+
29
+ listener_arn = None
30
+
31
+ try:
32
+ listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
33
+ except StopIteration:
34
+ pass
35
+
36
+ if not listener_arn:
37
+ raise ListenerNotFoundError()
38
+
39
+ return listener_arn
40
+
41
+
42
+ class LoadBalancerNotFoundError(Exception):
43
+ pass
44
+
45
+
46
+ class ListenerNotFoundError(Exception):
47
+ pass
48
+
49
+
50
+ class ListenerRuleNotFoundError(Exception):
51
+ pass
@@ -3321,12 +3321,13 @@
3321
3321
  "dev": true
3322
3322
  },
3323
3323
  "node_modules/micromatch": {
3324
- "version": "4.0.5",
3325
- "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
3326
- "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
3324
+ "version": "4.0.8",
3325
+ "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
3326
+ "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
3327
3327
  "dev": true,
3328
+ "license": "MIT",
3328
3329
  "dependencies": {
3329
- "braces": "^3.0.2",
3330
+ "braces": "^3.0.3",
3330
3331
  "picomatch": "^2.3.1"
3331
3332
  },
3332
3333
  "engines": {
@@ -7,6 +7,7 @@ from typing import Tuple
7
7
 
8
8
  import boto3
9
9
  import botocore
10
+ import botocore.exceptions
10
11
  import click
11
12
  import yaml
12
13
  from boto3 import Session
@@ -20,62 +21,71 @@ AWS_SESSION_CACHE = {}
20
21
 
21
22
 
22
23
  def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
23
- aws_profile = aws_profile if aws_profile else os.getenv("AWS_PROFILE")
24
+ REFRESH_TOKEN_MESSAGE = (
25
+ "To refresh this SSO session run `aws sso login` with the corresponding profile"
26
+ )
27
+ aws_profile = aws_profile or os.getenv("AWS_PROFILE")
24
28
  if aws_profile in AWS_SESSION_CACHE:
25
29
  return AWS_SESSION_CACHE[aws_profile]
26
30
 
27
- # Check that the aws profile exists and is set.
28
- click.secho(f"""Checking AWS connection for profile "{aws_profile}"...""", fg="cyan")
31
+ click.secho(f'Checking AWS connection for profile "{aws_profile}"...', fg="cyan")
29
32
 
30
33
  try:
31
34
  session = boto3.session.Session(profile_name=aws_profile)
35
+ sts = session.client("sts")
36
+ account_id, user_id = get_account_details(sts)
37
+ click.secho("Credentials are valid.", fg="green")
38
+
32
39
  except botocore.exceptions.ProfileNotFound:
33
- click.secho(f"""AWS profile "{aws_profile}" is not configured.""", fg="red")
34
- exit(1)
40
+ _handle_error(f'AWS profile "{aws_profile}" is not configured.')
35
41
  except botocore.exceptions.ClientError as e:
36
42
  if e.response["Error"]["Code"] == "ExpiredToken":
37
- click.secho(
38
- f"Credentials are NOT valid. \nPlease login with: aws sso login --profile {aws_profile}",
39
- fg="red",
43
+ _handle_error(
44
+ f"Credentials are NOT valid. \nPlease login with: aws sso login --profile {aws_profile}"
40
45
  )
41
- exit(1)
42
-
43
- sts = session.client("sts")
44
- try:
45
- account_id, user_id = get_account_details(sts)
46
- click.secho("Credentials are valid.", fg="green")
47
- except (
48
- botocore.exceptions.UnauthorizedSSOTokenError,
49
- botocore.exceptions.TokenRetrievalError,
50
- botocore.exceptions.SSOTokenLoadError,
51
- ):
52
- click.secho(
53
- "The SSO session associated with this profile has expired or is otherwise invalid."
54
- "To refresh this SSO session run `aws sso login` with the corresponding profile",
55
- fg="red",
46
+ except botocore.exceptions.NoCredentialsError:
47
+ _handle_error("There are no credentials set for this session.", REFRESH_TOKEN_MESSAGE)
48
+ except botocore.exceptions.UnauthorizedSSOTokenError:
49
+ _handle_error("The SSO Token used for this session is unauthorised.", REFRESH_TOKEN_MESSAGE)
50
+ except botocore.exceptions.TokenRetrievalError:
51
+ _handle_error("Unable to retrieve the Token for this session.", REFRESH_TOKEN_MESSAGE)
52
+ except botocore.exceptions.SSOTokenLoadError:
53
+ _handle_error(
54
+ "The SSO session associated with this profile has expired, is not set or is otherwise invalid.",
55
+ REFRESH_TOKEN_MESSAGE,
56
56
  )
57
- exit(1)
58
57
 
59
58
  alias_client = session.client("iam")
60
- account_name = alias_client.list_account_aliases()["AccountAliases"]
59
+ account_name = alias_client.list_account_aliases().get("AccountAliases", [])
60
+
61
+ _log_account_info(account_name, account_id)
62
+
63
+ click.echo(
64
+ click.style("User: ", fg="yellow")
65
+ + click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True)
66
+ )
67
+
68
+ AWS_SESSION_CACHE[aws_profile] = session
69
+ return session
70
+
71
+
72
+ def _handle_error(message: str, refresh_token_message: str = None) -> None:
73
+ full_message = message + (" " + refresh_token_message if refresh_token_message else "")
74
+ click.secho(full_message, fg="red")
75
+ exit(1)
76
+
77
+
78
+ def _log_account_info(account_name: list, account_id: str) -> None:
61
79
  if account_name:
62
80
  click.echo(
63
81
  click.style("Logged in with AWS account: ", fg="yellow")
64
- + click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True),
82
+ + click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True)
65
83
  )
66
84
  else:
67
85
  click.echo(
68
86
  click.style("Logged in with AWS account id: ", fg="yellow")
69
- + click.style(f"{account_id}", fg="white", bold=True),
87
+ + click.style(f"{account_id}", fg="white", bold=True)
70
88
  )
71
- click.echo(
72
- click.style("User: ", fg="yellow")
73
- + click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True),
74
- )
75
-
76
- AWS_SESSION_CACHE[aws_profile] = session
77
-
78
- return session
79
89
 
80
90
 
81
91
  class NoProfileForAccountIdError(Exception):
@@ -362,12 +372,12 @@ def get_connection_string(
362
372
 
363
373
 
364
374
  class Vpc:
365
- def __init__(self, subnets, security_groups):
375
+ def __init__(self, subnets: list[str], security_groups: list[str]):
366
376
  self.subnets = subnets
367
377
  self.security_groups = security_groups
368
378
 
369
379
 
370
- def get_vpc_info_by_name(session, app, env, vpc_name):
380
+ def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) -> Vpc:
371
381
  ec2_client = session.client("ec2")
372
382
  vpc_response = ec2_client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [vpc_name]}])
373
383
 
@@ -1,4 +1,5 @@
1
1
  import ipaddress
2
+ import os
2
3
  import re
3
4
  from pathlib import Path
4
5
 
@@ -11,6 +12,8 @@ from schema import Regex
11
12
  from schema import Schema
12
13
  from schema import SchemaError
13
14
  from yaml.parser import ParserError
15
+ from yamllint import config
16
+ from yamllint import linter
14
17
 
15
18
  from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
16
19
  from dbt_platform_helper.constants import ENVIRONMENTS_KEY
@@ -677,6 +680,25 @@ def _validate_environment_pipelines_triggers(config):
677
680
  abort_with_error(error_message + "\n ".join(errors))
678
681
 
679
682
 
683
+ def lint_yaml_for_duplicate_keys(file_path):
684
+ lint_yaml_config = """
685
+ rules:
686
+ key-duplicates: enable
687
+ """
688
+ yaml_config = config.YamlLintConfig(lint_yaml_config)
689
+
690
+ with open(file_path, "r") as yaml_file:
691
+ file_contents = yaml_file.read()
692
+ results = linter.run(file_contents, yaml_config)
693
+
694
+ parsed_results = [
695
+ "\t" + f"Line {result.line}: {result.message}".replace(" in mapping (key-duplicates)", "")
696
+ for result in results
697
+ ]
698
+
699
+ return parsed_results
700
+
701
+
680
702
  def load_and_validate_platform_config(
681
703
  path=PLATFORM_CONFIG_FILE, disable_aws_validation=False, disable_file_check=False
682
704
  ):
@@ -684,6 +706,13 @@ def load_and_validate_platform_config(
684
706
  config_file_check(path)
685
707
  try:
686
708
  conf = yaml.safe_load(Path(path).read_text())
709
+ duplicate_keys = lint_yaml_for_duplicate_keys(path)
710
+ if duplicate_keys:
711
+ abort_with_error(
712
+ "Duplicate keys found in platform-config:"
713
+ + os.linesep
714
+ + os.linesep.join(duplicate_keys)
715
+ )
687
716
  validate_platform_config(conf, disable_aws_validation)
688
717
  return conf
689
718
  except ParserError:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbt-platform-helper
3
- Version: 11.0.0
3
+ Version: 11.1.0
4
4
  Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
5
5
  License: MIT
6
6
  Author: Department for Business and Trade Platform Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.9
12
12
  Classifier: Programming Language :: Python :: 3.10
13
13
  Classifier: Programming Language :: Python :: 3.11
14
14
  Classifier: Programming Language :: Python :: 3.12
15
- Requires-Dist: Jinja2 (>=3.1.3,<4.0.0)
15
+ Requires-Dist: Jinja2 (==3.1.4)
16
16
  Requires-Dist: PyYAML (==6.0.1)
17
17
  Requires-Dist: aiohttp (>=3.8.4,<4.0.0)
18
18
  Requires-Dist: boto3 (>=1.28.24,<2.0.0)
@@ -34,6 +34,7 @@ Requires-Dist: schema (==0.7.5)
34
34
  Requires-Dist: semver (>=3.0.2,<4.0.0)
35
35
  Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
36
36
  Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
37
+ Requires-Dist: yamllint (>=1.35.1,<2.0.0)
37
38
  Description-Content-Type: text/markdown
38
39
 
39
40
  # DBT Platform Helper
@@ -1,4 +1,4 @@
1
- dbt_platform_helper/COMMANDS.md,sha256=u9g0Zf1RYk-pzE4hjSKg15EMpXglzSA9qEg3QWv559M,23843
1
+ dbt_platform_helper/COMMANDS.md,sha256=tS87wy3pq_jxScNedIUehkBAqavm9Sp3EhptCVQTeZs,25970
2
2
  dbt_platform_helper/README.md,sha256=B0qN2_u_ASqqgkGDWY2iwNGZt_9tUgMb9XqtaTuzYjw,1530
3
3
  dbt_platform_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
4
  dbt_platform_helper/addon-plans.yml,sha256=O46a_ODsGG9KXmQY_1XbSGqrpSaHSLDe-SdROzHx8Go,4545
@@ -10,10 +10,9 @@ dbt_platform_helper/commands/codebase.py,sha256=NchJzH-yxv5mXCe2rPyXVNHmXGEvjFUv
10
10
  dbt_platform_helper/commands/conduit.py,sha256=BC2cxIX14e7_BEdvuMiq8z85k_4OWL-RwX0QJFJrUak,15436
11
11
  dbt_platform_helper/commands/config.py,sha256=NOHea7OAjrl6XHlW6HMLn0m0T5lFPyNH3HXoyCOWsJk,12070
12
12
  dbt_platform_helper/commands/copilot.py,sha256=euid0FTlVtwKmBQ6vxt_HxtBdRYiVQvb-9CyrK1-MWc,16724
13
- dbt_platform_helper/commands/database.py,sha256=r7GXfyt_1Wj7KlGilnPopfHXuJ8Q7x35RVh9XKiAZUY,1178
14
- dbt_platform_helper/commands/database_helpers.py,sha256=7TE6ojEmrIcJ-XQf3MPKeRSuTO5GRBwzm8IoWYmjE3M,4967
13
+ dbt_platform_helper/commands/database.py,sha256=_HnuOxlfVIFGkDotGv0SGb6oWrnm517FSvLv0aGcLJQ,3542
15
14
  dbt_platform_helper/commands/dns.py,sha256=o7PkvHktZo0jmqbx0krJTL0R4GtWSf1rF2KDEWor8Ts,35211
16
- dbt_platform_helper/commands/environment.py,sha256=VNr7G1QstM8INGs8jOxL1jQRqDcWx2Q0jaaBXtbHhys,24819
15
+ dbt_platform_helper/commands/environment.py,sha256=QrG2ewchP2D1BmG067QpO-rR77F9lbNn2oVlM_zVS-g,7953
17
16
  dbt_platform_helper/commands/generate.py,sha256=YLCPb-xcPapGcsLn-7d1Am7BpGp5l0iecIDTOdNGjHk,722
18
17
  dbt_platform_helper/commands/notify.py,sha256=kVJ0s78QMiaEWPVKu_bbMko4DW2uJy2fu8-HNJsglyk,3748
19
18
  dbt_platform_helper/commands/pipeline.py,sha256=jQGwCRpJ_hXK988XmLHzRBHDWmhFzZb37wa75KuSd0M,5945
@@ -23,8 +22,12 @@ dbt_platform_helper/constants.py,sha256=fzN2VZt81mspNfdYpNef5_eEjDVsh8GUYmhBMTIf
23
22
  dbt_platform_helper/custom_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
23
  dbt_platform_helper/custom_resources/s3_object.py,sha256=0mhLuKD0-vwuN1qmnLCrLo2qL58FvtCjNNjH34kac6Y,2526
25
24
  dbt_platform_helper/default-extensions.yml,sha256=SU1ZitskbuEBpvE7efc3s56eAUF11j70brhj_XrNMMo,493
25
+ dbt_platform_helper/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
+ dbt_platform_helper/domain/database_copy.py,sha256=OY7X5wmlb-s0kOn77X24-QWLD13ru27xi9_2gV-st2k,8586
27
+ dbt_platform_helper/domain/maintenance_page.py,sha256=g9rYOeV7-AOMGJL9J8EdB--1xhl8pihOGDcbNK2IKV8,15729
26
28
  dbt_platform_helper/exceptions.py,sha256=dCQkqmlt5yycZgblwTtpkwGrP7ANQB_1MRORUWIYI-U,541
27
29
  dbt_platform_helper/jinja2_tags.py,sha256=jFyN_Sxmko1GSfvrqRIGQ80CCW8EwlCV3su0ahJPfoE,541
30
+ dbt_platform_helper/providers/load_balancers.py,sha256=e1SPrWbBWq95paSVd3Y5yORIrHAZxcVabBYDjPyUTsU,1430
28
31
  dbt_platform_helper/templates/.copilot/config.yml,sha256=J_bA9sCtBdCPBRImpCBRnYvhQd4vpLYIXIU-lq9vbkA,158
29
32
  dbt_platform_helper/templates/.copilot/image_build_run.sh,sha256=adYucYXEB-kAgZNjTQo0T6EIAY8sh_xCEvVhWKKQ8mw,164
30
33
  dbt_platform_helper/templates/.copilot/phases/build.sh,sha256=umKXePcRvx4XyrRY0fAWIyYFtNjqBI2L8vIJk-V7C60,121
@@ -67,7 +70,7 @@ dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts,sha25
67
70
  dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml,sha256=neXXpwjCrNRPTOxec3m8nRIFZ0bI4zq2WaPHf5eSU_Y,1090
68
71
  dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml,sha256=oHtRzH27IXJRyORWp7zvtjln-kTf3FgTdc9W_pBFBfU,1480
69
72
  dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json,sha256=ZbvoQdcj_k9k1GAD9qHUQcDfQPbMcBPjJwt2mu_S6ho,339
70
- dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json,sha256=Is83o58QXbeg2SkHmR79ATt91aFhVbO7kb1VF0qXpY8,152671
73
+ dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json,sha256=olH0o2L_csz-05gsjZ-GMKzNZqrkxciaJFUiAt7sYKc,152695
71
74
  dbt_platform_helper/templates/pipelines/codebase/overrides/package.json,sha256=XB0Pf63NSsGyowkPGTl1Nki167nRDXJdnxLSN3S_lQg,536
72
75
  dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts,sha256=v9m6EziRgFnrhF7inbr1KtuOh75FeC054vaWMoAi-qg,21500
73
76
  dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json,sha256=k6KabP-WwhFNgA1AFHNuonTEAnES6eR74jUuYUJEGOM,651
@@ -84,7 +87,7 @@ dbt_platform_helper/templates/svc/overrides/cfn.patches.yml,sha256=W7-d017akuUq9
84
87
  dbt_platform_helper/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
85
88
  dbt_platform_helper/utils/application.py,sha256=FzEoyTUlRJ7bt01cZAg-JHRQM4CqosVLOojdfwaDl6w,4113
86
89
  dbt_platform_helper/utils/arn_parser.py,sha256=1jY0elpAe4YL3ulrrCf1YiKmjI-7YXz4gJASqkIFHTc,1294
87
- dbt_platform_helper/utils/aws.py,sha256=H4oFnzJf0WXQ8xi19QlC3zKKA3HWPp5wNfm9hZGgMsI,13683
90
+ dbt_platform_helper/utils/aws.py,sha256=_NM76HOT2EvPwPe0ltuZ2_WH0NYdNLPWEnw7HgVQ6K8,14332
88
91
  dbt_platform_helper/utils/click.py,sha256=Fx4y4bbve1zypvog_sgK7tJtCocmzheoEFLBRv1lfdM,2943
89
92
  dbt_platform_helper/utils/cloudformation.py,sha256=A1z12IYpapCOngAvZUCc_Ig7S7tuSP3ceeVzVR-5BGY,1053
90
93
  dbt_platform_helper/utils/cloudfoundry.py,sha256=GnQ4fVLnDfOdNSrsJjI6ElZHqpgwINeoPn77cUH2UFY,484
@@ -94,11 +97,11 @@ dbt_platform_helper/utils/manifests.py,sha256=ji3UYHCxq9tTpkm4MlRa2y0-JOYYqq1pWZ
94
97
  dbt_platform_helper/utils/messages.py,sha256=aLx6s9utt__IqlDdeIYq4n82ERwludu2Zfqy0Q2t-x8,115
95
98
  dbt_platform_helper/utils/platform_config.py,sha256=zJYCIsgUk5kNerocTzD7Q2XIrkYhiCu65sx8KQ-d-o8,833
96
99
  dbt_platform_helper/utils/template.py,sha256=raRx4QUCVJtKfvJK08Egg6gwWcs3r3V4nPWcJW4xNhA,574
97
- dbt_platform_helper/utils/validation.py,sha256=fsUsEjabAtoswdEzyRBdVUdgRqYJeoE3PgNU-6kTb3k,26713
100
+ dbt_platform_helper/utils/validation.py,sha256=5gjq6x_RWGcUcu_te7qGiSBsI563QmBG4e7NLOWN6Go,27545
98
101
  dbt_platform_helper/utils/versioning.py,sha256=IBxdocJ8ZyJib38d1ja87tTuFE0iJ4npaDcAHQAKQ58,10825
99
102
  platform_helper.py,sha256=1lvPwynKODyi2U-ePKzJyFwRdKPs6_6zAYUPDYzDKMo,2300
100
- dbt_platform_helper-11.0.0.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
101
- dbt_platform_helper-11.0.0.dist-info/METADATA,sha256=72Dqxw1nN3tTWs-YTsW1izHx-rTIsc9o593g2RwuuXI,3126
102
- dbt_platform_helper-11.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
103
- dbt_platform_helper-11.0.0.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
104
- dbt_platform_helper-11.0.0.dist-info/RECORD,,
103
+ dbt_platform_helper-11.1.0.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
104
+ dbt_platform_helper-11.1.0.dist-info/METADATA,sha256=jc5BhsQgS20nrQjP0Orgqsu_rHM7UJFriAHin3UFMAQ,3161
105
+ dbt_platform_helper-11.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
106
+ dbt_platform_helper-11.1.0.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
107
+ dbt_platform_helper-11.1.0.dist-info/RECORD,,
@@ -1,145 +0,0 @@
1
- import boto3
2
- import click
3
-
4
- from dbt_platform_helper.utils.aws import Vpc
5
- from dbt_platform_helper.utils.aws import get_aws_session_or_abort
6
- from dbt_platform_helper.utils.aws import get_connection_string
7
- from dbt_platform_helper.utils.aws import get_vpc_info_by_name
8
-
9
-
10
- def run_database_copy_task(
11
- session: boto3.session.Session,
12
- account_id: str,
13
- app: str,
14
- env: str,
15
- database: str,
16
- vpc_config: Vpc,
17
- is_dump: bool,
18
- db_connection_string: str,
19
- ):
20
- client = session.client("ecs")
21
- action = "dump" if is_dump else "load"
22
- response = client.run_task(
23
- taskDefinition=f"arn:aws:ecs:eu-west-2:{account_id}:task-definition/{app}-{env}-{database}-{action}",
24
- cluster=f"{app}-{env}",
25
- capacityProviderStrategy=[
26
- {"capacityProvider": "FARGATE", "weight": 1, "base": 0},
27
- ],
28
- networkConfiguration={
29
- "awsvpcConfiguration": {
30
- "subnets": vpc_config.subnets,
31
- "securityGroups": vpc_config.security_groups,
32
- "assignPublicIp": "DISABLED",
33
- }
34
- },
35
- overrides={
36
- "containerOverrides": [
37
- {
38
- "name": f"{app}-{env}-{database}-{action}",
39
- "environment": [
40
- {"name": "DATA_COPY_OPERATION", "value": action.upper()},
41
- {"name": "DB_CONNECTION_STRING", "value": db_connection_string},
42
- ],
43
- }
44
- ]
45
- },
46
- )
47
-
48
- return response.get("tasks", [{}])[0].get("taskArn")
49
-
50
-
51
- class DatabaseCopy:
52
- def __init__(
53
- self,
54
- account_id,
55
- app,
56
- env,
57
- database,
58
- vpc_name,
59
- get_session_fn=get_aws_session_or_abort,
60
- run_database_copy_fn=run_database_copy_task,
61
- vpc_config_fn=get_vpc_info_by_name,
62
- db_connection_string_fn=get_connection_string,
63
- input_fn=click.prompt,
64
- echo_fn=click.secho,
65
- ):
66
- self.account_id = account_id
67
- self.app = app
68
- self.env = env
69
- self.database = database
70
- self.vpc_name = vpc_name
71
- self.get_session_fn = get_session_fn
72
- self.run_database_copy_fn = run_database_copy_fn
73
- self.vpc_config_fn = vpc_config_fn
74
- self.db_connection_string_fn = db_connection_string_fn
75
- self.input_fn = input_fn
76
- self.echo_fn = echo_fn
77
-
78
- def _execute_operation(self, is_dump):
79
- session = self.get_session_fn()
80
- vpc_config = self.vpc_config_fn(session, self.app, self.env, self.vpc_name)
81
- database_identifier = f"{self.app}-{self.env}-{self.database}"
82
- db_connection_string = self.db_connection_string_fn(
83
- session, self.app, self.env, database_identifier
84
- )
85
- task_arn = self.run_database_copy_fn(
86
- session,
87
- self.account_id,
88
- self.app,
89
- self.env,
90
- self.database,
91
- vpc_config,
92
- is_dump,
93
- db_connection_string,
94
- )
95
-
96
- self.echo_fn(
97
- f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
98
- fg="green",
99
- )
100
- self.tail_logs(is_dump)
101
- self.wait_for_task_to_stop(task_arn)
102
-
103
- def dump(self):
104
- self._execute_operation(True)
105
-
106
- def load(self):
107
- if self.is_confirmed_ready_to_load():
108
- self._execute_operation(False)
109
-
110
- def is_confirmed_ready_to_load(self):
111
- user_input = self.input_fn(
112
- f"Are all tasks using {self.database} in the {self.env} environment stopped? (y/n)"
113
- )
114
- return user_input.lower().strip() in ["y", "yes"]
115
-
116
- def tail_logs(self, is_dump: bool):
117
- action = "dump" if is_dump else "load"
118
- log_group_name = f"/ecs/{self.app}-{self.env}-{self.database}-{action}"
119
- log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id}:log-group:{log_group_name}"
120
- self.echo_fn(f"Tailing logs for {log_group_name}", fg="yellow")
121
- session = self.get_session_fn()
122
- response = session.client("logs").start_live_tail(logGroupIdentifiers=[log_group_arn])
123
-
124
- stopped = False
125
- for data in response["responseStream"]:
126
- if stopped:
127
- break
128
- results = data.get("sessionUpdate", {}).get("sessionResults", [])
129
- for result in results:
130
- message = result.get("message")
131
-
132
- if message:
133
- if message.startswith("Stopping data "):
134
- stopped = True
135
- self.echo_fn(message)
136
-
137
- def wait_for_task_to_stop(self, task_arn):
138
- self.echo_fn("Waiting for task to complete", fg="yellow")
139
- client = self.get_session_fn().client("ecs")
140
- waiter = client.get_waiter("tasks_stopped")
141
- waiter.wait(
142
- cluster=f"{self.app}-{self.env}",
143
- tasks=[task_arn],
144
- WaiterConfig={"Delay": 6, "MaxAttempts": 300},
145
- )