dbt-platform-helper 13.1.2__py3-none-any.whl → 13.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/commands/application.py +2 -4
- dbt_platform_helper/commands/codebase.py +1 -3
- dbt_platform_helper/commands/conduit.py +1 -3
- dbt_platform_helper/commands/config.py +14 -14
- dbt_platform_helper/commands/copilot.py +6 -4
- dbt_platform_helper/commands/environment.py +5 -5
- dbt_platform_helper/commands/generate.py +2 -3
- dbt_platform_helper/commands/notify.py +1 -3
- dbt_platform_helper/commands/pipeline.py +1 -3
- dbt_platform_helper/commands/secrets.py +1 -3
- dbt_platform_helper/commands/version.py +2 -2
- dbt_platform_helper/domain/codebase.py +10 -2
- dbt_platform_helper/domain/copilot.py +3 -0
- dbt_platform_helper/domain/maintenance_page.py +42 -15
- dbt_platform_helper/domain/pipelines.py +1 -1
- dbt_platform_helper/domain/terraform_environment.py +1 -1
- dbt_platform_helper/domain/versioning.py +125 -35
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/config_validator.py +15 -14
- dbt_platform_helper/providers/copilot.py +1 -1
- dbt_platform_helper/providers/io.py +3 -0
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +26 -15
- dbt_platform_helper/providers/semantic_version.py +7 -4
- dbt_platform_helper/providers/version.py +21 -3
- dbt_platform_helper/providers/yaml_file.py +0 -1
- dbt_platform_helper/utils/aws.py +27 -4
- dbt_platform_helper/utils/files.py +0 -6
- dbt_platform_helper/utils/{versioning.py → tool_versioning.py} +11 -6
- {dbt_platform_helper-13.1.2.dist-info → dbt_platform_helper-13.2.0.dist-info}/METADATA +1 -1
- {dbt_platform_helper-13.1.2.dist-info → dbt_platform_helper-13.2.0.dist-info}/RECORD +38 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/platform_helper_versioning.py +0 -107
- dbt_platform_helper/providers/redis.py +0 -34
- /dbt_platform_helper/providers/{aws.py → aws/exceptions.py} +0 -0
- {dbt_platform_helper-13.1.2.dist-info → dbt_platform_helper-13.2.0.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-13.1.2.dist-info → dbt_platform_helper-13.2.0.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-13.1.2.dist-info → dbt_platform_helper-13.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,67 +1,157 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
4
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
2
5
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
3
|
-
from dbt_platform_helper.providers.
|
|
4
|
-
|
|
6
|
+
from dbt_platform_helper.providers.semantic_version import (
|
|
7
|
+
IncompatibleMajorVersionException,
|
|
8
|
+
)
|
|
9
|
+
from dbt_platform_helper.providers.semantic_version import (
|
|
10
|
+
IncompatibleMinorVersionException,
|
|
5
11
|
)
|
|
6
12
|
from dbt_platform_helper.providers.semantic_version import PlatformHelperVersionStatus
|
|
7
13
|
from dbt_platform_helper.providers.semantic_version import SemanticVersion
|
|
8
|
-
from dbt_platform_helper.
|
|
14
|
+
from dbt_platform_helper.providers.version import DeprecatedVersionFileVersionProvider
|
|
15
|
+
from dbt_platform_helper.providers.version import InstalledVersionProvider
|
|
16
|
+
from dbt_platform_helper.providers.version import PyPiVersionProvider
|
|
17
|
+
from dbt_platform_helper.providers.yaml_file import YamlFileProvider
|
|
9
18
|
|
|
10
19
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
super().__init__(f"""Platform helper version could not be resolved.""")
|
|
20
|
+
def running_as_installed_package():
|
|
21
|
+
return "site-packages" in __file__
|
|
14
22
|
|
|
15
23
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
self.io = io or ClickIOProvider()
|
|
19
|
-
self.platform_helper_versioning = platform_helper_versioning or PlatformHelperVersioning(
|
|
20
|
-
io=self.io
|
|
21
|
-
)
|
|
24
|
+
def skip_version_checks():
|
|
25
|
+
return not running_as_installed_package() or "PLATFORM_TOOLS_SKIP_VERSION_CHECK" in os.environ
|
|
22
26
|
|
|
23
|
-
def get_required_platform_helper_version(
|
|
24
|
-
self, pipeline: str = None, version_status: PlatformHelperVersionStatus = None
|
|
25
|
-
) -> str:
|
|
26
|
-
pipeline_version = version_status.pipeline_overrides.get(pipeline)
|
|
27
|
-
version_precedence = [
|
|
28
|
-
pipeline_version,
|
|
29
|
-
version_status.platform_config_default,
|
|
30
|
-
version_status.deprecated_version_file,
|
|
31
|
-
]
|
|
32
|
-
non_null_version_precedence = [
|
|
33
|
-
f"{v}" if isinstance(v, SemanticVersion) else v for v in version_precedence if v
|
|
34
|
-
]
|
|
35
27
|
|
|
36
|
-
|
|
28
|
+
class PlatformHelperVersionNotFoundException(PlatformException):
|
|
29
|
+
def __init__(self):
|
|
30
|
+
super().__init__(f"""Platform helper version could not be resolved.""")
|
|
37
31
|
|
|
38
|
-
if not out:
|
|
39
|
-
raise PlatformHelperVersionNotFoundException
|
|
40
32
|
|
|
41
|
-
|
|
33
|
+
class PlatformHelperVersioning:
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
io: ClickIOProvider = ClickIOProvider(),
|
|
37
|
+
version_file_version_provider: DeprecatedVersionFileVersionProvider = DeprecatedVersionFileVersionProvider(
|
|
38
|
+
YamlFileProvider
|
|
39
|
+
),
|
|
40
|
+
config_provider: ConfigProvider = ConfigProvider(),
|
|
41
|
+
pypi_provider: PyPiVersionProvider = PyPiVersionProvider,
|
|
42
|
+
installed_version_provider: InstalledVersionProvider = InstalledVersionProvider(),
|
|
43
|
+
skip_versioning_checks: bool = None,
|
|
44
|
+
):
|
|
45
|
+
self.io = io
|
|
46
|
+
self.version_file_version_provider = version_file_version_provider
|
|
47
|
+
self.config_provider = config_provider
|
|
48
|
+
self.pypi_provider = pypi_provider
|
|
49
|
+
self.installed_version_provider = installed_version_provider
|
|
50
|
+
self.skip_versioning_checks = (
|
|
51
|
+
skip_versioning_checks if skip_versioning_checks is not None else skip_version_checks()
|
|
52
|
+
)
|
|
42
53
|
|
|
43
54
|
def get_required_version(self, pipeline=None):
|
|
44
|
-
version_status = self.
|
|
55
|
+
version_status = self._get_version_status()
|
|
45
56
|
self.io.process_messages(version_status.validate())
|
|
46
|
-
required_version = self.
|
|
57
|
+
required_version = self._resolve_required_version(pipeline, version_status)
|
|
47
58
|
self.io.info(required_version)
|
|
48
59
|
return required_version
|
|
49
60
|
|
|
50
61
|
# Used in the generate command
|
|
51
62
|
def check_platform_helper_version_mismatch(self):
|
|
52
|
-
if
|
|
63
|
+
if self.skip_versioning_checks:
|
|
53
64
|
return
|
|
54
65
|
|
|
55
|
-
version_status = self.
|
|
66
|
+
version_status = self._get_version_status()
|
|
56
67
|
self.io.process_messages(version_status.validate())
|
|
57
68
|
|
|
58
69
|
required_version = SemanticVersion.from_string(
|
|
59
|
-
self.
|
|
70
|
+
self._resolve_required_version(version_status=version_status)
|
|
60
71
|
)
|
|
61
72
|
|
|
62
|
-
if not version_status.
|
|
73
|
+
if not version_status.installed == required_version:
|
|
63
74
|
message = (
|
|
64
|
-
f"WARNING: You are running platform-helper v{version_status.
|
|
75
|
+
f"WARNING: You are running platform-helper v{version_status.installed} against "
|
|
65
76
|
f"v{required_version} specified for the project."
|
|
66
77
|
)
|
|
67
78
|
self.io.warn(message)
|
|
79
|
+
|
|
80
|
+
def check_if_needs_update(self):
|
|
81
|
+
if self.skip_versioning_checks:
|
|
82
|
+
return
|
|
83
|
+
|
|
84
|
+
version_status = self._get_version_status(include_project_versions=False)
|
|
85
|
+
|
|
86
|
+
message = (
|
|
87
|
+
f"You are running platform-helper v{version_status.installed}, upgrade to "
|
|
88
|
+
f"v{version_status.latest} by running run `pip install "
|
|
89
|
+
"--upgrade dbt-platform-helper`."
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
version_status.installed.validate_compatibility_with(version_status.latest)
|
|
94
|
+
except IncompatibleMajorVersionException:
|
|
95
|
+
self.io.error(message)
|
|
96
|
+
except IncompatibleMinorVersionException:
|
|
97
|
+
self.io.warn(message)
|
|
98
|
+
|
|
99
|
+
def _get_version_status(
|
|
100
|
+
self,
|
|
101
|
+
include_project_versions: bool = True,
|
|
102
|
+
) -> PlatformHelperVersionStatus:
|
|
103
|
+
locally_installed_version = self.installed_version_provider.get_installed_tool_version(
|
|
104
|
+
"dbt-platform-helper"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
latest_release = self.pypi_provider.get_latest_version("dbt-platform-helper")
|
|
108
|
+
|
|
109
|
+
if not include_project_versions:
|
|
110
|
+
return PlatformHelperVersionStatus(
|
|
111
|
+
installed=locally_installed_version,
|
|
112
|
+
latest=latest_release,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
platform_config_default, pipeline_overrides = None, {}
|
|
116
|
+
|
|
117
|
+
platform_config = self.config_provider.load_unvalidated_config_file()
|
|
118
|
+
|
|
119
|
+
if platform_config:
|
|
120
|
+
platform_config_default = SemanticVersion.from_string(
|
|
121
|
+
platform_config.get("default_versions", {}).get("platform-helper")
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
pipeline_overrides = {
|
|
125
|
+
name: pipeline.get("versions", {}).get("platform-helper")
|
|
126
|
+
for name, pipeline in platform_config.get("environment_pipelines", {}).items()
|
|
127
|
+
if pipeline.get("versions", {}).get("platform-helper")
|
|
128
|
+
}
|
|
129
|
+
out = PlatformHelperVersionStatus(
|
|
130
|
+
installed=locally_installed_version,
|
|
131
|
+
latest=latest_release,
|
|
132
|
+
deprecated_version_file=self.version_file_version_provider.get_required_version(),
|
|
133
|
+
platform_config_default=platform_config_default,
|
|
134
|
+
pipeline_overrides=pipeline_overrides,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
def _resolve_required_version(
|
|
140
|
+
self, pipeline: str = None, version_status: PlatformHelperVersionStatus = None
|
|
141
|
+
) -> str:
|
|
142
|
+
pipeline_version = version_status.pipeline_overrides.get(pipeline)
|
|
143
|
+
version_precedence = [
|
|
144
|
+
pipeline_version,
|
|
145
|
+
version_status.platform_config_default,
|
|
146
|
+
version_status.deprecated_version_file,
|
|
147
|
+
]
|
|
148
|
+
non_null_version_precedence = [
|
|
149
|
+
f"{v}" if isinstance(v, SemanticVersion) else v for v in version_precedence if v
|
|
150
|
+
]
|
|
151
|
+
|
|
152
|
+
out = non_null_version_precedence[0] if non_null_version_precedence else None
|
|
153
|
+
|
|
154
|
+
if not out:
|
|
155
|
+
raise PlatformHelperVersionNotFoundException
|
|
156
|
+
|
|
157
|
+
return out
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from typing import Protocol
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class GetVersionsProtocol(Protocol):
|
|
5
|
+
def get_supported_versions(self) -> list[str]: ...
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class GetReferenceProtocol(Protocol):
|
|
9
|
+
def get_reference(self) -> str: ...
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AwsGetVersionProtocol(GetReferenceProtocol, GetVersionsProtocol):
|
|
13
|
+
pass
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Opensearch:
|
|
5
|
+
|
|
6
|
+
def __init__(self, client: boto3.client):
|
|
7
|
+
self.client = client
|
|
8
|
+
self.engine = "OpenSearch"
|
|
9
|
+
|
|
10
|
+
def get_reference(self) -> str:
|
|
11
|
+
return self.engine.lower()
|
|
12
|
+
|
|
13
|
+
def get_supported_versions(self) -> list[str]:
|
|
14
|
+
response = self.client.list_versions()
|
|
15
|
+
all_versions = response["Versions"]
|
|
16
|
+
|
|
17
|
+
supported_versions = [
|
|
18
|
+
version.removeprefix(f"{self.engine}_")
|
|
19
|
+
for version in all_versions
|
|
20
|
+
if version.startswith(f"{self.engine}_")
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
return supported_versions
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Redis:
|
|
5
|
+
|
|
6
|
+
def __init__(self, client: boto3.client):
|
|
7
|
+
self.client = client
|
|
8
|
+
self.engine = "redis"
|
|
9
|
+
|
|
10
|
+
def get_reference(self) -> str:
|
|
11
|
+
return self.engine.lower()
|
|
12
|
+
|
|
13
|
+
def get_supported_versions(self) -> list[str]:
|
|
14
|
+
supported_versions_response = self.client.describe_cache_engine_versions(Engine=self.engine)
|
|
15
|
+
|
|
16
|
+
supported_versions = [
|
|
17
|
+
version["EngineVersion"]
|
|
18
|
+
for version in supported_versions_response["CacheEngineVersions"]
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
return supported_versions
|
|
@@ -1,10 +1,34 @@
|
|
|
1
1
|
import os
|
|
2
|
+
from abc import ABC
|
|
3
|
+
from abc import abstractmethod
|
|
2
4
|
from datetime import datetime
|
|
3
5
|
|
|
6
|
+
from dbt_platform_helper.providers.aws.interfaces import AwsGetVersionProtocol
|
|
4
7
|
from dbt_platform_helper.providers.yaml_file import YamlFileProvider
|
|
5
8
|
|
|
6
9
|
|
|
7
|
-
class
|
|
10
|
+
class GetDataStrategy(ABC):
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def retrieve_fresh_data(self):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def get_data_identifier(self):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class GetAWSVersionStrategy(GetDataStrategy):
|
|
21
|
+
def __init__(self, client_provider: AwsGetVersionProtocol):
|
|
22
|
+
self.client_provider = client_provider
|
|
23
|
+
|
|
24
|
+
def retrieve_fresh_data(self):
|
|
25
|
+
return self.client_provider.get_supported_versions()
|
|
26
|
+
|
|
27
|
+
def get_data_identifier(self):
|
|
28
|
+
return self.client_provider.get_reference()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Cache:
|
|
8
32
|
def __init__(
|
|
9
33
|
self,
|
|
10
34
|
file_provider: YamlFileProvider = None,
|
|
@@ -12,13 +36,25 @@ class CacheProvider:
|
|
|
12
36
|
self._cache_file = ".platform-helper-config-cache.yml"
|
|
13
37
|
self.file_provider = file_provider or YamlFileProvider
|
|
14
38
|
|
|
15
|
-
def
|
|
39
|
+
def get_data(self, strategy: GetDataStrategy):
|
|
40
|
+
"""Main method to retrieve caching data using the client-specific
|
|
41
|
+
strategy."""
|
|
42
|
+
cache_key = strategy.get_data_identifier()
|
|
43
|
+
if self._cache_refresh_required(cache_key):
|
|
44
|
+
data = strategy.retrieve_fresh_data()
|
|
45
|
+
self._update_cache(cache_key, data)
|
|
46
|
+
else:
|
|
47
|
+
data = self._read_from_cache(cache_key)
|
|
48
|
+
|
|
49
|
+
return data
|
|
50
|
+
|
|
51
|
+
def _read_from_cache(self, resource_name):
|
|
16
52
|
|
|
17
53
|
platform_helper_config = self.file_provider.load(self._cache_file)
|
|
18
54
|
|
|
19
55
|
return platform_helper_config.get(resource_name).get("versions")
|
|
20
56
|
|
|
21
|
-
def
|
|
57
|
+
def _update_cache(self, resource_name, supported_versions):
|
|
22
58
|
|
|
23
59
|
platform_helper_config = {}
|
|
24
60
|
|
|
@@ -40,7 +76,7 @@ class CacheProvider:
|
|
|
40
76
|
"# [!] This file is autogenerated via the platform-helper. Do not edit.\n",
|
|
41
77
|
)
|
|
42
78
|
|
|
43
|
-
def
|
|
79
|
+
def _cache_refresh_required(self, resource_name) -> bool:
|
|
44
80
|
"""
|
|
45
81
|
Checks if the platform-helper should reach out to AWS to 'refresh' its
|
|
46
82
|
cached values.
|
|
@@ -3,9 +3,11 @@ from typing import Callable
|
|
|
3
3
|
import boto3
|
|
4
4
|
|
|
5
5
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
6
|
+
from dbt_platform_helper.providers.aws.opensearch import Opensearch
|
|
7
|
+
from dbt_platform_helper.providers.aws.redis import Redis
|
|
8
|
+
from dbt_platform_helper.providers.cache import Cache
|
|
9
|
+
from dbt_platform_helper.providers.cache import GetAWSVersionStrategy
|
|
6
10
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
7
|
-
from dbt_platform_helper.providers.opensearch import OpensearchProvider
|
|
8
|
-
from dbt_platform_helper.providers.redis import RedisProvider
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
class ConfigValidatorError(PlatformException):
|
|
@@ -32,7 +34,7 @@ class ConfigValidator:
|
|
|
32
34
|
validation(config)
|
|
33
35
|
|
|
34
36
|
def _validate_extension_supported_versions(
|
|
35
|
-
self, config, extension_type, version_key
|
|
37
|
+
self, config, aws_provider, extension_type, version_key
|
|
36
38
|
):
|
|
37
39
|
extensions = config.get("extensions", {})
|
|
38
40
|
if not extensions:
|
|
@@ -44,7 +46,10 @@ class ConfigValidator:
|
|
|
44
46
|
if extension.get("type") == extension_type
|
|
45
47
|
]
|
|
46
48
|
|
|
47
|
-
|
|
49
|
+
# In this format so it can be monkey patched initially via mock_get_data fixture
|
|
50
|
+
cache_provider = Cache()
|
|
51
|
+
get_data_strategy = GetAWSVersionStrategy(aws_provider)
|
|
52
|
+
supported_extension_versions = cache_provider.get_data(get_data_strategy)
|
|
48
53
|
extensions_with_invalid_version = []
|
|
49
54
|
|
|
50
55
|
for extension in extensions_for_type:
|
|
@@ -74,21 +79,17 @@ class ConfigValidator:
|
|
|
74
79
|
def validate_supported_redis_versions(self, config):
|
|
75
80
|
return self._validate_extension_supported_versions(
|
|
76
81
|
config=config,
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
boto3.client("elasticache")
|
|
81
|
-
).get_supported_redis_versions,
|
|
82
|
+
aws_provider=Redis(boto3.client("elasticache")),
|
|
83
|
+
extension_type="redis", # TODO this is information which can live in the RedisProvider
|
|
84
|
+
version_key="engine", # TODO this is information which can live in the RedisProvider
|
|
82
85
|
)
|
|
83
86
|
|
|
84
87
|
def validate_supported_opensearch_versions(self, config):
|
|
85
88
|
return self._validate_extension_supported_versions(
|
|
86
89
|
config=config,
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
boto3.client("opensearch")
|
|
91
|
-
).get_supported_opensearch_versions,
|
|
90
|
+
aws_provider=Opensearch(boto3.client("opensearch")),
|
|
91
|
+
extension_type="opensearch", # TODO this is information which can live in the OpensearchProvider
|
|
92
|
+
version_key="engine", # TODO this is information which can live in the OpensearchProvider
|
|
92
93
|
)
|
|
93
94
|
|
|
94
95
|
def validate_environment_pipelines(self, config):
|
|
@@ -4,7 +4,7 @@ import time
|
|
|
4
4
|
from botocore.exceptions import ClientError
|
|
5
5
|
|
|
6
6
|
from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
|
|
7
|
-
from dbt_platform_helper.providers.aws import CreateTaskTimeoutException
|
|
7
|
+
from dbt_platform_helper.providers.aws.exceptions import CreateTaskTimeoutException
|
|
8
8
|
from dbt_platform_helper.providers.secrets import Secrets
|
|
9
9
|
from dbt_platform_helper.utils.application import Application
|
|
10
10
|
from dbt_platform_helper.utils.messages import abort_with_error
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class KMSProvider:
|
|
5
|
+
"""A provider class for interacting with the AWS KMS (Key Management
|
|
6
|
+
Service)."""
|
|
7
|
+
|
|
8
|
+
def __init__(self, kms_client: boto3.client):
|
|
9
|
+
self.kms_client = kms_client
|
|
10
|
+
|
|
11
|
+
def describe_key(self, alias_name: str) -> dict:
|
|
12
|
+
"""
|
|
13
|
+
Retrieves metadata about a KMS key using its alias.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
alias_name (str): The alias name of the KMS key.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
dict: A dictionary containing metadata about the specified KMS key.
|
|
20
|
+
"""
|
|
21
|
+
# The kms client can take an alias name as the KeyId
|
|
22
|
+
return self.kms_client.describe_key(KeyId=alias_name)
|
|
@@ -192,15 +192,14 @@ class LoadBalancerProvider:
|
|
|
192
192
|
rule_name: str,
|
|
193
193
|
priority: int,
|
|
194
194
|
conditions: list,
|
|
195
|
+
additional_tags: list = [],
|
|
195
196
|
):
|
|
196
197
|
return self.create_rule(
|
|
197
198
|
listener_arn=listener_arn,
|
|
198
199
|
priority=priority,
|
|
199
200
|
conditions=conditions,
|
|
200
201
|
actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
|
|
201
|
-
tags=[
|
|
202
|
-
{"Key": "name", "Value": rule_name},
|
|
203
|
-
],
|
|
202
|
+
tags=[{"Key": "name", "Value": rule_name}, *additional_tags],
|
|
204
203
|
)
|
|
205
204
|
|
|
206
205
|
def create_header_rule(
|
|
@@ -212,6 +211,7 @@ class LoadBalancerProvider:
|
|
|
212
211
|
rule_name: str,
|
|
213
212
|
priority: int,
|
|
214
213
|
conditions: list,
|
|
214
|
+
additional_tags: list = [],
|
|
215
215
|
):
|
|
216
216
|
|
|
217
217
|
combined_conditions = [
|
|
@@ -222,11 +222,16 @@ class LoadBalancerProvider:
|
|
|
222
222
|
] + conditions
|
|
223
223
|
|
|
224
224
|
self.create_forward_rule(
|
|
225
|
-
listener_arn,
|
|
225
|
+
listener_arn,
|
|
226
|
+
target_group_arn,
|
|
227
|
+
rule_name,
|
|
228
|
+
priority,
|
|
229
|
+
combined_conditions,
|
|
230
|
+
additional_tags,
|
|
226
231
|
)
|
|
227
232
|
|
|
228
|
-
self.io.
|
|
229
|
-
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\
|
|
233
|
+
self.io.debug(
|
|
234
|
+
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
|
|
230
235
|
)
|
|
231
236
|
|
|
232
237
|
def create_source_ip_rule(
|
|
@@ -237,6 +242,7 @@ class LoadBalancerProvider:
|
|
|
237
242
|
rule_name: str,
|
|
238
243
|
priority: int,
|
|
239
244
|
conditions: list,
|
|
245
|
+
additional_tags: list = [],
|
|
240
246
|
):
|
|
241
247
|
combined_conditions = [
|
|
242
248
|
{
|
|
@@ -246,24 +252,29 @@ class LoadBalancerProvider:
|
|
|
246
252
|
] + conditions
|
|
247
253
|
|
|
248
254
|
self.create_forward_rule(
|
|
249
|
-
listener_arn,
|
|
255
|
+
listener_arn,
|
|
256
|
+
target_group_arn,
|
|
257
|
+
rule_name,
|
|
258
|
+
priority,
|
|
259
|
+
combined_conditions,
|
|
260
|
+
additional_tags,
|
|
250
261
|
)
|
|
251
262
|
|
|
252
|
-
self.io.
|
|
253
|
-
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\
|
|
263
|
+
self.io.debug(
|
|
264
|
+
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.\n\n",
|
|
254
265
|
)
|
|
255
266
|
|
|
256
|
-
def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) ->
|
|
257
|
-
|
|
267
|
+
def delete_listener_rule_by_tags(self, tag_descriptions: list, tag_name: str) -> list:
|
|
268
|
+
deleted_rules = []
|
|
258
269
|
|
|
259
270
|
for description in tag_descriptions:
|
|
260
271
|
tags = {t["Key"]: t["Value"] for t in description["Tags"]}
|
|
261
272
|
if tags.get("name") == tag_name:
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
273
|
+
if description["ResourceArn"]:
|
|
274
|
+
self.evlb_client.delete_rule(RuleArn=description["ResourceArn"])
|
|
275
|
+
deleted_rules.append(description)
|
|
265
276
|
|
|
266
|
-
return
|
|
277
|
+
return deleted_rules
|
|
267
278
|
|
|
268
279
|
|
|
269
280
|
class LoadBalancerException(PlatformException):
|
|
@@ -35,6 +35,9 @@ class SemanticVersion:
|
|
|
35
35
|
return "unknown"
|
|
36
36
|
return ".".join([str(s) for s in [self.major, self.minor, self.patch]])
|
|
37
37
|
|
|
38
|
+
def __repr__(self) -> str:
|
|
39
|
+
return str(self)
|
|
40
|
+
|
|
38
41
|
def __lt__(self, other) -> bool:
|
|
39
42
|
return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
|
|
40
43
|
|
|
@@ -76,7 +79,7 @@ class SemanticVersion:
|
|
|
76
79
|
|
|
77
80
|
@dataclass
|
|
78
81
|
class VersionStatus:
|
|
79
|
-
|
|
82
|
+
installed: SemanticVersion = None
|
|
80
83
|
latest: SemanticVersion = None
|
|
81
84
|
|
|
82
85
|
def __str__(self):
|
|
@@ -87,7 +90,7 @@ class VersionStatus:
|
|
|
87
90
|
return f"{self.__class__.__name__}: {attrs_str}"
|
|
88
91
|
|
|
89
92
|
def is_outdated(self):
|
|
90
|
-
return self.
|
|
93
|
+
return self.installed != self.latest
|
|
91
94
|
|
|
92
95
|
def validate(self):
|
|
93
96
|
pass
|
|
@@ -95,7 +98,7 @@ class VersionStatus:
|
|
|
95
98
|
|
|
96
99
|
@dataclass
|
|
97
100
|
class PlatformHelperVersionStatus(VersionStatus):
|
|
98
|
-
|
|
101
|
+
installed: Optional[SemanticVersion] = None
|
|
99
102
|
latest: Optional[SemanticVersion] = None
|
|
100
103
|
deprecated_version_file: Optional[SemanticVersion] = None
|
|
101
104
|
platform_config_default: Optional[SemanticVersion] = None
|
|
@@ -137,7 +140,7 @@ class PlatformHelperVersionStatus(VersionStatus):
|
|
|
137
140
|
|
|
138
141
|
if not self.platform_config_default and not self.deprecated_version_file:
|
|
139
142
|
message = f"Cannot get dbt-platform-helper version from '{PLATFORM_CONFIG_FILE}'.\n"
|
|
140
|
-
message += f"{missing_default_version_message}{self.
|
|
143
|
+
message += f"{missing_default_version_message}{self.installed}\n"
|
|
141
144
|
errors.append(message)
|
|
142
145
|
|
|
143
146
|
return {
|
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
from abc import ABC
|
|
2
2
|
from importlib.metadata import PackageNotFoundError
|
|
3
3
|
from importlib.metadata import version
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import requests
|
|
6
7
|
|
|
8
|
+
from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
|
|
7
9
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
8
10
|
from dbt_platform_helper.providers.semantic_version import SemanticVersion
|
|
11
|
+
from dbt_platform_helper.providers.yaml_file import FileProviderException
|
|
12
|
+
from dbt_platform_helper.providers.yaml_file import YamlFileProvider
|
|
9
13
|
|
|
10
14
|
|
|
11
|
-
class
|
|
15
|
+
class InstalledVersionProviderException(PlatformException):
|
|
12
16
|
pass
|
|
13
17
|
|
|
14
18
|
|
|
15
|
-
class InstalledToolNotFoundException(
|
|
19
|
+
class InstalledToolNotFoundException(InstalledVersionProviderException):
|
|
16
20
|
def __init__(
|
|
17
21
|
self,
|
|
18
22
|
tool_name: str,
|
|
@@ -24,7 +28,7 @@ class VersionProvider(ABC):
|
|
|
24
28
|
pass
|
|
25
29
|
|
|
26
30
|
|
|
27
|
-
class
|
|
31
|
+
class InstalledVersionProvider:
|
|
28
32
|
@staticmethod
|
|
29
33
|
def get_installed_tool_version(tool_name: str) -> SemanticVersion:
|
|
30
34
|
try:
|
|
@@ -58,3 +62,17 @@ class PyPiVersionProvider(VersionProvider):
|
|
|
58
62
|
parsed_released_versions = [SemanticVersion.from_string(v) for v in released_versions]
|
|
59
63
|
parsed_released_versions.sort(reverse=True)
|
|
60
64
|
return parsed_released_versions[0]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class DeprecatedVersionFileVersionProvider(VersionProvider):
|
|
68
|
+
def __init__(self, file_provider: YamlFileProvider):
|
|
69
|
+
self.file_provider = file_provider or YamlFileProvider
|
|
70
|
+
|
|
71
|
+
def get_required_version(self) -> SemanticVersion:
|
|
72
|
+
deprecated_version_file = Path(PLATFORM_HELPER_VERSION_FILE)
|
|
73
|
+
try:
|
|
74
|
+
loaded_version = self.file_provider.load(deprecated_version_file)
|
|
75
|
+
version_from_file = SemanticVersion.from_string(loaded_version)
|
|
76
|
+
except FileProviderException:
|
|
77
|
+
version_from_file = None
|
|
78
|
+
return version_from_file
|
dbt_platform_helper/utils/aws.py
CHANGED
|
@@ -12,13 +12,16 @@ import botocore.exceptions
|
|
|
12
12
|
import click
|
|
13
13
|
import yaml
|
|
14
14
|
from boto3 import Session
|
|
15
|
+
from botocore.exceptions import ClientError
|
|
15
16
|
|
|
16
17
|
from dbt_platform_helper.constants import REFRESH_TOKEN_MESSAGE
|
|
17
18
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
18
|
-
from dbt_platform_helper.providers.aws import
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
from dbt_platform_helper.providers.aws import
|
|
19
|
+
from dbt_platform_helper.providers.aws.exceptions import (
|
|
20
|
+
CopilotCodebaseNotFoundException,
|
|
21
|
+
)
|
|
22
|
+
from dbt_platform_helper.providers.aws.exceptions import ImageNotFoundException
|
|
23
|
+
from dbt_platform_helper.providers.aws.exceptions import LogGroupNotFoundException
|
|
24
|
+
from dbt_platform_helper.providers.aws.exceptions import RepositoryNotFoundException
|
|
22
25
|
from dbt_platform_helper.providers.validation import ValidationException
|
|
23
26
|
|
|
24
27
|
SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
|
|
@@ -484,3 +487,23 @@ def wait_for_log_group_to_exist(log_client, log_group_name, attempts=30):
|
|
|
484
487
|
|
|
485
488
|
if not log_group_exists:
|
|
486
489
|
raise LogGroupNotFoundException(log_group_name)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def get_image_build_project(codebuild_client, application, codebase):
|
|
493
|
+
project_name = f"{application}-{codebase}-codebase-image-build"
|
|
494
|
+
response = codebuild_client.batch_get_projects(names=[project_name])
|
|
495
|
+
|
|
496
|
+
if bool(response.get("projects")):
|
|
497
|
+
return project_name
|
|
498
|
+
else:
|
|
499
|
+
return f"{application}-{codebase}-codebase-pipeline-image-build"
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def get_manual_release_pipeline(codepipeline_client, application, codebase):
|
|
503
|
+
pipeline_name = f"{application}-{codebase}-manual-release"
|
|
504
|
+
try:
|
|
505
|
+
codepipeline_client.get_pipeline(name=pipeline_name)
|
|
506
|
+
return pipeline_name
|
|
507
|
+
except ClientError as e:
|
|
508
|
+
if e.response["Error"]["Code"] == "PipelineNotFoundException":
|
|
509
|
+
return f"{pipeline_name}-pipeline"
|