qubership-pipelines-common-library 0.1.10__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qubership_pipelines_common_library/v1/execution/exec_command.py +15 -0
- qubership_pipelines_common_library/v1/execution/exec_context.py +11 -0
- qubership_pipelines_common_library/v1/execution/exec_context_file.py +14 -0
- qubership_pipelines_common_library/v1/execution/exec_info.py +8 -1
- qubership_pipelines_common_library/v1/execution/exec_logger.py +7 -0
- qubership_pipelines_common_library/v1/maven_client.py +286 -0
- qubership_pipelines_common_library/v1/utils/utils_aws.py +16 -0
- qubership_pipelines_common_library/v1/utils/utils_context.py +4 -1
- {qubership_pipelines_common_library-0.1.10.dist-info → qubership_pipelines_common_library-0.2.0.dist-info}/METADATA +3 -1
- {qubership_pipelines_common_library-0.1.10.dist-info → qubership_pipelines_common_library-0.2.0.dist-info}/RECORD +12 -10
- {qubership_pipelines_common_library-0.1.10.dist-info → qubership_pipelines_common_library-0.2.0.dist-info}/LICENSE +0 -0
- {qubership_pipelines_common_library-0.1.10.dist-info → qubership_pipelines_common_library-0.2.0.dist-info}/WHEEL +0 -0
|
@@ -27,12 +27,27 @@ class ExecutionCommand:
|
|
|
27
27
|
|
|
28
28
|
def __init__(self, context_path: str = None, input_params: dict = None, input_params_secure: dict = None,
|
|
29
29
|
folder_path: str = None, parent_context_to_reuse: ExecutionContext = None):
|
|
30
|
+
"""
|
|
31
|
+
Extendable interface intended to simplify working with input/output params and passing them between commands in different Pipeline Executors
|
|
32
|
+
|
|
33
|
+
Implementations are expected to override **`_validate`** and **`_execute`** methods
|
|
34
|
+
|
|
35
|
+
If **`context_path`** is not provided - context will be created dynamically using other provided params
|
|
36
|
+
|
|
37
|
+
Arguments:
|
|
38
|
+
context_path (str): Path to context-describing yaml, that should contain references to input/output param file locations
|
|
39
|
+
input_params (dict): Non-secure parameters that will be merged into dynamically created params
|
|
40
|
+
input_params_secure (dict): Secure parameters that will be merged into dynamically created params
|
|
41
|
+
folder_path (str): Folder path where dynamically-created context will be stored. Optional, will create new temp folder if missing.
|
|
42
|
+
parent_context_to_reuse (ExecutionContext): Optional, existing context to propagate input params from.
|
|
43
|
+
"""
|
|
30
44
|
if not context_path:
|
|
31
45
|
context_path = create_execution_context(input_params=input_params, input_params_secure=input_params_secure,
|
|
32
46
|
folder_path=folder_path, parent_context_to_reuse=parent_context_to_reuse)
|
|
33
47
|
self.context = ExecutionContext(context_path)
|
|
34
48
|
|
|
35
49
|
def run(self):
|
|
50
|
+
"""Runs command following its lifecycle"""
|
|
36
51
|
try:
|
|
37
52
|
if not self._validate():
|
|
38
53
|
logging.error(ExecutionCommand.FAILURE_MSG)
|
|
@@ -23,6 +23,12 @@ from qubership_pipelines_common_library.v1.execution.exec_logger import Executio
|
|
|
23
23
|
class ExecutionContext:
|
|
24
24
|
|
|
25
25
|
def __init__(self, context_path: str):
|
|
26
|
+
"""
|
|
27
|
+
Interface that provides references and shortcuts to navigating provided input params, storing any output params, and logging messages.
|
|
28
|
+
|
|
29
|
+
Arguments:
|
|
30
|
+
context_path (str): Path to context-describing yaml, that should contain references to input/output param file locations
|
|
31
|
+
"""
|
|
26
32
|
full_path = os.path.abspath(context_path)
|
|
27
33
|
self.context_path = full_path
|
|
28
34
|
self.context = ExecutionContextFile(full_path)
|
|
@@ -47,6 +53,7 @@ class ExecutionContext:
|
|
|
47
53
|
self.__input_params_load()
|
|
48
54
|
|
|
49
55
|
def output_params_save(self):
|
|
56
|
+
"""Stores output_param files to disk"""
|
|
50
57
|
if self.context.get("paths.output.params"):
|
|
51
58
|
logging.info(f"Writing insecure param file '{self.context.get('paths.output.params')}'")
|
|
52
59
|
self.output_params.save(self.context.get("paths.output.params"))
|
|
@@ -55,6 +62,7 @@ class ExecutionContext:
|
|
|
55
62
|
self.output_params_secure.save(self.context.get("paths.output.params_secure"))
|
|
56
63
|
|
|
57
64
|
def input_param_get(self, path, def_value=None):
|
|
65
|
+
"""Gets parameter from provided params files by its param path, supporting dot-separated nested keys (e.g. 'parent_obj.child_obj.param_name')"""
|
|
58
66
|
value = self.input_params.get(path, def_value)
|
|
59
67
|
if value == def_value:
|
|
60
68
|
value = self.input_params_secure.get(path, def_value)
|
|
@@ -63,12 +71,15 @@ class ExecutionContext:
|
|
|
63
71
|
return value
|
|
64
72
|
|
|
65
73
|
def output_param_set(self, path, value):
|
|
74
|
+
"""Sets param by path in non-secure output params"""
|
|
66
75
|
return self.output_params.set(path, value)
|
|
67
76
|
|
|
68
77
|
def output_param_secure_set(self, path, value):
|
|
78
|
+
"""Sets param by path in secure output params"""
|
|
69
79
|
return self.output_params_secure.set(path, value)
|
|
70
80
|
|
|
71
81
|
def validate(self, names, silent=False):
|
|
82
|
+
"""Validates that all provided param `names` are present among provided param files"""
|
|
72
83
|
valid = True
|
|
73
84
|
for key in names:
|
|
74
85
|
if not self.__validate_param(key):
|
|
@@ -35,6 +35,11 @@ class ExecutionContextFile:
|
|
|
35
35
|
SUPPORTED_API_VERSIONS = [API_VERSION_V1]
|
|
36
36
|
|
|
37
37
|
def __init__(self, path=None):
|
|
38
|
+
"""
|
|
39
|
+
Interface to work with **`params`** and **`context`** files, used in **`ExecutionContext`**.
|
|
40
|
+
|
|
41
|
+
Provides methods to init default content for different types of descriptors (e.g. **`init_context_descriptor`**, **`init_params`**)
|
|
42
|
+
"""
|
|
38
43
|
self.content = {
|
|
39
44
|
"kind": "",
|
|
40
45
|
"apiVersion": ""
|
|
@@ -44,12 +49,14 @@ class ExecutionContextFile:
|
|
|
44
49
|
self.load(path)
|
|
45
50
|
|
|
46
51
|
def init_empty(self):
|
|
52
|
+
""""""
|
|
47
53
|
self.content = {
|
|
48
54
|
"kind": "",
|
|
49
55
|
"apiVersion": ""
|
|
50
56
|
}
|
|
51
57
|
|
|
52
58
|
def init_context_descriptor(self, context_folder_path: str = None):
|
|
59
|
+
""""""
|
|
53
60
|
if context_folder_path is None:
|
|
54
61
|
context_folder_path = ""
|
|
55
62
|
ctx_path = Path(context_folder_path)
|
|
@@ -87,6 +94,7 @@ class ExecutionContextFile:
|
|
|
87
94
|
return self
|
|
88
95
|
|
|
89
96
|
def init_params(self):
|
|
97
|
+
""""""
|
|
90
98
|
self.content = {
|
|
91
99
|
"kind": ExecutionContextFile.KIND_PARAMS_INSECURE,
|
|
92
100
|
"apiVersion": ExecutionContextFile.API_VERSION_V1,
|
|
@@ -103,6 +111,7 @@ class ExecutionContextFile:
|
|
|
103
111
|
return self
|
|
104
112
|
|
|
105
113
|
def init_params_secure(self):
|
|
114
|
+
""""""
|
|
106
115
|
self.content = {
|
|
107
116
|
"kind": ExecutionContextFile.KIND_PARAMS_SECURE,
|
|
108
117
|
"apiVersion": ExecutionContextFile.API_VERSION_V1,
|
|
@@ -119,6 +128,7 @@ class ExecutionContextFile:
|
|
|
119
128
|
return self
|
|
120
129
|
|
|
121
130
|
def load(self, path):
|
|
131
|
+
"""Loads and validates file as one of supported types of descriptors"""
|
|
122
132
|
full_path = os.path.abspath(path)
|
|
123
133
|
try:
|
|
124
134
|
self.content = UtilsFile.read_yaml(full_path)
|
|
@@ -135,17 +145,21 @@ class ExecutionContextFile:
|
|
|
135
145
|
self.init_empty()
|
|
136
146
|
|
|
137
147
|
def save(self, path):
|
|
148
|
+
"""Writes current file content from memory to disk"""
|
|
138
149
|
# TODO: support encryption with SOPS
|
|
139
150
|
UtilsFile.write_yaml(path, self.content)
|
|
140
151
|
|
|
141
152
|
def get(self, path, def_value=None):
|
|
153
|
+
"""Gets parameter from current file content by its param path, supporting dot-separated nested keys (e.g. 'parent_obj.child_obj.param_name')"""
|
|
142
154
|
return UtilsDictionary.get_by_path(self.content, path, def_value)
|
|
143
155
|
|
|
144
156
|
def set(self, path, value):
|
|
157
|
+
"""Sets parameter in current file content"""
|
|
145
158
|
UtilsDictionary.set_by_path(self.content, path, value)
|
|
146
159
|
return self
|
|
147
160
|
|
|
148
161
|
def set_multiple(self, dict):
|
|
162
|
+
"""Sets multiple parameters in current file content"""
|
|
149
163
|
for key in dict:
|
|
150
164
|
UtilsDictionary.set_by_path(self.content, key, dict[key])
|
|
151
165
|
return self
|
|
@@ -31,6 +31,9 @@ class ExecutionInfo:
|
|
|
31
31
|
STATUSES_COMPLETE = [STATUS_SUCCESS, STATUS_UNSTABLE, STATUS_FAILED, STATUS_ABORTED]
|
|
32
32
|
|
|
33
33
|
def __init__(self):
|
|
34
|
+
"""
|
|
35
|
+
Describes trackable running processes (e.g. triggered GitHub workflow)
|
|
36
|
+
"""
|
|
34
37
|
self.url = "" # url to pipeline execution
|
|
35
38
|
self.id = "" # unique id of the execution
|
|
36
39
|
self.status = ExecutionInfo.STATUS_UNKNOWN # current status of the pipeline
|
|
@@ -40,11 +43,13 @@ class ExecutionInfo:
|
|
|
40
43
|
self.params = {} # optional params used to run the pipe
|
|
41
44
|
|
|
42
45
|
def start(self):
|
|
46
|
+
"""Records start time for described process and transitions its status to **`IN_PROGRESS`**"""
|
|
43
47
|
self.time_start = datetime.now()
|
|
44
48
|
self.status = ExecutionInfo.STATUS_IN_PROGRESS
|
|
45
49
|
return self
|
|
46
50
|
|
|
47
51
|
def stop(self, status: str = None):
|
|
52
|
+
"""Records finish time for described process, and optionally transitions its status to passed value"""
|
|
48
53
|
if status:
|
|
49
54
|
self.with_status(status)
|
|
50
55
|
self.time_stop = datetime.now()
|
|
@@ -66,9 +71,11 @@ class ExecutionInfo:
|
|
|
66
71
|
return self.time_stop
|
|
67
72
|
|
|
68
73
|
def get_duration(self):
|
|
74
|
+
"""Returns duration of this process after it's finished"""
|
|
69
75
|
return self.time_stop - self.time_start
|
|
70
|
-
|
|
76
|
+
|
|
71
77
|
def get_duration_str(self):
|
|
78
|
+
"""Returns formatted duration of this process as `hh:mm:ss` string after it's finished """
|
|
72
79
|
seconds = int(self.get_duration().total_seconds())
|
|
73
80
|
parts = [seconds / 3600, (seconds % 3600) / 60, seconds % 60]
|
|
74
81
|
strings = list(map(lambda x: str(int(x)).zfill(2), parts))
|
|
@@ -23,6 +23,13 @@ class ExecutionLogger:
|
|
|
23
23
|
DEFAULT_FORMAT = u'[%(asctime)s] [%(levelname)-5s] [class=%(filename)s:%(lineno)-3s] %(message)s'
|
|
24
24
|
|
|
25
25
|
def __init__(self, path_logs):
|
|
26
|
+
"""
|
|
27
|
+
Default logger used by **`ExecutionCommands`**, implicitly initialized when using Context.
|
|
28
|
+
|
|
29
|
+
Reference to it is available from instance of **`ExecutionContext`**.
|
|
30
|
+
|
|
31
|
+
Provides common logging methods of different log levels - e.g. **`debug`**, **`info`**, **`error`**
|
|
32
|
+
"""
|
|
26
33
|
# todo: Currently all commands (if more than one are invoked in one go) will reuse same logger
|
|
27
34
|
# Also, file handlers are never removed
|
|
28
35
|
self.path_logs = path_logs
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import pathlib
|
|
4
|
+
import re
|
|
5
|
+
import requests
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from xml.etree import ElementTree
|
|
9
|
+
from requests.auth import HTTPBasicAuth
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Artifact:
|
|
13
|
+
def __init__(self, artifact_id, version, extension='jar'):
|
|
14
|
+
self.artifact_id = artifact_id
|
|
15
|
+
self.version = version
|
|
16
|
+
self.extension = "jar" if not extension else extension
|
|
17
|
+
|
|
18
|
+
def is_snapshot(self):
|
|
19
|
+
return self.version and self.version.endswith("SNAPSHOT")
|
|
20
|
+
|
|
21
|
+
@staticmethod
|
|
22
|
+
def from_string(artifact_str: str):
|
|
23
|
+
parts = artifact_str.split(":")
|
|
24
|
+
if len(parts) == 3:
|
|
25
|
+
group, artifact, version = parts[0], parts[1], parts[-1]
|
|
26
|
+
return Artifact(artifact, version)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class MavenArtifactSearcher:
|
|
30
|
+
"""
|
|
31
|
+
Allows searching for specific maven artifacts in different repositories without knowing full coordinates
|
|
32
|
+
(e.g. knowing only `artifact_id` and `version`, but not its `group_id`)
|
|
33
|
+
|
|
34
|
+
Supports different Maven repository providers: Artifactory, Nexus, AWS, GCP
|
|
35
|
+
|
|
36
|
+
Start by initializing this client with one of implementations:
|
|
37
|
+
``maven_client = MavenArtifactSearcher(registry_url).with_artifactory(artifactory_user, artifactory_token)``
|
|
38
|
+
|
|
39
|
+
Then find your artifacts using
|
|
40
|
+
``maven_client.find_artifact_urls('art_id', '1.0.0')``
|
|
41
|
+
|
|
42
|
+
Additionally, perform filtering of returned results, and then download necessary artifacts with
|
|
43
|
+
``maven_client.download_artifact(one_of_the_returned_urls, './my_artifact.jar')``
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
TIMESTAMP_VERSION_PATTERN = "^(.*-)?([0-9]{8}\\.[0-9]{6}-[0-9]+)$"
|
|
47
|
+
|
|
48
|
+
def __init__(self, registry_url: str, params: dict = None, **kwargs):
|
|
49
|
+
self.is_init = False
|
|
50
|
+
self._search_func = None
|
|
51
|
+
self._download_func = None
|
|
52
|
+
self.registry_url = registry_url.rstrip("/")
|
|
53
|
+
self.params = params if params else {}
|
|
54
|
+
self._session = requests.Session()
|
|
55
|
+
self._session.verify = self.params.get('verify', True)
|
|
56
|
+
self.timeout = self.params.get('timeout', None)
|
|
57
|
+
|
|
58
|
+
def find_artifact_urls(self, artifact_id: str = None, version: str = None, extension: str = "jar",
|
|
59
|
+
artifact: Artifact = None) -> list[str]:
|
|
60
|
+
"""
|
|
61
|
+
Finds and returns list of URLs (or resource IDs, for specific providers) to target artifacts.
|
|
62
|
+
Client should be initialized with one of providers first.
|
|
63
|
+
Doesn't require `group_id` to find artifacts.
|
|
64
|
+
Works with either `artifact_id`/`version` or `Artifact` class as input parameters.
|
|
65
|
+
"""
|
|
66
|
+
self._check_init()
|
|
67
|
+
if not artifact:
|
|
68
|
+
artifact = Artifact(artifact_id=artifact_id, version=version, extension=extension)
|
|
69
|
+
if not artifact.artifact_id or not artifact.version:
|
|
70
|
+
raise Exception(f"Artifact 'artifact_id' and 'version' must be specified!")
|
|
71
|
+
logging.debug(f"Searching for '{artifact.artifact_id}' in {self.registry_url}...")
|
|
72
|
+
return self._search_func(artifact=artifact)
|
|
73
|
+
|
|
74
|
+
def download_artifact(self, url: str, local_path: str):
|
|
75
|
+
"""
|
|
76
|
+
Downloads maven artifact from `url` to a `local_path` location
|
|
77
|
+
(you need to provide full path, including filename, since we can't determine it from resource urls for some providers).
|
|
78
|
+
`url` should be one of values returned by `find_artifact_urls`.
|
|
79
|
+
Client should be initialized with one of providers first.
|
|
80
|
+
"""
|
|
81
|
+
self._check_init()
|
|
82
|
+
self._create_dir(local_path)
|
|
83
|
+
logging.debug(f"Downloading artifact from '{url}' to '{local_path}'...")
|
|
84
|
+
return self._download_func(url=url, local_path=local_path)
|
|
85
|
+
|
|
86
|
+
def _check_init(self):
|
|
87
|
+
if not self.is_init:
|
|
88
|
+
raise Exception("Init client with one of registry implementations first, e.g. '.with_artifactory'!")
|
|
89
|
+
|
|
90
|
+
def _create_dir(self, local_path: str):
|
|
91
|
+
directory = os.path.dirname(local_path)
|
|
92
|
+
if directory:
|
|
93
|
+
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
|
|
94
|
+
|
|
95
|
+
def _generic_download(self, url: str, local_path: str):
|
|
96
|
+
response = self._session.get(url=url, timeout=self.timeout)
|
|
97
|
+
response.raise_for_status()
|
|
98
|
+
with open(local_path, 'wb') as file:
|
|
99
|
+
file.write(response.content)
|
|
100
|
+
|
|
101
|
+
def with_artifactory(self, username: str = None, password: str = None):
|
|
102
|
+
"""
|
|
103
|
+
Initializes this client to work with **JFrog Artifactory** maven repositories.
|
|
104
|
+
Requires `username` and its `password` or `token`.
|
|
105
|
+
"""
|
|
106
|
+
if password:
|
|
107
|
+
self._session.auth = HTTPBasicAuth(username, password)
|
|
108
|
+
self._search_func = self._artifactory_search
|
|
109
|
+
self._download_func = self._generic_download
|
|
110
|
+
self.is_init = True
|
|
111
|
+
return self
|
|
112
|
+
|
|
113
|
+
def _artifactory_search(self, artifact: Artifact = None) -> list[str]:
|
|
114
|
+
# 1.0, 1.1 - release
|
|
115
|
+
# 1.0-SNAPSHOT - snapshot head
|
|
116
|
+
# 1.0-123456-2025125-1 - specific snapshot
|
|
117
|
+
timestamp_version_match = re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version)
|
|
118
|
+
if timestamp_version_match:
|
|
119
|
+
base_version = timestamp_version_match.group(1) + "SNAPSHOT"
|
|
120
|
+
else:
|
|
121
|
+
base_version = artifact.version
|
|
122
|
+
response = self._session.get(url=f"{self.registry_url}/api/search/gavc",
|
|
123
|
+
params={"a": artifact.artifact_id, "v": base_version, "specific": "true"},
|
|
124
|
+
timeout=self.timeout)
|
|
125
|
+
if response.status_code != 200:
|
|
126
|
+
raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {response.status_code}!")
|
|
127
|
+
return [result["downloadUri"] for result in response.json()["results"]
|
|
128
|
+
if result["ext"] == artifact.extension and (not timestamp_version_match or result["downloadUri"].endswith(f"{artifact.version}.{artifact.extension}"))]
|
|
129
|
+
|
|
130
|
+
def with_nexus(self, username: str = None, password: str = None):
|
|
131
|
+
"""
|
|
132
|
+
Initializes this client to work with **Sonatype Nexus Repository** for maven artifacts.
|
|
133
|
+
Requires `username` and its `password` or `token`.
|
|
134
|
+
"""
|
|
135
|
+
if password:
|
|
136
|
+
self._session.auth = HTTPBasicAuth(username, password)
|
|
137
|
+
self._search_func = self._nexus_search
|
|
138
|
+
self._download_func = self._generic_download
|
|
139
|
+
self.is_init = True
|
|
140
|
+
return self
|
|
141
|
+
|
|
142
|
+
def _nexus_search(self, artifact: Artifact = None) -> list[str]:
|
|
143
|
+
search_params = {
|
|
144
|
+
"maven.artifactId": artifact.artifact_id,
|
|
145
|
+
"maven.extension": artifact.extension
|
|
146
|
+
}
|
|
147
|
+
if artifact.version.endswith("-SNAPSHOT"):
|
|
148
|
+
search_params["maven.baseVersion"] = artifact.version
|
|
149
|
+
else:
|
|
150
|
+
search_params["version"] = artifact.version
|
|
151
|
+
response = self._session.get(url=f"{self.registry_url}/service/rest/v1/search/assets",
|
|
152
|
+
params=search_params,
|
|
153
|
+
timeout=self.timeout)
|
|
154
|
+
if response.status_code != 200:
|
|
155
|
+
raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {response.status_code}!")
|
|
156
|
+
return [result["downloadUrl"] for result in response.json()["items"]]
|
|
157
|
+
|
|
158
|
+
def with_aws_code_artifact(self, access_key: str, secret_key: str, domain: str, region_name: str, repository: str):
|
|
159
|
+
"""
|
|
160
|
+
Initializes this client to work with **AWS Code Artifact** repository.
|
|
161
|
+
Requires `access_key` and `secret_key` of a service account.
|
|
162
|
+
Also requires `domain`, `region_name` and `repository` of used AWS instance.
|
|
163
|
+
"""
|
|
164
|
+
import boto3
|
|
165
|
+
from botocore.config import Config
|
|
166
|
+
self._aws_client = boto3.client(service_name='codeartifact',
|
|
167
|
+
config=Config(region_name=region_name),
|
|
168
|
+
aws_access_key_id=access_key,
|
|
169
|
+
aws_secret_access_key=secret_key,
|
|
170
|
+
)
|
|
171
|
+
self._domain = domain
|
|
172
|
+
self._repository = repository
|
|
173
|
+
self._search_func = self._aws_search
|
|
174
|
+
self._download_func = self._aws_download
|
|
175
|
+
self.is_init = True
|
|
176
|
+
return self
|
|
177
|
+
|
|
178
|
+
def _aws_search(self, artifact: Artifact = None) -> list[str]:
|
|
179
|
+
list_packages_response = self._aws_client.list_packages(domain=self._domain, repository=self._repository,
|
|
180
|
+
format="maven", packagePrefix=artifact.artifact_id)
|
|
181
|
+
# namespace == group_id
|
|
182
|
+
namespaces = [package.get('namespace') for package in list_packages_response.get('packages')
|
|
183
|
+
if package.get('package') == artifact.artifact_id]
|
|
184
|
+
if not namespaces:
|
|
185
|
+
logging.warning(f"Found no packages with artifactId = {artifact.artifact_id}!")
|
|
186
|
+
return []
|
|
187
|
+
if len(namespaces) > 1:
|
|
188
|
+
logging.warning(f"Found multiple namespaces with same artifactId = {artifact.artifact_id}:\n{namespaces}")
|
|
189
|
+
|
|
190
|
+
results = []
|
|
191
|
+
for namespace in namespaces:
|
|
192
|
+
try:
|
|
193
|
+
resp = self._aws_client.list_package_version_assets(domain=self._domain, repository=self._repository,
|
|
194
|
+
format="maven", package=artifact.artifact_id,
|
|
195
|
+
packageVersion=artifact.version,
|
|
196
|
+
namespace=namespace)
|
|
197
|
+
for asset in resp.get('assets'):
|
|
198
|
+
if asset.get('name').lower().endswith(artifact.extension.lower()):
|
|
199
|
+
results.append(f"{resp.get('namespace')}/{resp.get('package')}/{resp.get('version')}/{asset.get('name')}")
|
|
200
|
+
except Exception:
|
|
201
|
+
logging.warning(f"Specific version ({artifact.version}) of package ({namespace}.{artifact.artifact_id}) not found!")
|
|
202
|
+
return results
|
|
203
|
+
|
|
204
|
+
def _aws_download(self, url: str, local_path: str):
|
|
205
|
+
"""`url` is actually AWS-specific `resource_id`, expected to be `namespace/package/version/asset_name`"""
|
|
206
|
+
asset_parts = url.split("/")
|
|
207
|
+
response = self._aws_client.get_package_version_asset(domain=self._domain, repository=self._repository,
|
|
208
|
+
format="maven", namespace=asset_parts[0],
|
|
209
|
+
package=asset_parts[1], packageVersion=asset_parts[2],
|
|
210
|
+
asset=asset_parts[3]
|
|
211
|
+
)
|
|
212
|
+
with open(local_path, 'wb') as file:
|
|
213
|
+
file.write(response.get('asset').read())
|
|
214
|
+
|
|
215
|
+
def with_gcp_artifact_registry(self, credential_params: dict, project: str, region_name: str, repository: str):
|
|
216
|
+
"""
|
|
217
|
+
Initializes this client to work with **Google Cloud Artifact Registry** repository.
|
|
218
|
+
Supports different types of authorization in `credential_params` dict:
|
|
219
|
+
- `service_account_key` key -> requires content of key-file (generate key-file for your service account first)
|
|
220
|
+
- `oidc_token_path` and `audience` key -> path to text file ("/path/to/token/file.txt") with your OIDC token and your required audience.
|
|
221
|
+
Audience should be "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID"
|
|
222
|
+
|
|
223
|
+
Also requires `project`, `region_name` and `repository` of used GCP instance.
|
|
224
|
+
"""
|
|
225
|
+
from google.cloud import artifactregistry_v1
|
|
226
|
+
from google.auth.transport.requests import AuthorizedSession
|
|
227
|
+
self._gcp_scopes = ['https://www.googleapis.com/auth/cloud-platform']
|
|
228
|
+
creds = self._gcp_get_credentials(credential_params)
|
|
229
|
+
self._gcp_client = artifactregistry_v1.ArtifactRegistryClient(credentials=creds)
|
|
230
|
+
self._session = AuthorizedSession(credentials=creds)
|
|
231
|
+
|
|
232
|
+
self._gcp_download_url = f"https://{region_name}-maven.pkg.dev/{project}/{repository}"
|
|
233
|
+
self._repo_resource_id = f"projects/{project}/locations/{region_name}/repositories/{repository}"
|
|
234
|
+
self._search_func = self._gcp_search
|
|
235
|
+
self._download_func = self._generic_download
|
|
236
|
+
self.is_init = True
|
|
237
|
+
return self
|
|
238
|
+
|
|
239
|
+
def _gcp_get_credentials(self, credential_params: dict):
|
|
240
|
+
if service_account_key := credential_params.get("service_account_key"):
|
|
241
|
+
from google.oauth2 import service_account
|
|
242
|
+
return service_account.Credentials.from_service_account_info(
|
|
243
|
+
info=json.loads(service_account_key),
|
|
244
|
+
scopes=self._gcp_scopes,
|
|
245
|
+
)
|
|
246
|
+
if credential_params.get("oidc_token_path") and credential_params.get("audience"):
|
|
247
|
+
from google.auth import identity_pool
|
|
248
|
+
return identity_pool.Credentials(
|
|
249
|
+
audience=credential_params.get("audience"),
|
|
250
|
+
subject_token_type="urn:ietf:params:oauth:token-type:jwt",
|
|
251
|
+
credential_source={"file": credential_params.get("oidc_token_path")},
|
|
252
|
+
scopes=self._gcp_scopes,
|
|
253
|
+
)
|
|
254
|
+
raise Exception("No valid authentication params found in credential_params!")
|
|
255
|
+
|
|
256
|
+
def _gcp_search(self, artifact: Artifact = None) -> list[str]:
|
|
257
|
+
timestamp_version_match = re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version)
|
|
258
|
+
is_snapshot = artifact.version.endswith("-SNAPSHOT")
|
|
259
|
+
if timestamp_version_match:
|
|
260
|
+
base_version = timestamp_version_match.group(1) + "SNAPSHOT"
|
|
261
|
+
else:
|
|
262
|
+
base_version = artifact.version
|
|
263
|
+
|
|
264
|
+
response_pager = self._gcp_client.list_maven_artifacts(parent=self._repo_resource_id)
|
|
265
|
+
for gav in response_pager:
|
|
266
|
+
if gav.artifact_id == artifact.artifact_id and gav.version == base_version:
|
|
267
|
+
artifact_folder_url = "{gcp_download_url}/{group_path}/{artifact_id}/{version}".format(
|
|
268
|
+
gcp_download_url=self._gcp_download_url,
|
|
269
|
+
group_path=gav.group_id.replace('.', '/'),
|
|
270
|
+
artifact_id=gav.artifact_id,
|
|
271
|
+
version=gav.version
|
|
272
|
+
)
|
|
273
|
+
if is_snapshot:
|
|
274
|
+
latest_snapshot_version = self._find_latest_snapshot_version(artifact_folder_url, artifact.version)
|
|
275
|
+
return [f"{artifact_folder_url}/{gav.artifact_id}-{latest_snapshot_version}.{artifact.extension}"]
|
|
276
|
+
else:
|
|
277
|
+
return [f"{artifact_folder_url}/{gav.artifact_id}-{artifact.version}.{artifact.extension}"]
|
|
278
|
+
return []
|
|
279
|
+
|
|
280
|
+
def _find_latest_snapshot_version(self, artifact_folder_url: str, snapshot_version: str) -> str:
|
|
281
|
+
response = self._session.get(url=f"{artifact_folder_url}/maven-metadata.xml", timeout=self.timeout)
|
|
282
|
+
response.raise_for_status()
|
|
283
|
+
xml = ElementTree.fromstring(response.content)
|
|
284
|
+
timestamp = xml.findall("./versioning/snapshot/timestamp")[0].text
|
|
285
|
+
build_number = xml.findall("./versioning/snapshot/buildNumber")[0].text
|
|
286
|
+
return snapshot_version.replace("SNAPSHOT", timestamp + "-" + build_number)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class AWSCodeArtifactHelper:
|
|
2
|
+
@staticmethod
|
|
3
|
+
def get_authorization_token(access_key: str, secret_key: str, domain: str, region_name: str):
|
|
4
|
+
"""
|
|
5
|
+
Fetches 12-hour temporary authorization token (using long-term credentials).
|
|
6
|
+
This token is necessary for accessing CodeArtifact using standard maven interface
|
|
7
|
+
"""
|
|
8
|
+
import boto3
|
|
9
|
+
from botocore.config import Config
|
|
10
|
+
client = boto3.client(service_name='codeartifact',
|
|
11
|
+
config=Config(region_name=region_name),
|
|
12
|
+
aws_access_key_id=access_key,
|
|
13
|
+
aws_secret_access_key=secret_key,
|
|
14
|
+
)
|
|
15
|
+
response = client.get_authorization_token(domain=domain)
|
|
16
|
+
return response.get('authorizationToken')
|
|
@@ -40,7 +40,10 @@ def init_context(context_path):
|
|
|
40
40
|
|
|
41
41
|
def create_execution_context(input_params: dict = None, input_params_secure: dict = None, folder_path: str = None,
|
|
42
42
|
parent_context_to_reuse: ExecutionContext = None):
|
|
43
|
-
"""
|
|
43
|
+
"""
|
|
44
|
+
Dynamically creates **`ExecutionContext`** using provided params.
|
|
45
|
+
|
|
46
|
+
Arguments:
|
|
44
47
|
input_params: dict (will be merged into created input params)
|
|
45
48
|
input_params_secure: dict (will be merged into created secure input params)
|
|
46
49
|
folder_path: str (optional, will generate new temp)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: qubership-pipelines-common-library
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Qubership Pipelines common library
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: Igor Lebedev
|
|
@@ -12,8 +12,10 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
14
|
Requires-Dist: GitPython (>=3.1.43,<4.0.0)
|
|
15
|
+
Requires-Dist: boto3 (>=1.39.4,<2.0.0)
|
|
15
16
|
Requires-Dist: click (>=8.1.7,<9.0.0)
|
|
16
17
|
Requires-Dist: ghapi (>=1.0.6,<2.0.0)
|
|
18
|
+
Requires-Dist: google-cloud-artifact-registry (>=1.16.1,<2.0.0)
|
|
17
19
|
Requires-Dist: http-exceptions (>=0.2.10,<0.3.0)
|
|
18
20
|
Requires-Dist: kubernetes (>=29.0.0,<30.0.0)
|
|
19
21
|
Requires-Dist: minio (>=7.2.12,<8.0.0)
|
|
@@ -2,29 +2,31 @@ qubership_pipelines_common_library/__init__.py,sha256=91r6ljRCMIXiH1mE5cME45Ostb
|
|
|
2
2
|
qubership_pipelines_common_library/v1/__init__.py,sha256=QczIlSYNOtXMuMWSznhV_BkXMM5KLn1wOogtlT2kcy0,598
|
|
3
3
|
qubership_pipelines_common_library/v1/artifactory_client.py,sha256=Gwf21BXUYNpKT_Y_wMyM07WlpDNTIBSUkSIsJlWfURg,4105
|
|
4
4
|
qubership_pipelines_common_library/v1/execution/__init__.py,sha256=QczIlSYNOtXMuMWSznhV_BkXMM5KLn1wOogtlT2kcy0,598
|
|
5
|
-
qubership_pipelines_common_library/v1/execution/exec_command.py,sha256=
|
|
6
|
-
qubership_pipelines_common_library/v1/execution/exec_context.py,sha256=
|
|
7
|
-
qubership_pipelines_common_library/v1/execution/exec_context_file.py,sha256=
|
|
8
|
-
qubership_pipelines_common_library/v1/execution/exec_info.py,sha256=
|
|
9
|
-
qubership_pipelines_common_library/v1/execution/exec_logger.py,sha256=
|
|
5
|
+
qubership_pipelines_common_library/v1/execution/exec_command.py,sha256=q499vODvHg4oP5Bd6xCAjrAMjTKtMZLGE5njth7vuY0,3317
|
|
6
|
+
qubership_pipelines_common_library/v1/execution/exec_context.py,sha256=R9Kmb4t3QRXCJTMhC3qcPtxtyvCrIV037Ix9P_VD5YI,6055
|
|
7
|
+
qubership_pipelines_common_library/v1/execution/exec_context_file.py,sha256=kbuL9mA21qhaueVe6SWvI3OM49Ekrm8v1lj1FFspBq4,7397
|
|
8
|
+
qubership_pipelines_common_library/v1/execution/exec_info.py,sha256=RsHaSdzAnOzR5XRlpU2F0IYkEAcaWBEONDlkgUPpFWs,4102
|
|
9
|
+
qubership_pipelines_common_library/v1/execution/exec_logger.py,sha256=rtSCLo3mqtwIc2S_tBs0uizehdthBGfygB1Vpwa-sRA,3102
|
|
10
10
|
qubership_pipelines_common_library/v1/git_client.py,sha256=uop4dREW0HoaAbGHSzp3P4vk1Hk-VrPK5RhAP3Hj51o,6100
|
|
11
11
|
qubership_pipelines_common_library/v1/github_client.py,sha256=cyAbPau94XfpVVvgRVk7sVgPWtp4Q-Bx-6HHgjQG3Xc,14607
|
|
12
12
|
qubership_pipelines_common_library/v1/gitlab_client.py,sha256=I8o0qBm55oO99-sDHatYaFQEniGzE3gyielocOJNDtI,8633
|
|
13
13
|
qubership_pipelines_common_library/v1/jenkins_client.py,sha256=VsD4KQNmLTeFvyVnY0m1xPv3s5bb-sNbgO6SwTJ2FfY,8597
|
|
14
14
|
qubership_pipelines_common_library/v1/kube_client.py,sha256=rbdc0Q2r6AhJ49FKr-15_1r9Uit4_6U68rWwGYDjdWc,12715
|
|
15
15
|
qubership_pipelines_common_library/v1/log_client.py,sha256=DTJ8aI_37l570RyolDC2cHaOkkccZWi7cFE6qYUuQeo,1514
|
|
16
|
+
qubership_pipelines_common_library/v1/maven_client.py,sha256=DbyPp6lh17op04GGeq2jIbk-SyVzCCHRcr2ox-eUv54,15054
|
|
16
17
|
qubership_pipelines_common_library/v1/minio_client.py,sha256=4KlkCJvtgGKQOujChxRtKrpoZVukooMLfj5D8C9CKC4,4343
|
|
17
18
|
qubership_pipelines_common_library/v1/utils/__init__.py,sha256=QczIlSYNOtXMuMWSznhV_BkXMM5KLn1wOogtlT2kcy0,598
|
|
18
19
|
qubership_pipelines_common_library/v1/utils/rest.py,sha256=MaCS6L6Khs_HaWoi3WNj9Go33d9zEVErLP5T8iVRyHA,3068
|
|
19
20
|
qubership_pipelines_common_library/v1/utils/utils.py,sha256=5PhXyFC1Zfuz0KDrWC9QgacTLVVk8zu0-6wxYS0bmzE,1865
|
|
21
|
+
qubership_pipelines_common_library/v1/utils/utils_aws.py,sha256=BPPnHBzPPXPqFijtAiw16sTPu1tFZjS95GkSMX_HdjA,808
|
|
20
22
|
qubership_pipelines_common_library/v1/utils/utils_cli.py,sha256=0bBoeaZQ35zXaK6KO0EZvaQXKiH5kx3PcmAwyj80mzQ,2886
|
|
21
|
-
qubership_pipelines_common_library/v1/utils/utils_context.py,sha256=
|
|
23
|
+
qubership_pipelines_common_library/v1/utils/utils_context.py,sha256=IlMFXGxS8zJw33Gu3SbOUcj88wquIkobBlWkdFbR7MA,3767
|
|
22
24
|
qubership_pipelines_common_library/v1/utils/utils_dictionary.py,sha256=6wGAoBmLzPGGqdtkoqU9RtMBYuOO-UkZsZDh7GzubjA,1365
|
|
23
25
|
qubership_pipelines_common_library/v1/utils/utils_file.py,sha256=6tCGosFjtycGJq0LtR53MiAyR8-VAxiT0-1quJ6FhcE,2233
|
|
24
26
|
qubership_pipelines_common_library/v1/utils/utils_json.py,sha256=QczIlSYNOtXMuMWSznhV_BkXMM5KLn1wOogtlT2kcy0,598
|
|
25
27
|
qubership_pipelines_common_library/v1/utils/utils_string.py,sha256=Phx5ZXPRjhjg9AaSPx6WLX9zQvwJH1txslfnG3jJ43w,993
|
|
26
28
|
qubership_pipelines_common_library/v1/webex_client.py,sha256=3Ij4EGRX6bCq23dmj24E0TZ29Fq-7vd5Ejlqo0hbFvU,2860
|
|
27
|
-
qubership_pipelines_common_library-0.
|
|
28
|
-
qubership_pipelines_common_library-0.
|
|
29
|
-
qubership_pipelines_common_library-0.
|
|
30
|
-
qubership_pipelines_common_library-0.
|
|
29
|
+
qubership_pipelines_common_library-0.2.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
30
|
+
qubership_pipelines_common_library-0.2.0.dist-info/METADATA,sha256=UsznhtBrgJu66xboqQxFwKScb8b5SHdMcaYvGUpmO1E,2510
|
|
31
|
+
qubership_pipelines_common_library-0.2.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
32
|
+
qubership_pipelines_common_library-0.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|