qubership-pipelines-common-library 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qubership_pipelines_common_library/__init__.py +1 -1
- qubership_pipelines_common_library/v1/artifactory_client.py +1 -1
- qubership_pipelines_common_library/v1/execution/exec_command.py +11 -1
- qubership_pipelines_common_library/v1/execution/exec_context.py +6 -6
- qubership_pipelines_common_library/v1/execution/exec_context_file.py +1 -1
- qubership_pipelines_common_library/v1/execution/exec_logger.py +7 -5
- qubership_pipelines_common_library/v1/github_client.py +1 -1
- qubership_pipelines_common_library/v1/gitlab_client.py +11 -7
- qubership_pipelines_common_library/v1/jenkins_client.py +55 -18
- qubership_pipelines_common_library/v1/maven_client.py +2 -2
- qubership_pipelines_common_library/v1/minio_client.py +1 -1
- qubership_pipelines_common_library/v1/utils/rest.py +1 -1
- qubership_pipelines_common_library/v1/utils/utils.py +1 -1
- qubership_pipelines_common_library/v1/utils/utils_cli.py +43 -9
- qubership_pipelines_common_library/v1/utils/utils_dictionary.py +1 -1
- qubership_pipelines_common_library/v1/utils/utils_logging.py +53 -0
- qubership_pipelines_common_library/v2/artifacts_finder/__init__.py +0 -0
- qubership_pipelines_common_library/v2/artifacts_finder/artifact_finder.py +56 -0
- qubership_pipelines_common_library/v2/artifacts_finder/auth/__init__.py +0 -0
- qubership_pipelines_common_library/v2/artifacts_finder/auth/aws_credentials.py +106 -0
- qubership_pipelines_common_library/v2/artifacts_finder/auth/azure_credentials.py +72 -0
- qubership_pipelines_common_library/v2/artifacts_finder/auth/gcp_credentials.py +88 -0
- qubership_pipelines_common_library/v2/artifacts_finder/model/__init__.py +0 -0
- qubership_pipelines_common_library/v2/artifacts_finder/model/artifact.py +20 -0
- qubership_pipelines_common_library/v2/artifacts_finder/model/artifact_provider.py +35 -0
- qubership_pipelines_common_library/v2/artifacts_finder/model/credentials.py +16 -0
- qubership_pipelines_common_library/v2/artifacts_finder/model/credentials_provider.py +16 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/__init__.py +0 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/artifactory.py +52 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/aws_code_artifact.py +79 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/azure_artifacts.py +98 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/gcp_artifact_registry.py +50 -0
- qubership_pipelines_common_library/v2/artifacts_finder/providers/nexus.py +41 -0
- qubership_pipelines_common_library/v2/github/github_run_pipeline_command.py +3 -3
- qubership_pipelines_common_library/v2/gitlab/custom_extensions.py +1 -1
- qubership_pipelines_common_library/v2/gitlab/gitlab_run_pipeline_command.py +4 -4
- qubership_pipelines_common_library/v2/jenkins/__init__.py +0 -0
- qubership_pipelines_common_library/v2/jenkins/custom_extensions.py +63 -0
- qubership_pipelines_common_library/v2/jenkins/jenkins_client.py +5 -0
- qubership_pipelines_common_library/v2/jenkins/jenkins_pipeline_data_importer.py +31 -0
- qubership_pipelines_common_library/v2/jenkins/jenkins_run_pipeline_command.py +165 -0
- qubership_pipelines_common_library/v2/jenkins/safe_jenkins_client.py +14 -0
- qubership_pipelines_common_library/v2/podman/podman_command.md +7 -1
- qubership_pipelines_common_library/v2/podman/podman_command.py +4 -4
- qubership_pipelines_common_library/v2/sops/sops_client.py +2 -2
- qubership_pipelines_common_library/v2/utils/retry_decorator.py +5 -5
- {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/METADATA +5 -3
- qubership_pipelines_common_library-2.0.1.dist-info/RECORD +76 -0
- qubership_pipelines_common_library-2.0.0.dist-info/RECORD +0 -52
- {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/WHEEL +0 -0
- {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import boto3
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
from botocore.config import Config
|
|
5
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
6
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AwsCredentialsProvider(CloudCredentialsProvider):
|
|
10
|
+
|
|
11
|
+
DEFAULT_ROLE_SESSION_NAME = "codeartifact-session"
|
|
12
|
+
|
|
13
|
+
access_key: str
|
|
14
|
+
secret_key: str
|
|
15
|
+
session_token: str
|
|
16
|
+
region_name: str
|
|
17
|
+
role_arn: str
|
|
18
|
+
role_session_name: str
|
|
19
|
+
_auth_type = None
|
|
20
|
+
|
|
21
|
+
class AuthType(StrEnum):
|
|
22
|
+
DIRECT = 'DIRECT'
|
|
23
|
+
ASSUME_ROLE = 'ASSUME_ROLE'
|
|
24
|
+
|
|
25
|
+
def with_direct_credentials(self,
|
|
26
|
+
access_key: str,
|
|
27
|
+
secret_key: str,
|
|
28
|
+
region_name: str,
|
|
29
|
+
session_token: str | None = None,
|
|
30
|
+
):
|
|
31
|
+
self.access_key = access_key
|
|
32
|
+
self.secret_key = secret_key
|
|
33
|
+
self.session_token = session_token
|
|
34
|
+
self.region_name = region_name
|
|
35
|
+
self.validate_mandatory_attrs(["access_key", "secret_key", "region_name"])
|
|
36
|
+
self._auth_type = self.AuthType.DIRECT
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def with_assume_role(self,
|
|
40
|
+
access_key: str,
|
|
41
|
+
secret_key: str,
|
|
42
|
+
region_name: str,
|
|
43
|
+
role_arn: str,
|
|
44
|
+
role_session_name: str | None = DEFAULT_ROLE_SESSION_NAME,
|
|
45
|
+
):
|
|
46
|
+
self.access_key = access_key
|
|
47
|
+
self.secret_key = secret_key
|
|
48
|
+
self.region_name = region_name
|
|
49
|
+
self.role_arn = role_arn
|
|
50
|
+
self.role_session_name = role_session_name
|
|
51
|
+
self.validate_mandatory_attrs(["access_key", "secret_key", "region_name", "role_arn", "role_session_name"])
|
|
52
|
+
self._auth_type = self.AuthType.ASSUME_ROLE
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def get_credentials(self) -> Credentials:
|
|
56
|
+
if self._auth_type is self.AuthType.DIRECT:
|
|
57
|
+
return self._get_direct_credentials()
|
|
58
|
+
elif self._auth_type is self.AuthType.ASSUME_ROLE:
|
|
59
|
+
return self._get_assume_role_credentials()
|
|
60
|
+
else:
|
|
61
|
+
raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
|
|
62
|
+
|
|
63
|
+
def get_ecr_authorization_token(self) -> str:
|
|
64
|
+
creds = self.get_credentials()
|
|
65
|
+
ecr_client = boto3.client(
|
|
66
|
+
service_name="ecr",
|
|
67
|
+
config=Config(region_name=self.region_name),
|
|
68
|
+
aws_access_key_id=creds["access_key"],
|
|
69
|
+
aws_secret_access_key=creds["secret_key"],
|
|
70
|
+
aws_session_token=creds["session_token"],
|
|
71
|
+
)
|
|
72
|
+
ecr_authorization_token = ecr_client.get_authorization_token()
|
|
73
|
+
ecr_authorization_data = ecr_authorization_token["authorizationData"][0]
|
|
74
|
+
return ecr_authorization_data["authorizationToken"]
|
|
75
|
+
|
|
76
|
+
def _get_direct_credentials(self) -> Credentials:
|
|
77
|
+
return Credentials(
|
|
78
|
+
access_key=self.access_key,
|
|
79
|
+
secret_key=self.secret_key,
|
|
80
|
+
session_token=self.session_token,
|
|
81
|
+
region_name=self.region_name,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def _get_assume_role_credentials(self) -> Credentials:
|
|
85
|
+
creds = self._assume_role()
|
|
86
|
+
return Credentials(
|
|
87
|
+
access_key=creds["AccessKeyId"],
|
|
88
|
+
secret_key=creds["SecretAccessKey"],
|
|
89
|
+
session_token=creds["SessionToken"],
|
|
90
|
+
role_arn=self.role_arn,
|
|
91
|
+
role_session_name=self.role_session_name,
|
|
92
|
+
region_name=self.region_name,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def _assume_role(self) -> dict:
|
|
96
|
+
sts_client = boto3.client(
|
|
97
|
+
service_name="sts",
|
|
98
|
+
config=Config(region_name=self.region_name),
|
|
99
|
+
aws_access_key_id=self.access_key,
|
|
100
|
+
aws_secret_access_key=self.secret_key,
|
|
101
|
+
)
|
|
102
|
+
assumed = sts_client.assume_role(
|
|
103
|
+
RoleArn=self.role_arn,
|
|
104
|
+
RoleSessionName=self.role_session_name,
|
|
105
|
+
)
|
|
106
|
+
return assumed["Credentials"]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
5
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AzureCredentialsProvider(CloudCredentialsProvider):
|
|
9
|
+
|
|
10
|
+
tenant_id: str
|
|
11
|
+
client_id: str
|
|
12
|
+
client_secret: str
|
|
13
|
+
target_resource: str
|
|
14
|
+
_auth_data: dict
|
|
15
|
+
_auth_type = None
|
|
16
|
+
|
|
17
|
+
class AuthType(StrEnum):
|
|
18
|
+
OAUTH2 = 'OAUTH2'
|
|
19
|
+
|
|
20
|
+
def with_oauth2(self,
|
|
21
|
+
tenant_id: str,
|
|
22
|
+
client_id: str,
|
|
23
|
+
client_secret: str,
|
|
24
|
+
target_resource: str,
|
|
25
|
+
):
|
|
26
|
+
self.tenant_id = tenant_id
|
|
27
|
+
self.client_id = client_id
|
|
28
|
+
self.client_secret = client_secret
|
|
29
|
+
self.target_resource = target_resource
|
|
30
|
+
self.validate_mandatory_attrs(["tenant_id", "client_id", "client_secret", "target_resource"])
|
|
31
|
+
self._auth_data = {
|
|
32
|
+
"grant_type": "client_credentials",
|
|
33
|
+
"client_id": client_id,
|
|
34
|
+
"client_secret": client_secret,
|
|
35
|
+
"scope": f"{target_resource}/.default"
|
|
36
|
+
}
|
|
37
|
+
self._auth_type = self.AuthType.OAUTH2
|
|
38
|
+
return self
|
|
39
|
+
|
|
40
|
+
def with_oauth2_custom_data(self,
|
|
41
|
+
tenant_id: str,
|
|
42
|
+
custom_auth_data: dict,
|
|
43
|
+
):
|
|
44
|
+
self.tenant_id = tenant_id
|
|
45
|
+
self._auth_data = custom_auth_data
|
|
46
|
+
self.validate_mandatory_attrs(["tenant_id", "_auth_data"])
|
|
47
|
+
self._auth_type = self.AuthType.OAUTH2
|
|
48
|
+
return self
|
|
49
|
+
|
|
50
|
+
def get_credentials(self) -> Credentials:
|
|
51
|
+
if self._auth_type is self.AuthType.OAUTH2:
|
|
52
|
+
return self._get_oauth2_credentials()
|
|
53
|
+
else:
|
|
54
|
+
raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
|
|
55
|
+
|
|
56
|
+
def _get_oauth2_credentials(self) -> Credentials:
|
|
57
|
+
token_url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token"
|
|
58
|
+
response = requests.post(
|
|
59
|
+
token_url,
|
|
60
|
+
data=self._auth_data,
|
|
61
|
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
62
|
+
)
|
|
63
|
+
response.raise_for_status()
|
|
64
|
+
response_json = response.json()
|
|
65
|
+
access_token = response_json.get("access_token")
|
|
66
|
+
if not access_token:
|
|
67
|
+
raise Exception(f"Failed to get access token from {token_url}: {response_json}")
|
|
68
|
+
|
|
69
|
+
return Credentials(
|
|
70
|
+
access_token=access_token,
|
|
71
|
+
tenant_id=self.tenant_id,
|
|
72
|
+
)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from enum import StrEnum
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from google.auth.transport.requests import AuthorizedSession, Request
|
|
6
|
+
|
|
7
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
8
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class GcpCredentialsProvider(CloudCredentialsProvider):
|
|
12
|
+
|
|
13
|
+
DEFAULT_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
|
|
14
|
+
DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"
|
|
15
|
+
|
|
16
|
+
service_account_key_content: str
|
|
17
|
+
oidc_credential_source: dict
|
|
18
|
+
audience: str
|
|
19
|
+
scopes: list[str]
|
|
20
|
+
subject_token_type: str
|
|
21
|
+
_auth_type = None
|
|
22
|
+
|
|
23
|
+
class AuthType(StrEnum):
|
|
24
|
+
SA_KEY = 'SA_KEY'
|
|
25
|
+
OIDC_CREDS = 'OIDC_CREDS'
|
|
26
|
+
|
|
27
|
+
def with_service_account_key(self,
|
|
28
|
+
service_account_key_content: str | None = None,
|
|
29
|
+
service_account_key_path: str | Path | None = None,
|
|
30
|
+
scopes: list[str] = None,
|
|
31
|
+
):
|
|
32
|
+
if scopes is None:
|
|
33
|
+
scopes = self.DEFAULT_SCOPES
|
|
34
|
+
self.scopes = scopes
|
|
35
|
+
self.service_account_key_content = service_account_key_content
|
|
36
|
+
if service_account_key_path:
|
|
37
|
+
with open(service_account_key_path, 'r') as key_file:
|
|
38
|
+
self.service_account_key_content = key_file.read()
|
|
39
|
+
self.validate_mandatory_attrs(["service_account_key_content"])
|
|
40
|
+
self._auth_type = self.AuthType.SA_KEY
|
|
41
|
+
return self
|
|
42
|
+
|
|
43
|
+
def with_oidc_creds(self,
|
|
44
|
+
oidc_credential_source: dict,
|
|
45
|
+
audience: str,
|
|
46
|
+
oidc_token_file_path: str = None,
|
|
47
|
+
subject_token_type: str = DEFAULT_SUBJECT_TOKEN_TYPE,
|
|
48
|
+
scopes: list[str] = None,
|
|
49
|
+
):
|
|
50
|
+
if scopes is None:
|
|
51
|
+
scopes = self.DEFAULT_SCOPES
|
|
52
|
+
self.scopes = scopes
|
|
53
|
+
self.oidc_credential_source = oidc_credential_source
|
|
54
|
+
if oidc_token_file_path:
|
|
55
|
+
self.oidc_credential_source = {"file": oidc_token_file_path}
|
|
56
|
+
self.audience = audience
|
|
57
|
+
self.subject_token_type = subject_token_type
|
|
58
|
+
self.validate_mandatory_attrs(["oidc_credential_source", "audience"])
|
|
59
|
+
self._auth_type = self.AuthType.OIDC_CREDS
|
|
60
|
+
return self
|
|
61
|
+
|
|
62
|
+
def get_credentials(self) -> Credentials:
|
|
63
|
+
if self._auth_type == self.AuthType.SA_KEY:
|
|
64
|
+
from google.oauth2 import service_account
|
|
65
|
+
google_creds = service_account.Credentials.from_service_account_info(
|
|
66
|
+
info=json.loads(self.service_account_key_content),
|
|
67
|
+
scopes=self.scopes,
|
|
68
|
+
)
|
|
69
|
+
elif self._auth_type == self.AuthType.OIDC_CREDS:
|
|
70
|
+
from google.auth import identity_pool
|
|
71
|
+
google_creds = identity_pool.Credentials(
|
|
72
|
+
audience=self.audience,
|
|
73
|
+
subject_token_type=self.subject_token_type,
|
|
74
|
+
credential_source=self.oidc_credential_source,
|
|
75
|
+
scopes=self.scopes,
|
|
76
|
+
)
|
|
77
|
+
else:
|
|
78
|
+
raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
|
|
79
|
+
|
|
80
|
+
google_creds.refresh(Request())
|
|
81
|
+
creds = Credentials(
|
|
82
|
+
google_credentials_object=google_creds,
|
|
83
|
+
gcp_authorization_token=google_creds.token, # It can be used in Basic Authorization (in some Execution Commands)
|
|
84
|
+
authorized_session=AuthorizedSession(credentials=google_creds),
|
|
85
|
+
)
|
|
86
|
+
if self._auth_type == self.AuthType.SA_KEY:
|
|
87
|
+
creds.service_account_key_content = self.service_account_key_content
|
|
88
|
+
return creds
|
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
class Artifact:
|
|
2
|
+
def __init__(self, group_id = None, artifact_id = None, version = None, extension='jar'):
|
|
3
|
+
self.group_id = group_id
|
|
4
|
+
self.artifact_id = artifact_id
|
|
5
|
+
self.version = version
|
|
6
|
+
self.extension = "jar" if not extension else extension
|
|
7
|
+
|
|
8
|
+
def is_snapshot(self):
|
|
9
|
+
return self.version and self.version.endswith("SNAPSHOT")
|
|
10
|
+
|
|
11
|
+
def get_filename(self) -> str:
|
|
12
|
+
return f"{self.artifact_id}-{self.version}.{self.extension}"
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
def from_string(artifact_str: str):
|
|
16
|
+
parts = artifact_str.split(":")
|
|
17
|
+
if len(parts) == 3:
|
|
18
|
+
group, artifact, version = parts[0], parts[1], parts[-1]
|
|
19
|
+
return Artifact(group, artifact, version)
|
|
20
|
+
raise Exception("Invalid artifact string")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ArtifactProvider(ABC):
|
|
9
|
+
"""Base class for all artifact providers"""
|
|
10
|
+
|
|
11
|
+
TIMESTAMP_VERSION_PATTERN = "^(.*-)?([0-9]{8}\\.[0-9]{6}-[0-9]+)$"
|
|
12
|
+
|
|
13
|
+
def __init__(self, params: dict = None, **kwargs):
|
|
14
|
+
self.params = params if params else {}
|
|
15
|
+
self._session = requests.Session()
|
|
16
|
+
self._session.verify = self.params.get('verify', True)
|
|
17
|
+
self.timeout = self.params.get('timeout', None)
|
|
18
|
+
|
|
19
|
+
def generic_download(self, resource_url: str, local_path: str | Path):
|
|
20
|
+
response = self._session.get(url=resource_url, timeout=self.timeout)
|
|
21
|
+
response.raise_for_status()
|
|
22
|
+
with open(local_path, 'wb') as file:
|
|
23
|
+
file.write(response.content)
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
@abstractmethod
|
|
30
|
+
def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def get_provider_name(self) -> str:
|
|
35
|
+
pass
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class Credentials(dict):
|
|
2
|
+
def __init__(self, *args, **kwargs):
|
|
3
|
+
super().__init__(*args, **kwargs)
|
|
4
|
+
self.__dict__ = self # allow attribute access
|
|
5
|
+
|
|
6
|
+
def __getitem__(self, key):
|
|
7
|
+
return self.get(key)
|
|
8
|
+
|
|
9
|
+
def __getattr__(self, key):
|
|
10
|
+
try:
|
|
11
|
+
return self[key]
|
|
12
|
+
except KeyError:
|
|
13
|
+
return None
|
|
14
|
+
|
|
15
|
+
def __setattr__(self, key, value):
|
|
16
|
+
self[key] = value
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CloudCredentialsProvider(ABC):
|
|
7
|
+
"""Base class for all cloud credentials"""
|
|
8
|
+
|
|
9
|
+
@abstractmethod
|
|
10
|
+
def get_credentials(self) -> Credentials:
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
def validate_mandatory_attrs(self, attrs_names) -> None:
|
|
14
|
+
missing_attrs = [attr for attr in attrs_names if getattr(self, attr, None) is None]
|
|
15
|
+
if missing_attrs:
|
|
16
|
+
raise ValueError(f"The following mandatory attributes are not set: {', '.join(missing_attrs)}")
|
|
File without changes
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
|
|
6
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ArtifactoryProvider(ArtifactProvider):
|
|
10
|
+
|
|
11
|
+
def __init__(self, registry_url: str, username: str = None, password: str = None, **kwargs):
|
|
12
|
+
"""
|
|
13
|
+
Initializes this client to work with **JFrog Artifactory** maven repositories.
|
|
14
|
+
Requires `username` and its `password` or `token`.
|
|
15
|
+
"""
|
|
16
|
+
super().__init__(**kwargs)
|
|
17
|
+
self.registry_url = registry_url
|
|
18
|
+
if password:
|
|
19
|
+
from requests.auth import HTTPBasicAuth
|
|
20
|
+
self._session.auth = HTTPBasicAuth(username, password)
|
|
21
|
+
|
|
22
|
+
def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
|
|
23
|
+
return self.generic_download(resource_url=resource_url, local_path=local_path)
|
|
24
|
+
|
|
25
|
+
def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
|
|
26
|
+
timestamp_version_match = re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version)
|
|
27
|
+
if timestamp_version_match:
|
|
28
|
+
base_version = timestamp_version_match.group(1) + "SNAPSHOT"
|
|
29
|
+
else:
|
|
30
|
+
base_version = artifact.version
|
|
31
|
+
|
|
32
|
+
search_params = {
|
|
33
|
+
**({"g": artifact.group_id} if artifact.group_id else {}),
|
|
34
|
+
"a": artifact.artifact_id,
|
|
35
|
+
"v": base_version,
|
|
36
|
+
"specific": "true"
|
|
37
|
+
}
|
|
38
|
+
search_api_url = f"{self.registry_url}/api/search/gavc"
|
|
39
|
+
logging.debug(f"Search URL: {search_api_url}"f"\nSearch Parameters: {search_params}")
|
|
40
|
+
|
|
41
|
+
response = self._session.get(url=search_api_url,
|
|
42
|
+
params=search_params,
|
|
43
|
+
timeout=self.timeout)
|
|
44
|
+
if response.status_code != 200:
|
|
45
|
+
raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {response.status_code}!")
|
|
46
|
+
|
|
47
|
+
return [result["downloadUri"] for result in response.json()["results"]
|
|
48
|
+
if result["ext"] == artifact.extension
|
|
49
|
+
and (not timestamp_version_match or result["downloadUri"].endswith(f"{artifact.version}.{artifact.extension}"))]
|
|
50
|
+
|
|
51
|
+
def get_provider_name(self) -> str:
|
|
52
|
+
return "artifactory"
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import boto3
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from botocore.config import Config
|
|
6
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
|
|
7
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
|
|
8
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AwsCodeArtifactProvider(ArtifactProvider):
|
|
12
|
+
|
|
13
|
+
def __init__(self, credentials: Credentials, domain: str, repository: str, package_format: str = "generic", **kwargs):
|
|
14
|
+
"""
|
|
15
|
+
Initializes this client to work with **AWS Code Artifact** for generic or maven artifacts.
|
|
16
|
+
Requires `Credentials` provided by `AwsCredentialsProvider`.
|
|
17
|
+
"""
|
|
18
|
+
super().__init__(**kwargs)
|
|
19
|
+
self._credentials = credentials
|
|
20
|
+
self._domain = domain
|
|
21
|
+
self._repository = repository
|
|
22
|
+
self._format = package_format
|
|
23
|
+
self._aws_client = boto3.client(
|
|
24
|
+
service_name='codeartifact',
|
|
25
|
+
config=Config(region_name=credentials.region_name),
|
|
26
|
+
aws_access_key_id=credentials.access_key,
|
|
27
|
+
aws_secret_access_key=credentials.secret_key,
|
|
28
|
+
aws_session_token=credentials.session_token,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
|
|
32
|
+
""" 'resource_url' is actually AWS-specific resource_id, expected to be "namespace/package/version/asset_name" """
|
|
33
|
+
asset_parts = resource_url.split("/")
|
|
34
|
+
response = self._aws_client.get_package_version_asset(
|
|
35
|
+
domain=self._domain, repository=self._repository,
|
|
36
|
+
format=self._format, namespace=asset_parts[0],
|
|
37
|
+
package=asset_parts[1], packageVersion=asset_parts[2],
|
|
38
|
+
asset=asset_parts[3]
|
|
39
|
+
)
|
|
40
|
+
with open(local_path, 'wb') as file:
|
|
41
|
+
file.write(response.get('asset').read())
|
|
42
|
+
|
|
43
|
+
def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
|
|
44
|
+
list_packages_response = self._aws_client.list_packages(
|
|
45
|
+
domain=self._domain, repository=self._repository,
|
|
46
|
+
format=self._format, packagePrefix=artifact.artifact_id
|
|
47
|
+
)
|
|
48
|
+
logging.debug(f"list_packages_response: {list_packages_response}")
|
|
49
|
+
|
|
50
|
+
namespaces = [package.get('namespace') for package in list_packages_response.get('packages')
|
|
51
|
+
if package.get('package') == artifact.artifact_id]
|
|
52
|
+
logging.debug(f"namespaces: {namespaces}")
|
|
53
|
+
|
|
54
|
+
if not namespaces:
|
|
55
|
+
logging.warning(f"Found no packages with artifactId = {artifact.artifact_id}!")
|
|
56
|
+
return []
|
|
57
|
+
if len(namespaces) > 1:
|
|
58
|
+
logging.warning(f"Found multiple namespaces with same artifactId = {artifact.artifact_id}:\n{namespaces}")
|
|
59
|
+
|
|
60
|
+
results = []
|
|
61
|
+
for namespace in namespaces:
|
|
62
|
+
try:
|
|
63
|
+
assets_response = self._aws_client.list_package_version_assets(
|
|
64
|
+
domain=self._domain, repository=self._repository,
|
|
65
|
+
format=self._format, package=artifact.artifact_id,
|
|
66
|
+
packageVersion=artifact.version, namespace=namespace
|
|
67
|
+
)
|
|
68
|
+
logging.debug(f"assets: {assets_response}")
|
|
69
|
+
for asset in assets_response.get('assets'):
|
|
70
|
+
if asset.get('name').lower().endswith(artifact.extension.lower()):
|
|
71
|
+
results.append(f"{assets_response.get('namespace')}/{assets_response.get('package')}/"
|
|
72
|
+
f"{assets_response.get('version')}/{asset.get('name')}")
|
|
73
|
+
except Exception:
|
|
74
|
+
logging.warning(f"Specific version ({artifact.version}) of package ({namespace}.{artifact.artifact_id}) not found!")
|
|
75
|
+
logging.info(f"AWS search results: {results}")
|
|
76
|
+
return results
|
|
77
|
+
|
|
78
|
+
def get_provider_name(self) -> str:
|
|
79
|
+
return "aws_code_artifact"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from requests.auth import HTTPBasicAuth
|
|
6
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
|
|
7
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
|
|
8
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AzureArtifactsProvider(ArtifactProvider):
|
|
12
|
+
|
|
13
|
+
def __init__(self, credentials: Credentials, organization: str, project: str, feed: str, **kwargs):
|
|
14
|
+
"""
|
|
15
|
+
Initializes this client to work with **Azure Artifacts** for generic artifacts.
|
|
16
|
+
Requires `Credentials` provided by `AzureCredentialsProvider`.
|
|
17
|
+
"""
|
|
18
|
+
super().__init__(**kwargs)
|
|
19
|
+
self._credentials = credentials
|
|
20
|
+
self._session.auth = HTTPBasicAuth("", self._credentials.access_token)
|
|
21
|
+
self.organization = organization
|
|
22
|
+
self.project = project
|
|
23
|
+
self.feed = feed
|
|
24
|
+
|
|
25
|
+
def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
|
|
26
|
+
return self.generic_download(resource_url=resource_url, local_path=local_path)
|
|
27
|
+
|
|
28
|
+
def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
|
|
29
|
+
acceptable_versions = [artifact.version]
|
|
30
|
+
if timestamp_version_match := re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version):
|
|
31
|
+
acceptable_versions.append(timestamp_version_match.group(1) + "SNAPSHOT")
|
|
32
|
+
|
|
33
|
+
# Try to find package with name ~ "artifact_id"
|
|
34
|
+
feeds_search_url = f"https://feeds.dev.azure.com/{self.organization}/{self.project}/_apis/packaging/feeds/{self.feed}/packages"
|
|
35
|
+
feed_search_params = {
|
|
36
|
+
"includeAllVersions": "true",
|
|
37
|
+
"packageNameQuery": artifact.artifact_id,
|
|
38
|
+
"protocolType": "maven",
|
|
39
|
+
"api-version": "7.1",
|
|
40
|
+
}
|
|
41
|
+
feeds_response = self._session.get(url=feeds_search_url, params=feed_search_params, timeout=self.timeout)
|
|
42
|
+
feeds_response_json = feeds_response.json()
|
|
43
|
+
if feeds_response.status_code != 200:
|
|
44
|
+
logging.error(f"Feeds search error ({feeds_response.status_code}) response: {feeds_response_json}")
|
|
45
|
+
raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {feeds_response.status_code}!")
|
|
46
|
+
|
|
47
|
+
logging.debug(f"Feeds search response: {feeds_response_json}")
|
|
48
|
+
if feeds_response_json.get("count") > 1:
|
|
49
|
+
logging.warning("Found more than 1 feeds. Use the first one.")
|
|
50
|
+
elif feeds_response_json.get("count") == 0:
|
|
51
|
+
logging.warning("No feeds were found.")
|
|
52
|
+
return []
|
|
53
|
+
feed = feeds_response_json.get("value")[0]
|
|
54
|
+
feed_links = feed.get("_links", {})
|
|
55
|
+
|
|
56
|
+
# Get feed versions
|
|
57
|
+
feed_versions_url = feed_links.get("versions", {}).get("href", "")
|
|
58
|
+
feed_versions_response = self._session.get(url=feed_versions_url, timeout=self.timeout)
|
|
59
|
+
feed_versions_response_json = feed_versions_response.json()
|
|
60
|
+
if feed_versions_response.status_code != 200:
|
|
61
|
+
logging.error(f"Feed versions error ({feed_versions_response.status_code}) response: {feed_versions_response_json}")
|
|
62
|
+
raise Exception(f"Could not find feed versions, search request returned {feed_versions_response.status_code}!")
|
|
63
|
+
logging.debug(f"Feed versions response: {feed_versions_response_json}")
|
|
64
|
+
feed_versions = feed_versions_response_json.get("value")
|
|
65
|
+
|
|
66
|
+
# Filter by acceptable versions
|
|
67
|
+
logging.debug(f"Filtering by acceptable versions: '{acceptable_versions}'")
|
|
68
|
+
feed_version = [f for f in feed_versions if (f.get('protocolMetadata').get('data').get('version') in acceptable_versions)]
|
|
69
|
+
if len(feed_version) == 0:
|
|
70
|
+
logging.warning("All feed versions filtered.")
|
|
71
|
+
return []
|
|
72
|
+
filtered_feed_version = feed_version[0]
|
|
73
|
+
|
|
74
|
+
# Search for target file
|
|
75
|
+
files = [f for f in filtered_feed_version.get("files") if f.get('name').startswith(f"{artifact.artifact_id}-{artifact.version}") and f.get('name').endswith(artifact.extension)]
|
|
76
|
+
logging.debug(f"Files found: {files}")
|
|
77
|
+
if len(files) == 0:
|
|
78
|
+
logging.warning("All files filtered.")
|
|
79
|
+
return []
|
|
80
|
+
target_file = files[0]
|
|
81
|
+
|
|
82
|
+
# Build download url
|
|
83
|
+
feed_id = feed_links.get("feed").get("href").split("/")[-1] # take id from link to feed
|
|
84
|
+
feed_version = filtered_feed_version.get("version")
|
|
85
|
+
group_id = filtered_feed_version.get('protocolMetadata').get('data').get("groupId")
|
|
86
|
+
artifact_id = filtered_feed_version.get('protocolMetadata').get('data').get("artifactId")
|
|
87
|
+
target_file_name = target_file.get("name")
|
|
88
|
+
|
|
89
|
+
download_url = (
|
|
90
|
+
f"https://pkgs.dev.azure.com/{self.organization}/{self.project}/_apis/packaging/feeds/{feed_id}/maven/"
|
|
91
|
+
f"{group_id}/{artifact_id}/{feed_version}/{target_file_name}/content"
|
|
92
|
+
f"?api-version=7.1-preview.1"
|
|
93
|
+
)
|
|
94
|
+
logging.info(f"Azure search resulting url: {download_url}")
|
|
95
|
+
return [download_url]
|
|
96
|
+
|
|
97
|
+
def get_provider_name(self) -> str:
|
|
98
|
+
return "azure_artifacts"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from google.cloud import artifactregistry_v1
|
|
3
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
|
|
4
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
|
|
5
|
+
from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class GcpArtifactRegistryProvider(ArtifactProvider):
|
|
9
|
+
|
|
10
|
+
def __init__(self, credentials: Credentials, project: str, region_name: str, repository: str, **kwargs):
|
|
11
|
+
"""
|
|
12
|
+
Initializes this client to work with **GCP Artifact Registry** for generic artifacts.
|
|
13
|
+
Requires `Credentials` provided by `GcpCredentialsProvider`.
|
|
14
|
+
"""
|
|
15
|
+
super().__init__(**kwargs)
|
|
16
|
+
self._credentials = credentials
|
|
17
|
+
self._project = project
|
|
18
|
+
self._region_name = region_name
|
|
19
|
+
self._repository = repository
|
|
20
|
+
self._repo_resource_id = f"projects/{project}/locations/{region_name}/repositories/{repository}"
|
|
21
|
+
|
|
22
|
+
self._gcp_client = artifactregistry_v1.ArtifactRegistryClient(
|
|
23
|
+
credentials=self._credentials.google_credentials_object
|
|
24
|
+
)
|
|
25
|
+
self._authorized_session = self._credentials.authorized_session
|
|
26
|
+
|
|
27
|
+
def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
|
|
28
|
+
response = self._authorized_session.get(url=resource_url, timeout=self.timeout)
|
|
29
|
+
response.raise_for_status()
|
|
30
|
+
with open(local_path, 'wb') as file:
|
|
31
|
+
file.write(response.content)
|
|
32
|
+
|
|
33
|
+
def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
|
|
34
|
+
# works with both "Maven" and "Generic" type repositories
|
|
35
|
+
name_filter = f"{self._repo_resource_id}/files/*{artifact.artifact_id}-{artifact.version}.{artifact.extension}"
|
|
36
|
+
list_files_request = artifactregistry_v1.ListFilesRequest(
|
|
37
|
+
parent=f"{self._repo_resource_id}",
|
|
38
|
+
filter=f'name="{name_filter}"',
|
|
39
|
+
)
|
|
40
|
+
files = self._gcp_client.list_files(request=list_files_request)
|
|
41
|
+
# logging.debug(f"[GCP search_artifacts] files: {files}")
|
|
42
|
+
|
|
43
|
+
urls = []
|
|
44
|
+
for file in files:
|
|
45
|
+
download_url = f"https://artifactregistry.googleapis.com/download/v1/{file.name}:download?alt=media"
|
|
46
|
+
urls.append(download_url)
|
|
47
|
+
return urls
|
|
48
|
+
|
|
49
|
+
def get_provider_name(self) -> str:
|
|
50
|
+
return "gcp_artifact_registry"
|