qubership-pipelines-common-library 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. qubership_pipelines_common_library/__init__.py +1 -1
  2. qubership_pipelines_common_library/v1/artifactory_client.py +1 -1
  3. qubership_pipelines_common_library/v1/execution/exec_command.py +11 -1
  4. qubership_pipelines_common_library/v1/execution/exec_context.py +6 -6
  5. qubership_pipelines_common_library/v1/execution/exec_context_file.py +1 -1
  6. qubership_pipelines_common_library/v1/execution/exec_logger.py +7 -5
  7. qubership_pipelines_common_library/v1/github_client.py +1 -1
  8. qubership_pipelines_common_library/v1/gitlab_client.py +11 -7
  9. qubership_pipelines_common_library/v1/jenkins_client.py +55 -18
  10. qubership_pipelines_common_library/v1/maven_client.py +2 -2
  11. qubership_pipelines_common_library/v1/minio_client.py +1 -1
  12. qubership_pipelines_common_library/v1/utils/rest.py +1 -1
  13. qubership_pipelines_common_library/v1/utils/utils.py +1 -1
  14. qubership_pipelines_common_library/v1/utils/utils_cli.py +43 -9
  15. qubership_pipelines_common_library/v1/utils/utils_dictionary.py +1 -1
  16. qubership_pipelines_common_library/v1/utils/utils_logging.py +53 -0
  17. qubership_pipelines_common_library/v2/artifacts_finder/__init__.py +0 -0
  18. qubership_pipelines_common_library/v2/artifacts_finder/artifact_finder.py +56 -0
  19. qubership_pipelines_common_library/v2/artifacts_finder/auth/__init__.py +0 -0
  20. qubership_pipelines_common_library/v2/artifacts_finder/auth/aws_credentials.py +106 -0
  21. qubership_pipelines_common_library/v2/artifacts_finder/auth/azure_credentials.py +72 -0
  22. qubership_pipelines_common_library/v2/artifacts_finder/auth/gcp_credentials.py +88 -0
  23. qubership_pipelines_common_library/v2/artifacts_finder/model/__init__.py +0 -0
  24. qubership_pipelines_common_library/v2/artifacts_finder/model/artifact.py +20 -0
  25. qubership_pipelines_common_library/v2/artifacts_finder/model/artifact_provider.py +35 -0
  26. qubership_pipelines_common_library/v2/artifacts_finder/model/credentials.py +16 -0
  27. qubership_pipelines_common_library/v2/artifacts_finder/model/credentials_provider.py +16 -0
  28. qubership_pipelines_common_library/v2/artifacts_finder/providers/__init__.py +0 -0
  29. qubership_pipelines_common_library/v2/artifacts_finder/providers/artifactory.py +52 -0
  30. qubership_pipelines_common_library/v2/artifacts_finder/providers/aws_code_artifact.py +79 -0
  31. qubership_pipelines_common_library/v2/artifacts_finder/providers/azure_artifacts.py +98 -0
  32. qubership_pipelines_common_library/v2/artifacts_finder/providers/gcp_artifact_registry.py +50 -0
  33. qubership_pipelines_common_library/v2/artifacts_finder/providers/nexus.py +41 -0
  34. qubership_pipelines_common_library/v2/github/github_run_pipeline_command.py +3 -3
  35. qubership_pipelines_common_library/v2/gitlab/custom_extensions.py +1 -1
  36. qubership_pipelines_common_library/v2/gitlab/gitlab_run_pipeline_command.py +4 -4
  37. qubership_pipelines_common_library/v2/jenkins/__init__.py +0 -0
  38. qubership_pipelines_common_library/v2/jenkins/custom_extensions.py +63 -0
  39. qubership_pipelines_common_library/v2/jenkins/jenkins_client.py +5 -0
  40. qubership_pipelines_common_library/v2/jenkins/jenkins_pipeline_data_importer.py +31 -0
  41. qubership_pipelines_common_library/v2/jenkins/jenkins_run_pipeline_command.py +165 -0
  42. qubership_pipelines_common_library/v2/jenkins/safe_jenkins_client.py +14 -0
  43. qubership_pipelines_common_library/v2/podman/podman_command.md +7 -1
  44. qubership_pipelines_common_library/v2/podman/podman_command.py +4 -4
  45. qubership_pipelines_common_library/v2/sops/sops_client.py +2 -2
  46. qubership_pipelines_common_library/v2/utils/retry_decorator.py +5 -5
  47. {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/METADATA +5 -3
  48. qubership_pipelines_common_library-2.0.1.dist-info/RECORD +76 -0
  49. qubership_pipelines_common_library-2.0.0.dist-info/RECORD +0 -52
  50. {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/WHEEL +0 -0
  51. {qubership_pipelines_common_library-2.0.0.dist-info → qubership_pipelines_common_library-2.0.1.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,106 @@
1
+ import boto3
2
+
3
+ from enum import StrEnum
4
+ from botocore.config import Config
5
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
6
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
7
+
8
+
9
+ class AwsCredentialsProvider(CloudCredentialsProvider):
10
+
11
+ DEFAULT_ROLE_SESSION_NAME = "codeartifact-session"
12
+
13
+ access_key: str
14
+ secret_key: str
15
+ session_token: str
16
+ region_name: str
17
+ role_arn: str
18
+ role_session_name: str
19
+ _auth_type = None
20
+
21
+ class AuthType(StrEnum):
22
+ DIRECT = 'DIRECT'
23
+ ASSUME_ROLE = 'ASSUME_ROLE'
24
+
25
+ def with_direct_credentials(self,
26
+ access_key: str,
27
+ secret_key: str,
28
+ region_name: str,
29
+ session_token: str | None = None,
30
+ ):
31
+ self.access_key = access_key
32
+ self.secret_key = secret_key
33
+ self.session_token = session_token
34
+ self.region_name = region_name
35
+ self.validate_mandatory_attrs(["access_key", "secret_key", "region_name"])
36
+ self._auth_type = self.AuthType.DIRECT
37
+ return self
38
+
39
+ def with_assume_role(self,
40
+ access_key: str,
41
+ secret_key: str,
42
+ region_name: str,
43
+ role_arn: str,
44
+ role_session_name: str | None = DEFAULT_ROLE_SESSION_NAME,
45
+ ):
46
+ self.access_key = access_key
47
+ self.secret_key = secret_key
48
+ self.region_name = region_name
49
+ self.role_arn = role_arn
50
+ self.role_session_name = role_session_name
51
+ self.validate_mandatory_attrs(["access_key", "secret_key", "region_name", "role_arn", "role_session_name"])
52
+ self._auth_type = self.AuthType.ASSUME_ROLE
53
+ return self
54
+
55
+ def get_credentials(self) -> Credentials:
56
+ if self._auth_type is self.AuthType.DIRECT:
57
+ return self._get_direct_credentials()
58
+ elif self._auth_type is self.AuthType.ASSUME_ROLE:
59
+ return self._get_assume_role_credentials()
60
+ else:
61
+ raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
62
+
63
+ def get_ecr_authorization_token(self) -> str:
64
+ creds = self.get_credentials()
65
+ ecr_client = boto3.client(
66
+ service_name="ecr",
67
+ config=Config(region_name=self.region_name),
68
+ aws_access_key_id=creds["access_key"],
69
+ aws_secret_access_key=creds["secret_key"],
70
+ aws_session_token=creds["session_token"],
71
+ )
72
+ ecr_authorization_token = ecr_client.get_authorization_token()
73
+ ecr_authorization_data = ecr_authorization_token["authorizationData"][0]
74
+ return ecr_authorization_data["authorizationToken"]
75
+
76
+ def _get_direct_credentials(self) -> Credentials:
77
+ return Credentials(
78
+ access_key=self.access_key,
79
+ secret_key=self.secret_key,
80
+ session_token=self.session_token,
81
+ region_name=self.region_name,
82
+ )
83
+
84
+ def _get_assume_role_credentials(self) -> Credentials:
85
+ creds = self._assume_role()
86
+ return Credentials(
87
+ access_key=creds["AccessKeyId"],
88
+ secret_key=creds["SecretAccessKey"],
89
+ session_token=creds["SessionToken"],
90
+ role_arn=self.role_arn,
91
+ role_session_name=self.role_session_name,
92
+ region_name=self.region_name,
93
+ )
94
+
95
+ def _assume_role(self) -> dict:
96
+ sts_client = boto3.client(
97
+ service_name="sts",
98
+ config=Config(region_name=self.region_name),
99
+ aws_access_key_id=self.access_key,
100
+ aws_secret_access_key=self.secret_key,
101
+ )
102
+ assumed = sts_client.assume_role(
103
+ RoleArn=self.role_arn,
104
+ RoleSessionName=self.role_session_name,
105
+ )
106
+ return assumed["Credentials"]
@@ -0,0 +1,72 @@
1
+ import requests
2
+
3
+ from enum import StrEnum
4
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
5
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
6
+
7
+
8
+ class AzureCredentialsProvider(CloudCredentialsProvider):
9
+
10
+ tenant_id: str
11
+ client_id: str
12
+ client_secret: str
13
+ target_resource: str
14
+ _auth_data: dict
15
+ _auth_type = None
16
+
17
+ class AuthType(StrEnum):
18
+ OAUTH2 = 'OAUTH2'
19
+
20
+ def with_oauth2(self,
21
+ tenant_id: str,
22
+ client_id: str,
23
+ client_secret: str,
24
+ target_resource: str,
25
+ ):
26
+ self.tenant_id = tenant_id
27
+ self.client_id = client_id
28
+ self.client_secret = client_secret
29
+ self.target_resource = target_resource
30
+ self.validate_mandatory_attrs(["tenant_id", "client_id", "client_secret", "target_resource"])
31
+ self._auth_data = {
32
+ "grant_type": "client_credentials",
33
+ "client_id": client_id,
34
+ "client_secret": client_secret,
35
+ "scope": f"{target_resource}/.default"
36
+ }
37
+ self._auth_type = self.AuthType.OAUTH2
38
+ return self
39
+
40
+ def with_oauth2_custom_data(self,
41
+ tenant_id: str,
42
+ custom_auth_data: dict,
43
+ ):
44
+ self.tenant_id = tenant_id
45
+ self._auth_data = custom_auth_data
46
+ self.validate_mandatory_attrs(["tenant_id", "_auth_data"])
47
+ self._auth_type = self.AuthType.OAUTH2
48
+ return self
49
+
50
+ def get_credentials(self) -> Credentials:
51
+ if self._auth_type is self.AuthType.OAUTH2:
52
+ return self._get_oauth2_credentials()
53
+ else:
54
+ raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
55
+
56
+ def _get_oauth2_credentials(self) -> Credentials:
57
+ token_url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token"
58
+ response = requests.post(
59
+ token_url,
60
+ data=self._auth_data,
61
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
62
+ )
63
+ response.raise_for_status()
64
+ response_json = response.json()
65
+ access_token = response_json.get("access_token")
66
+ if not access_token:
67
+ raise Exception(f"Failed to get access token from {token_url}: {response_json}")
68
+
69
+ return Credentials(
70
+ access_token=access_token,
71
+ tenant_id=self.tenant_id,
72
+ )
@@ -0,0 +1,88 @@
1
+ import json
2
+ from enum import StrEnum
3
+ from pathlib import Path
4
+
5
+ from google.auth.transport.requests import AuthorizedSession, Request
6
+
7
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
8
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
9
+
10
+
11
+ class GcpCredentialsProvider(CloudCredentialsProvider):
12
+
13
+ DEFAULT_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
14
+ DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"
15
+
16
+ service_account_key_content: str
17
+ oidc_credential_source: dict
18
+ audience: str
19
+ scopes: list[str]
20
+ subject_token_type: str
21
+ _auth_type = None
22
+
23
+ class AuthType(StrEnum):
24
+ SA_KEY = 'SA_KEY'
25
+ OIDC_CREDS = 'OIDC_CREDS'
26
+
27
+ def with_service_account_key(self,
28
+ service_account_key_content: str | None = None,
29
+ service_account_key_path: str | Path | None = None,
30
+ scopes: list[str] = None,
31
+ ):
32
+ if scopes is None:
33
+ scopes = self.DEFAULT_SCOPES
34
+ self.scopes = scopes
35
+ self.service_account_key_content = service_account_key_content
36
+ if service_account_key_path:
37
+ with open(service_account_key_path, 'r') as key_file:
38
+ self.service_account_key_content = key_file.read()
39
+ self.validate_mandatory_attrs(["service_account_key_content"])
40
+ self._auth_type = self.AuthType.SA_KEY
41
+ return self
42
+
43
+ def with_oidc_creds(self,
44
+ oidc_credential_source: dict,
45
+ audience: str,
46
+ oidc_token_file_path: str = None,
47
+ subject_token_type: str = DEFAULT_SUBJECT_TOKEN_TYPE,
48
+ scopes: list[str] = None,
49
+ ):
50
+ if scopes is None:
51
+ scopes = self.DEFAULT_SCOPES
52
+ self.scopes = scopes
53
+ self.oidc_credential_source = oidc_credential_source
54
+ if oidc_token_file_path:
55
+ self.oidc_credential_source = {"file": oidc_token_file_path}
56
+ self.audience = audience
57
+ self.subject_token_type = subject_token_type
58
+ self.validate_mandatory_attrs(["oidc_credential_source", "audience"])
59
+ self._auth_type = self.AuthType.OIDC_CREDS
60
+ return self
61
+
62
+ def get_credentials(self) -> Credentials:
63
+ if self._auth_type == self.AuthType.SA_KEY:
64
+ from google.oauth2 import service_account
65
+ google_creds = service_account.Credentials.from_service_account_info(
66
+ info=json.loads(self.service_account_key_content),
67
+ scopes=self.scopes,
68
+ )
69
+ elif self._auth_type == self.AuthType.OIDC_CREDS:
70
+ from google.auth import identity_pool
71
+ google_creds = identity_pool.Credentials(
72
+ audience=self.audience,
73
+ subject_token_type=self.subject_token_type,
74
+ credential_source=self.oidc_credential_source,
75
+ scopes=self.scopes,
76
+ )
77
+ else:
78
+ raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
79
+
80
+ google_creds.refresh(Request())
81
+ creds = Credentials(
82
+ google_credentials_object=google_creds,
83
+ gcp_authorization_token=google_creds.token, # It can be used in Basic Authorization (in some Execution Commands)
84
+ authorized_session=AuthorizedSession(credentials=google_creds),
85
+ )
86
+ if self._auth_type == self.AuthType.SA_KEY:
87
+ creds.service_account_key_content = self.service_account_key_content
88
+ return creds
@@ -0,0 +1,20 @@
1
+ class Artifact:
2
+ def __init__(self, group_id = None, artifact_id = None, version = None, extension='jar'):
3
+ self.group_id = group_id
4
+ self.artifact_id = artifact_id
5
+ self.version = version
6
+ self.extension = "jar" if not extension else extension
7
+
8
+ def is_snapshot(self):
9
+ return self.version and self.version.endswith("SNAPSHOT")
10
+
11
+ def get_filename(self) -> str:
12
+ return f"{self.artifact_id}-{self.version}.{self.extension}"
13
+
14
+ @staticmethod
15
+ def from_string(artifact_str: str):
16
+ parts = artifact_str.split(":")
17
+ if len(parts) == 3:
18
+ group, artifact, version = parts[0], parts[1], parts[-1]
19
+ return Artifact(group, artifact, version)
20
+ raise Exception("Invalid artifact string")
@@ -0,0 +1,35 @@
1
+ import requests
2
+
3
+ from pathlib import Path
4
+ from abc import ABC, abstractmethod
5
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
6
+
7
+
8
+ class ArtifactProvider(ABC):
9
+ """Base class for all artifact providers"""
10
+
11
+ TIMESTAMP_VERSION_PATTERN = "^(.*-)?([0-9]{8}\\.[0-9]{6}-[0-9]+)$"
12
+
13
+ def __init__(self, params: dict = None, **kwargs):
14
+ self.params = params if params else {}
15
+ self._session = requests.Session()
16
+ self._session.verify = self.params.get('verify', True)
17
+ self.timeout = self.params.get('timeout', None)
18
+
19
+ def generic_download(self, resource_url: str, local_path: str | Path):
20
+ response = self._session.get(url=resource_url, timeout=self.timeout)
21
+ response.raise_for_status()
22
+ with open(local_path, 'wb') as file:
23
+ file.write(response.content)
24
+
25
+ @abstractmethod
26
+ def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
27
+ pass
28
+
29
+ @abstractmethod
30
+ def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
31
+ pass
32
+
33
+ @abstractmethod
34
+ def get_provider_name(self) -> str:
35
+ pass
@@ -0,0 +1,16 @@
1
+ class Credentials(dict):
2
+ def __init__(self, *args, **kwargs):
3
+ super().__init__(*args, **kwargs)
4
+ self.__dict__ = self # allow attribute access
5
+
6
+ def __getitem__(self, key):
7
+ return self.get(key)
8
+
9
+ def __getattr__(self, key):
10
+ try:
11
+ return self[key]
12
+ except KeyError:
13
+ return None
14
+
15
+ def __setattr__(self, key, value):
16
+ self[key] = value
@@ -0,0 +1,16 @@
1
+ from abc import ABC, abstractmethod
2
+
3
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
4
+
5
+
6
+ class CloudCredentialsProvider(ABC):
7
+ """Base class for all cloud credentials"""
8
+
9
+ @abstractmethod
10
+ def get_credentials(self) -> Credentials:
11
+ pass
12
+
13
+ def validate_mandatory_attrs(self, attrs_names) -> None:
14
+ missing_attrs = [attr for attr in attrs_names if getattr(self, attr, None) is None]
15
+ if missing_attrs:
16
+ raise ValueError(f"The following mandatory attributes are not set: {', '.join(missing_attrs)}")
@@ -0,0 +1,52 @@
1
+ import logging
2
+ import re
3
+
4
+ from pathlib import Path
5
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
6
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
7
+
8
+
9
+ class ArtifactoryProvider(ArtifactProvider):
10
+
11
+ def __init__(self, registry_url: str, username: str = None, password: str = None, **kwargs):
12
+ """
13
+ Initializes this client to work with **JFrog Artifactory** maven repositories.
14
+ Requires `username` and its `password` or `token`.
15
+ """
16
+ super().__init__(**kwargs)
17
+ self.registry_url = registry_url
18
+ if password:
19
+ from requests.auth import HTTPBasicAuth
20
+ self._session.auth = HTTPBasicAuth(username, password)
21
+
22
+ def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
23
+ return self.generic_download(resource_url=resource_url, local_path=local_path)
24
+
25
+ def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
26
+ timestamp_version_match = re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version)
27
+ if timestamp_version_match:
28
+ base_version = timestamp_version_match.group(1) + "SNAPSHOT"
29
+ else:
30
+ base_version = artifact.version
31
+
32
+ search_params = {
33
+ **({"g": artifact.group_id} if artifact.group_id else {}),
34
+ "a": artifact.artifact_id,
35
+ "v": base_version,
36
+ "specific": "true"
37
+ }
38
+ search_api_url = f"{self.registry_url}/api/search/gavc"
39
+ logging.debug(f"Search URL: {search_api_url}"f"\nSearch Parameters: {search_params}")
40
+
41
+ response = self._session.get(url=search_api_url,
42
+ params=search_params,
43
+ timeout=self.timeout)
44
+ if response.status_code != 200:
45
+ raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {response.status_code}!")
46
+
47
+ return [result["downloadUri"] for result in response.json()["results"]
48
+ if result["ext"] == artifact.extension
49
+ and (not timestamp_version_match or result["downloadUri"].endswith(f"{artifact.version}.{artifact.extension}"))]
50
+
51
+ def get_provider_name(self) -> str:
52
+ return "artifactory"
@@ -0,0 +1,79 @@
1
+ import logging
2
+ import boto3
3
+
4
+ from pathlib import Path
5
+ from botocore.config import Config
6
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
7
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
8
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
9
+
10
+
11
+ class AwsCodeArtifactProvider(ArtifactProvider):
12
+
13
+ def __init__(self, credentials: Credentials, domain: str, repository: str, package_format: str = "generic", **kwargs):
14
+ """
15
+ Initializes this client to work with **AWS Code Artifact** for generic or maven artifacts.
16
+ Requires `Credentials` provided by `AwsCredentialsProvider`.
17
+ """
18
+ super().__init__(**kwargs)
19
+ self._credentials = credentials
20
+ self._domain = domain
21
+ self._repository = repository
22
+ self._format = package_format
23
+ self._aws_client = boto3.client(
24
+ service_name='codeartifact',
25
+ config=Config(region_name=credentials.region_name),
26
+ aws_access_key_id=credentials.access_key,
27
+ aws_secret_access_key=credentials.secret_key,
28
+ aws_session_token=credentials.session_token,
29
+ )
30
+
31
+ def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
32
+ """ 'resource_url' is actually AWS-specific resource_id, expected to be "namespace/package/version/asset_name" """
33
+ asset_parts = resource_url.split("/")
34
+ response = self._aws_client.get_package_version_asset(
35
+ domain=self._domain, repository=self._repository,
36
+ format=self._format, namespace=asset_parts[0],
37
+ package=asset_parts[1], packageVersion=asset_parts[2],
38
+ asset=asset_parts[3]
39
+ )
40
+ with open(local_path, 'wb') as file:
41
+ file.write(response.get('asset').read())
42
+
43
+ def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
44
+ list_packages_response = self._aws_client.list_packages(
45
+ domain=self._domain, repository=self._repository,
46
+ format=self._format, packagePrefix=artifact.artifact_id
47
+ )
48
+ logging.debug(f"list_packages_response: {list_packages_response}")
49
+
50
+ namespaces = [package.get('namespace') for package in list_packages_response.get('packages')
51
+ if package.get('package') == artifact.artifact_id]
52
+ logging.debug(f"namespaces: {namespaces}")
53
+
54
+ if not namespaces:
55
+ logging.warning(f"Found no packages with artifactId = {artifact.artifact_id}!")
56
+ return []
57
+ if len(namespaces) > 1:
58
+ logging.warning(f"Found multiple namespaces with same artifactId = {artifact.artifact_id}:\n{namespaces}")
59
+
60
+ results = []
61
+ for namespace in namespaces:
62
+ try:
63
+ assets_response = self._aws_client.list_package_version_assets(
64
+ domain=self._domain, repository=self._repository,
65
+ format=self._format, package=artifact.artifact_id,
66
+ packageVersion=artifact.version, namespace=namespace
67
+ )
68
+ logging.debug(f"assets: {assets_response}")
69
+ for asset in assets_response.get('assets'):
70
+ if asset.get('name').lower().endswith(artifact.extension.lower()):
71
+ results.append(f"{assets_response.get('namespace')}/{assets_response.get('package')}/"
72
+ f"{assets_response.get('version')}/{asset.get('name')}")
73
+ except Exception:
74
+ logging.warning(f"Specific version ({artifact.version}) of package ({namespace}.{artifact.artifact_id}) not found!")
75
+ logging.info(f"AWS search results: {results}")
76
+ return results
77
+
78
+ def get_provider_name(self) -> str:
79
+ return "aws_code_artifact"
@@ -0,0 +1,98 @@
1
+ import logging
2
+ import re
3
+
4
+ from pathlib import Path
5
+ from requests.auth import HTTPBasicAuth
6
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
7
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
8
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
9
+
10
+
11
+ class AzureArtifactsProvider(ArtifactProvider):
12
+
13
+ def __init__(self, credentials: Credentials, organization: str, project: str, feed: str, **kwargs):
14
+ """
15
+ Initializes this client to work with **Azure Artifacts** for generic artifacts.
16
+ Requires `Credentials` provided by `AzureCredentialsProvider`.
17
+ """
18
+ super().__init__(**kwargs)
19
+ self._credentials = credentials
20
+ self._session.auth = HTTPBasicAuth("", self._credentials.access_token)
21
+ self.organization = organization
22
+ self.project = project
23
+ self.feed = feed
24
+
25
+ def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
26
+ return self.generic_download(resource_url=resource_url, local_path=local_path)
27
+
28
+ def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
29
+ acceptable_versions = [artifact.version]
30
+ if timestamp_version_match := re.match(self.TIMESTAMP_VERSION_PATTERN, artifact.version):
31
+ acceptable_versions.append(timestamp_version_match.group(1) + "SNAPSHOT")
32
+
33
+ # Try to find package with name ~ "artifact_id"
34
+ feeds_search_url = f"https://feeds.dev.azure.com/{self.organization}/{self.project}/_apis/packaging/feeds/{self.feed}/packages"
35
+ feed_search_params = {
36
+ "includeAllVersions": "true",
37
+ "packageNameQuery": artifact.artifact_id,
38
+ "protocolType": "maven",
39
+ "api-version": "7.1",
40
+ }
41
+ feeds_response = self._session.get(url=feeds_search_url, params=feed_search_params, timeout=self.timeout)
42
+ feeds_response_json = feeds_response.json()
43
+ if feeds_response.status_code != 200:
44
+ logging.error(f"Feeds search error ({feeds_response.status_code}) response: {feeds_response_json}")
45
+ raise Exception(f"Could not find '{artifact.artifact_id}' - search request returned {feeds_response.status_code}!")
46
+
47
+ logging.debug(f"Feeds search response: {feeds_response_json}")
48
+ if feeds_response_json.get("count") > 1:
49
+ logging.warning("Found more than 1 feeds. Use the first one.")
50
+ elif feeds_response_json.get("count") == 0:
51
+ logging.warning("No feeds were found.")
52
+ return []
53
+ feed = feeds_response_json.get("value")[0]
54
+ feed_links = feed.get("_links", {})
55
+
56
+ # Get feed versions
57
+ feed_versions_url = feed_links.get("versions", {}).get("href", "")
58
+ feed_versions_response = self._session.get(url=feed_versions_url, timeout=self.timeout)
59
+ feed_versions_response_json = feed_versions_response.json()
60
+ if feed_versions_response.status_code != 200:
61
+ logging.error(f"Feed versions error ({feed_versions_response.status_code}) response: {feed_versions_response_json}")
62
+ raise Exception(f"Could not find feed versions, search request returned {feed_versions_response.status_code}!")
63
+ logging.debug(f"Feed versions response: {feed_versions_response_json}")
64
+ feed_versions = feed_versions_response_json.get("value")
65
+
66
+ # Filter by acceptable versions
67
+ logging.debug(f"Filtering by acceptable versions: '{acceptable_versions}'")
68
+ feed_version = [f for f in feed_versions if (f.get('protocolMetadata').get('data').get('version') in acceptable_versions)]
69
+ if len(feed_version) == 0:
70
+ logging.warning("All feed versions filtered.")
71
+ return []
72
+ filtered_feed_version = feed_version[0]
73
+
74
+ # Search for target file
75
+ files = [f for f in filtered_feed_version.get("files") if f.get('name').startswith(f"{artifact.artifact_id}-{artifact.version}") and f.get('name').endswith(artifact.extension)]
76
+ logging.debug(f"Files found: {files}")
77
+ if len(files) == 0:
78
+ logging.warning("All files filtered.")
79
+ return []
80
+ target_file = files[0]
81
+
82
+ # Build download url
83
+ feed_id = feed_links.get("feed").get("href").split("/")[-1] # take id from link to feed
84
+ feed_version = filtered_feed_version.get("version")
85
+ group_id = filtered_feed_version.get('protocolMetadata').get('data').get("groupId")
86
+ artifact_id = filtered_feed_version.get('protocolMetadata').get('data').get("artifactId")
87
+ target_file_name = target_file.get("name")
88
+
89
+ download_url = (
90
+ f"https://pkgs.dev.azure.com/{self.organization}/{self.project}/_apis/packaging/feeds/{feed_id}/maven/"
91
+ f"{group_id}/{artifact_id}/{feed_version}/{target_file_name}/content"
92
+ f"?api-version=7.1-preview.1"
93
+ )
94
+ logging.info(f"Azure search resulting url: {download_url}")
95
+ return [download_url]
96
+
97
+ def get_provider_name(self) -> str:
98
+ return "azure_artifacts"
@@ -0,0 +1,50 @@
1
+ from pathlib import Path
2
+ from google.cloud import artifactregistry_v1
3
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
4
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
5
+ from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
6
+
7
+
8
+ class GcpArtifactRegistryProvider(ArtifactProvider):
9
+
10
+ def __init__(self, credentials: Credentials, project: str, region_name: str, repository: str, **kwargs):
11
+ """
12
+ Initializes this client to work with **GCP Artifact Registry** for generic artifacts.
13
+ Requires `Credentials` provided by `GcpCredentialsProvider`.
14
+ """
15
+ super().__init__(**kwargs)
16
+ self._credentials = credentials
17
+ self._project = project
18
+ self._region_name = region_name
19
+ self._repository = repository
20
+ self._repo_resource_id = f"projects/{project}/locations/{region_name}/repositories/{repository}"
21
+
22
+ self._gcp_client = artifactregistry_v1.ArtifactRegistryClient(
23
+ credentials=self._credentials.google_credentials_object
24
+ )
25
+ self._authorized_session = self._credentials.authorized_session
26
+
27
+ def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
28
+ response = self._authorized_session.get(url=resource_url, timeout=self.timeout)
29
+ response.raise_for_status()
30
+ with open(local_path, 'wb') as file:
31
+ file.write(response.content)
32
+
33
+ def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
34
+ # works with both "Maven" and "Generic" type repositories
35
+ name_filter = f"{self._repo_resource_id}/files/*{artifact.artifact_id}-{artifact.version}.{artifact.extension}"
36
+ list_files_request = artifactregistry_v1.ListFilesRequest(
37
+ parent=f"{self._repo_resource_id}",
38
+ filter=f'name="{name_filter}"',
39
+ )
40
+ files = self._gcp_client.list_files(request=list_files_request)
41
+ # logging.debug(f"[GCP search_artifacts] files: {files}")
42
+
43
+ urls = []
44
+ for file in files:
45
+ download_url = f"https://artifactregistry.googleapis.com/download/v1/{file.name}:download?alt=media"
46
+ urls.append(download_url)
47
+ return urls
48
+
49
+ def get_provider_name(self) -> str:
50
+ return "gcp_artifact_registry"