qubership-pipelines-common-library 0.2.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

qubership_pipelines_common_library/v1/maven_client.py

@@ -22,7 +22,7 @@ class Artifact:
     def from_string(artifact_str: str):
         parts = artifact_str.split(":")
         if len(parts) == 3:
-            group, artifact, version = parts[0], parts[1], parts[-1]
+            artifact, version = parts[1], parts[-1]
             return Artifact(artifact, version)
 
 
@@ -67,7 +67,7 @@ class MavenArtifactSearcher:
         if not artifact:
             artifact = Artifact(artifact_id=artifact_id, version=version, extension=extension)
         if not artifact.artifact_id or not artifact.version:
-            raise Exception(f"Artifact 'artifact_id' and 'version' must be specified!")
+            raise Exception("Artifact 'artifact_id' and 'version' must be specified!")
         logging.debug(f"Searching for '{artifact.artifact_id}' in {self.registry_url}...")
         return self._search_func(artifact=artifact)
 

qubership_pipelines_common_library/v1/minio_client.py

@@ -68,7 +68,7 @@ class MinioClient:
 
     def put_file(self, bucket_name: str, path: str, local_path: str):
         """"""
-        result = self.minio.fput_object(bucket_name, path, local_path)
+        self.minio.fput_object(bucket_name, path, local_path)
 
     def get_text_file_content(self, bucket_name: str, file_path: str):
         """"""

qubership_pipelines_common_library/v1/utils/rest.py

@@ -77,4 +77,4 @@ class RestClient:
                 return False
         except requests.RequestException as e:
             logging.error(f"Error: {e}")
-            return False
+            return False

qubership_pipelines_common_library/v1/utils/utils.py

@@ -47,4 +47,4 @@ def recursive_merge(source: dict, target: dict):
          source[key] = recursive_merge(source[key], value)
       else:
          source[key] = value
-   return source
+   return source

qubership_pipelines_common_library/v1/utils/utils_cli.py

@@ -1,14 +1,21 @@
+import logging
 import re
+
 import click
-import logging
-import sys, os
+from rich import box
+from rich.logging import RichHandler
+from rich.panel import Panel
+
 from qubership_pipelines_common_library.v1.execution.exec_logger import ExecutionLogger
+from qubership_pipelines_common_library.v1.utils.utils_logging import rich_console, ExtendedReprHighlighter, \
+    LevelColorFilter
 
 DEFAULT_CONTEXT_FILE_PATH = 'context.yaml'
 
 
 def utils_cli(func):
     """Decorator to add CLI options for logging level, context path and custom input params."""
+
     @click.option('--log-level', default='INFO', show_default=True,
                   type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False),
                   help="Set the logging level")
@@ -20,23 +27,50 @@ def utils_cli(func):
     @click.pass_context
     def wrapper(ctx, *args, log_level, **kwargs):
         ExecutionLogger.EXECUTION_LOG_LEVEL = getattr(logging, log_level.upper(), logging.INFO)
-        _configure_global_logger(logging.getLogger(), log_level, ExecutionLogger.DEFAULT_FORMAT)
+        _configure_global_logger(logging.getLogger(), log_level)
+        _print_command_name()
         _transform_kwargs(kwargs)
         return ctx.invoke(func, *args, **kwargs)
+
     return wrapper
 
 
-def _configure_global_logger(global_logger: logging.Logger, log_level: str, formatter_str: str):
+def _configure_global_logger(global_logger: logging.Logger, log_level: str):
     """Configure the global logger with a specific log level and formatter."""
-    log_level_value = getattr(logging, log_level.upper(), logging.INFO)
     global_logger.setLevel(logging.DEBUG)
     if global_logger.hasHandlers():
         global_logger.handlers.clear()
     global_logger.propagate = True
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setLevel(log_level_value)
-    stdout_handler.setFormatter(logging.Formatter(formatter_str))
-    global_logger.addHandler(stdout_handler)
+    rich_handler = RichHandler(
+        console=rich_console,
+        show_time=False,
+        show_level=False,
+        show_path=False,
+        enable_link_path=False,
+        rich_tracebacks=True,
+        tracebacks_show_locals=False,
+        markup=True,
+        highlighter=ExtendedReprHighlighter(),
+    )
+    rich_handler.addFilter(LevelColorFilter())
+    rich_handler.setFormatter(logging.Formatter(ExecutionLogger.LEVELNAME_COLORED_FORMAT))
+    log_level_value = getattr(logging, log_level.upper(), logging.INFO)
+    rich_handler.setLevel(log_level_value)
+    global_logger.addHandler(rich_handler)
+
+
+def _print_command_name():
+    try:
+        click_context = click.get_current_context()
+        command_name = click_context.command.name or click_context.info_name
+    except RuntimeError:
+        logging.getLogger().warning("Can't find command name.")
+        command_name = ""
+
+    command_panel = Panel(f"command_name = {command_name}", expand=False, padding=(0, 1), box=box.ROUNDED)
+    rich_console.print()
+    rich_console.print(command_panel)
+    rich_console.print()
 
 
 def _transform_kwargs(kwargs):

qubership_pipelines_common_library/v1/utils/utils_dictionary.py

@@ -39,4 +39,4 @@ class UtilsDictionary:
             elif key not in curr:
                 curr[key] = {}
             curr = curr[key]
-        return input_dict
+        return input_dict

qubership_pipelines_common_library/v1/utils/utils_file.py

@@ -66,3 +66,20 @@ class UtilsFile:
             func(path)
         else:
             raise
+
+    @staticmethod
+    def create_parent_dirs(filepath):
+        if directory := os.path.dirname(filepath):
+            os.makedirs(directory, exist_ok=True)
+
+    @staticmethod
+    def create_exec_dir(execution_folder_path: str | Path, exists_ok: bool = False) -> Path:
+        import shutil
+        exec_dir = Path(execution_folder_path)
+        if exec_dir.exists() and not exists_ok:
+            if exec_dir.is_dir():
+                shutil.rmtree(exec_dir)
+            else:
+                raise FileExistsError(f"Path '{execution_folder_path}' exists and is a file, not a directory.")
+        exec_dir.mkdir(parents=True, exist_ok=exists_ok)
+        return exec_dir

qubership_pipelines_common_library/v1/utils/utils_logging.py

@@ -0,0 +1,53 @@
+import logging
+
+from rich.console import Console
+from rich.highlighter import ReprHighlighter
+from rich.theme import Theme
+
+soft_theme = Theme({
+    "repr.number": "rgb(180,200,255)",
+    "repr.bool_true": "rgb(140,230,140)",
+    "repr.bool_false": "rgb(230,140,140)",
+    "repr.none": "rgb(200,200,200) italic",
+    "repr.path": "rgb(190,220,160)",
+    "repr.filename": "rgb(160,210,190)",
+    "repr.url": "rgb(130,180,255) underline",
+    "repr.uuid": "rgb(200,180,220)",
+    "repr.attrib_name": "rgb(220,200,130)",
+    "repr.attrib_value": "rgb(170,190,220)",
+    "repr.str": "rgb(140,180,140)",
+    "repr.tag_name": "rgb(200,170,220)",
+    "repr.tag_value": "rgb(170,200,220)",
+
+    "repr.time": "rgb(160,190,220) italic",
+})
+
+rich_console = Console(
+    theme=soft_theme,
+    force_terminal=True,
+    no_color=False,
+    highlight=True,
+    width=150,
+)
+
+level_colors = {
+    logging.DEBUG: "steel_blue",
+    logging.INFO: "light_sea_green",
+    logging.WARNING: "orange3",
+    logging.ERROR: "indian_red",
+    logging.CRITICAL: "bold medium_violet_red",
+}
+
+
+class LevelColorFilter(logging.Filter):
+    def filter(self, record):
+        color = level_colors.get(record.levelno, "default")
+        record.levelname_color_open_tag = f"[{color}]"
+        record.levelname_color_close_tag = "[/]"
+        return True
+
+
+class ExtendedReprHighlighter(ReprHighlighter):
+    highlights = ReprHighlighter.highlights + [
+        r"(?P<time>\b([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:[.,]\d{1,9})?\b)",
+    ]

qubership_pipelines_common_library/v2/artifacts_finder/artifact_finder.py

@@ -0,0 +1,56 @@
+import logging
+from pathlib import Path
+
+from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
+from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact_provider import ArtifactProvider
+
+
+class ArtifactFinder:
+    """
+    Allows searching for specific descriptor artifacts in different repositories without knowing full coordinates
+    (e.g. knowing only `artifact_id` and `version`, but not its `group_id`)
+
+    Supports different repository providers: Artifactory, Nexus, AWS, GCP, Azure
+
+    Provides different auth methods for Cloud Providers, implementing `CloudCredentialsProvider` interface
+
+    Start by initializing this client with one of implementations:
+    ``finder = ArtifactFinder(artifact_provider=ArtifactoryProvider(registry_url="https://our_url", username="user", password="password"))``
+
+    Then find your artifacts using
+    ``resource_urls = finder.find_artifact_urls(artifact_id='art_id', version='1.0.0', extension='json')``
+
+    Additionally, perform filtering of returned results (if you expect to find more than one artifact), and then download necessary artifacts with
+    ``finder.download_artifact(one_of_the_returned_resource_urls, './my_artifact.json')``
+
+    For more complex providers (e.g. AWS Code Artifact), you need to use specific Credential Providers
+    As an example:
+    ```
+    aws_creds = AwsCredentialsProvider().with_assume_role(...all the required params...).get_credentials()
+    aws_code_artifact_provider = AwsCodeArtifactProvider(creds=creds, domain='our_domain', project='our_project')
+    finder = ArtifactFinder(artifact_provider=aws_code_artifact_provider)
+    ```
+    """
+
+    def __init__(self, artifact_provider: ArtifactProvider, **kwargs):
+        if not artifact_provider:
+            raise Exception("Initialize ArtifactFinder with one of registry artifact providers first!")
+        self.provider = artifact_provider
+
+    def find_artifact_urls(self, artifact_id: str = None, version: str = None, group_id: str = None,
+                           extension: str = "jar", artifact: Artifact = None) -> list[str]:
+        if not artifact:
+            artifact = Artifact(group_id=group_id, artifact_id=artifact_id, version=version, extension=extension)
+        if not artifact.artifact_id or not artifact.version:
+            raise Exception("Artifact 'artifact_id' and 'version' must be specified!")
+        logging.debug(f"Searching for '{artifact.artifact_id}:{artifact.version}' in {self.provider.get_provider_name()}...")
+        return self.provider.search_artifacts(artifact=artifact)
+
+    def download_artifact(self, resource_url: str, local_path: str | Path, artifact: Artifact = None):
+        from qubership_pipelines_common_library.v1.utils.utils_file import UtilsFile
+        download_path = Path(local_path)
+        if artifact:
+            download_path = download_path.joinpath(artifact.get_filename())
+        UtilsFile.create_parent_dirs(download_path)
+        logging.debug(f"Downloading artifact from '{resource_url}' to '{download_path}'...")
+        return self.provider.download_artifact(resource_url=resource_url, local_path=download_path)

qubership_pipelines_common_library/v2/artifacts_finder/auth/aws_credentials.py

@@ -0,0 +1,106 @@
+import boto3
+
+from enum import StrEnum
+from botocore.config import Config
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
+
+
+class AwsCredentialsProvider(CloudCredentialsProvider):
+
+    DEFAULT_ROLE_SESSION_NAME = "codeartifact-session"
+
+    access_key: str
+    secret_key: str
+    session_token: str
+    region_name: str
+    role_arn: str
+    role_session_name: str
+    _auth_type = None
+
+    class AuthType(StrEnum):
+        DIRECT = 'DIRECT'
+        ASSUME_ROLE = 'ASSUME_ROLE'
+
+    def with_direct_credentials(self,
+                                access_key: str,
+                                secret_key: str,
+                                region_name: str,
+                                session_token: str | None = None,
+                                ):
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.session_token = session_token
+        self.region_name = region_name
+        self.validate_mandatory_attrs(["access_key", "secret_key", "region_name"])
+        self._auth_type = self.AuthType.DIRECT
+        return self
+
+    def with_assume_role(self,
+                         access_key: str,
+                         secret_key: str,
+                         region_name: str,
+                         role_arn: str,
+                         role_session_name: str | None = DEFAULT_ROLE_SESSION_NAME,
+                         ):
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.region_name = region_name
+        self.role_arn = role_arn
+        self.role_session_name = role_session_name
+        self.validate_mandatory_attrs(["access_key", "secret_key", "region_name", "role_arn", "role_session_name"])
+        self._auth_type = self.AuthType.ASSUME_ROLE
+        return self
+
+    def get_credentials(self) -> Credentials:
+        if self._auth_type is self.AuthType.DIRECT:
+            return self._get_direct_credentials()
+        elif self._auth_type is self.AuthType.ASSUME_ROLE:
+            return self._get_assume_role_credentials()
+        else:
+            raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
+
+    def get_ecr_authorization_token(self) -> str:
+        creds = self.get_credentials()
+        ecr_client = boto3.client(
+            service_name="ecr",
+            config=Config(region_name=self.region_name),
+            aws_access_key_id=creds["access_key"],
+            aws_secret_access_key=creds["secret_key"],
+            aws_session_token=creds["session_token"],
+        )
+        ecr_authorization_token = ecr_client.get_authorization_token()
+        ecr_authorization_data = ecr_authorization_token["authorizationData"][0]
+        return ecr_authorization_data["authorizationToken"]
+
+    def _get_direct_credentials(self) -> Credentials:
+        return Credentials(
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            session_token=self.session_token,
+            region_name=self.region_name,
+        )
+
+    def _get_assume_role_credentials(self) -> Credentials:
+        creds = self._assume_role()
+        return Credentials(
+            access_key=creds["AccessKeyId"],
+            secret_key=creds["SecretAccessKey"],
+            session_token=creds["SessionToken"],
+            role_arn=self.role_arn,
+            role_session_name=self.role_session_name,
+            region_name=self.region_name,
+        )
+
+    def _assume_role(self) -> dict:
+        sts_client = boto3.client(
+            service_name="sts",
+            config=Config(region_name=self.region_name),
+            aws_access_key_id=self.access_key,
+            aws_secret_access_key=self.secret_key,
+        )
+        assumed = sts_client.assume_role(
+            RoleArn=self.role_arn,
+            RoleSessionName=self.role_session_name,
+        )
+        return assumed["Credentials"]

qubership_pipelines_common_library/v2/artifacts_finder/auth/azure_credentials.py

@@ -0,0 +1,72 @@
+import requests
+
+from enum import StrEnum
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
+
+
+class AzureCredentialsProvider(CloudCredentialsProvider):
+
+    tenant_id: str
+    client_id: str
+    client_secret: str
+    target_resource: str
+    _auth_data: dict
+    _auth_type = None
+
+    class AuthType(StrEnum):
+        OAUTH2 = 'OAUTH2'
+
+    def with_oauth2(self,
+                    tenant_id: str,
+                    client_id: str,
+                    client_secret: str,
+                    target_resource: str,
+                    ):
+        self.tenant_id = tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.target_resource = target_resource
+        self.validate_mandatory_attrs(["tenant_id", "client_id", "client_secret", "target_resource"])
+        self._auth_data = {
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "scope": f"{target_resource}/.default"
+        }
+        self._auth_type = self.AuthType.OAUTH2
+        return self
+
+    def with_oauth2_custom_data(self,
+                                tenant_id: str,
+                                custom_auth_data: dict,
+                                ):
+        self.tenant_id = tenant_id
+        self._auth_data = custom_auth_data
+        self.validate_mandatory_attrs(["tenant_id", "_auth_data"])
+        self._auth_type = self.AuthType.OAUTH2
+        return self
+
+    def get_credentials(self) -> Credentials:
+        if self._auth_type is self.AuthType.OAUTH2:
+            return self._get_oauth2_credentials()
+        else:
+            raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
+
+    def _get_oauth2_credentials(self) -> Credentials:
+        token_url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token"
+        response = requests.post(
+            token_url,
+            data=self._auth_data,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+        )
+        response.raise_for_status()
+        response_json = response.json()
+        access_token = response_json.get("access_token")
+        if not access_token:
+            raise Exception(f"Failed to get access token from {token_url}: {response_json}")
+
+        return Credentials(
+            access_token=access_token,
+            tenant_id=self.tenant_id,
+        )

qubership_pipelines_common_library/v2/artifacts_finder/auth/gcp_credentials.py

@@ -0,0 +1,88 @@
+import json
+from enum import StrEnum
+from pathlib import Path
+
+from google.auth.transport.requests import AuthorizedSession, Request
+
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials_provider import CloudCredentialsProvider
+
+
+class GcpCredentialsProvider(CloudCredentialsProvider):
+
+    DEFAULT_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
+    DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"
+
+    service_account_key_content: str
+    oidc_credential_source: dict
+    audience: str
+    scopes: list[str]
+    subject_token_type: str
+    _auth_type = None
+
+    class AuthType(StrEnum):
+        SA_KEY = 'SA_KEY'
+        OIDC_CREDS = 'OIDC_CREDS'
+
+    def with_service_account_key(self,
+                                 service_account_key_content: str | None = None,
+                                 service_account_key_path: str | Path | None = None,
+                                 scopes: list[str] = None,
+                                 ):
+        if scopes is None:
+            scopes = self.DEFAULT_SCOPES
+        self.scopes = scopes
+        self.service_account_key_content = service_account_key_content
+        if service_account_key_path:
+            with open(service_account_key_path, 'r') as key_file:
+                self.service_account_key_content = key_file.read()
+        self.validate_mandatory_attrs(["service_account_key_content"])
+        self._auth_type = self.AuthType.SA_KEY
+        return self
+
+    def with_oidc_creds(self,
+                        oidc_credential_source: dict,
+                        audience: str,
+                        oidc_token_file_path: str = None,
+                        subject_token_type: str = DEFAULT_SUBJECT_TOKEN_TYPE,
+                        scopes: list[str] = None,
+                        ):
+        if scopes is None:
+            scopes = self.DEFAULT_SCOPES
+        self.scopes = scopes
+        self.oidc_credential_source = oidc_credential_source
+        if oidc_token_file_path:
+            self.oidc_credential_source = {"file": oidc_token_file_path}
+        self.audience = audience
+        self.subject_token_type = subject_token_type
+        self.validate_mandatory_attrs(["oidc_credential_source", "audience"])
+        self._auth_type = self.AuthType.OIDC_CREDS
+        return self
+
+    def get_credentials(self) -> Credentials:
+        if self._auth_type == self.AuthType.SA_KEY:
+            from google.oauth2 import service_account
+            google_creds = service_account.Credentials.from_service_account_info(
+                info=json.loads(self.service_account_key_content),
+                scopes=self.scopes,
+            )
+        elif self._auth_type == self.AuthType.OIDC_CREDS:
+            from google.auth import identity_pool
+            google_creds = identity_pool.Credentials(
+                audience=self.audience,
+                subject_token_type=self.subject_token_type,
+                credential_source=self.oidc_credential_source,
+                scopes=self.scopes,
+            )
+        else:
+            raise ValueError("Need to initialize this provider with AuthType via .with_*auth_type* method first!")
+
+        google_creds.refresh(Request())
+        creds = Credentials(
+            google_credentials_object=google_creds,
+            gcp_authorization_token=google_creds.token, # It can be used in Basic Authorization (in some Execution Commands)
+            authorized_session=AuthorizedSession(credentials=google_creds),
+        )
+        if self._auth_type == self.AuthType.SA_KEY:
+            creds.service_account_key_content = self.service_account_key_content
+        return creds

qubership_pipelines_common_library/v2/artifacts_finder/model/artifact.py

@@ -0,0 +1,20 @@
+class Artifact:
+    def __init__(self, group_id = None, artifact_id = None, version = None, extension='jar'):
+        self.group_id = group_id
+        self.artifact_id = artifact_id
+        self.version = version
+        self.extension = "jar" if not extension else extension
+
+    def is_snapshot(self):
+        return self.version and self.version.endswith("SNAPSHOT")
+
+    def get_filename(self) -> str:
+        return f"{self.artifact_id}-{self.version}.{self.extension}"
+
+    @staticmethod
+    def from_string(artifact_str: str):
+        parts = artifact_str.split(":")
+        if len(parts) == 3:
+            group, artifact, version = parts[0], parts[1], parts[-1]
+            return Artifact(group, artifact, version)
+        raise Exception("Invalid artifact string")

qubership_pipelines_common_library/v2/artifacts_finder/model/artifact_provider.py

@@ -0,0 +1,35 @@
+import requests
+
+from pathlib import Path
+from abc import ABC, abstractmethod
+from qubership_pipelines_common_library.v2.artifacts_finder.model.artifact import Artifact
+
+
+class ArtifactProvider(ABC):
+    """Base class for all artifact providers"""
+
+    TIMESTAMP_VERSION_PATTERN = "^(.*-)?([0-9]{8}\\.[0-9]{6}-[0-9]+)$"
+
+    def __init__(self, params: dict = None, **kwargs):
+        self.params = params if params else {}
+        self._session = requests.Session()
+        self._session.verify = self.params.get('verify', True)
+        self.timeout = self.params.get('timeout', None)
+
+    def generic_download(self, resource_url: str, local_path: str | Path):
+        response = self._session.get(url=resource_url, timeout=self.timeout)
+        response.raise_for_status()
+        with open(local_path, 'wb') as file:
+            file.write(response.content)
+
+    @abstractmethod
+    def download_artifact(self, resource_url: str, local_path: str | Path, **kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def search_artifacts(self, artifact: Artifact, **kwargs) -> list[str]:
+        pass
+
+    @abstractmethod
+    def get_provider_name(self) -> str:
+        pass

qubership_pipelines_common_library/v2/artifacts_finder/model/credentials.py

@@ -0,0 +1,16 @@
+class Credentials(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self  # allow attribute access
+
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError:
+            return None
+
+    def __setattr__(self, key, value):
+        self[key] = value

qubership_pipelines_common_library/v2/artifacts_finder/model/credentials_provider.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+from qubership_pipelines_common_library.v2.artifacts_finder.model.credentials import Credentials
+
+
+class CloudCredentialsProvider(ABC):
+    """Base class for all cloud credentials"""
+
+    @abstractmethod
+    def get_credentials(self) -> Credentials:
+        pass
+
+    def validate_mandatory_attrs(self, attrs_names) -> None:
+        missing_attrs = [attr for attr in attrs_names if getattr(self, attr, None) is None]
+        if missing_attrs:
+            raise ValueError(f"The following mandatory attributes are not set: {', '.join(missing_attrs)}")