qubership-pipelines-common-library 0.1.7__tar.gz → 0.1.9__tar.gz

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qubership-pipelines-common-library
-Version: 0.1.7
+Version: 0.1.9
 Summary: Qubership Pipelines common library
 License: Apache-2.0
 Author: Igor Lebedev
@@ -12,9 +12,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: GitPython (>=3.1.43,<4.0.0)
-Requires-Dist: PyArtifactory (>=2.7.1,<3.0.0)
-Requires-Dist: PyGithub (>=2.6.1,<3.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: ghapi (>=1.0.6,<2.0.0)
 Requires-Dist: http-exceptions (>=0.2.10,<0.3.0)
 Requires-Dist: kubernetes (>=29.0.0,<30.0.0)
 Requires-Dist: minio (>=7.2.12,<8.0.0)
@@ -64,6 +63,7 @@ pip install qubership-pipelines-common-library-py39
 
 ## Sample implementation
 
-Sample implementation of CLI commands using this library is available at [Quber CLI](https://github.com/LightlessOne/Quber-CLI?tab=readme-ov-file)
+Sample implementation of CLI commands using this library is available at [qubership-pipelines-cli-command-samples](https://github.com/Netcracker/qubership-pipelines-cli-command-samples)
+
+It includes reference python implementation along with the [Development Guide](https://github.com/Netcracker/qubership-pipelines-cli-command-samples/blob/main/docs/development.md)
 
-It includes reference python implementation along with [Development Guide](https://github.com/LightlessOne/Quber-CLI/blob/master/docs/development.md)

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/README.md

@@ -36,6 +36,6 @@ pip install qubership-pipelines-common-library-py39
 
 ## Sample implementation
 
-Sample implementation of CLI commands using this library is available at [Quber CLI](https://github.com/LightlessOne/Quber-CLI?tab=readme-ov-file)
+Sample implementation of CLI commands using this library is available at [qubership-pipelines-cli-command-samples](https://github.com/Netcracker/qubership-pipelines-cli-command-samples)
 
-It includes reference python implementation along with [Development Guide](https://github.com/LightlessOne/Quber-CLI/blob/master/docs/development.md)
+It includes reference python implementation along with the [Development Guide](https://github.com/Netcracker/qubership-pipelines-cli-command-samples/blob/main/docs/development.md)

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qubership-pipelines-common-library"
-version = "0.1.7"
+version = "0.1.9"
 description = "Qubership Pipelines common library"
 authors = ["Igor Lebedev <lebedev.light@gmail.com>"]
 readme = "README.md"
@@ -13,7 +13,6 @@ pyyaml = "^6.0.2"
 click = "^8.1.7"
 requests = "^2.32.3"
 urllib3 = "^2.2.3"
-PyArtifactory = "^2.7.1"
 python-jenkins = "^1.8.2"
 GitPython = "^3.1.43"
 http-exceptions = "^0.2.10"
@@ -21,7 +20,7 @@ python-gitlab = "^4.13.0"
 minio = "^7.2.12"
 kubernetes = "^29.0.0"
 webexpythonsdk = "2.0.1"
-PyGithub = "^2.6.1"
+ghapi = "^1.0.6"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^6.0.0"
@@ -33,4 +32,4 @@ markers = [
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"

qubership_pipelines_common_library-0.1.9/qubership_pipelines_common_library/v1/artifactory_client.py

@@ -0,0 +1,119 @@
+# Copyright 2024 NetCracker Technology Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from dataclasses import dataclass
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+class ArtifactoryClient:
+    def __init__(self, params: dict):
+        """
+        **`params`** is a dictionary with following mandatory params:
+
+        Arguments:
+            url (str): Artifactory host url
+            username (str): User used in auth request
+            password (str): Token used in auth request
+        """
+        self.url = params.get("url")
+        self.user = params.get("username")
+        self.token = params.get("password")
+        self.artifactory = ArtifactoryAPI(self.url, HTTPBasicAuth(self.user, self.token))
+        logging.info("Artifactory Client configured for %s", params.get("url"))
+
+    def get_artifact_properties(self, path_to_artifact: str):
+        """"""
+        try:
+            properties = self.artifactory.get_artifact_properties(artifact_path=path_to_artifact)
+        except ArtifactoryError:
+            logging.error("There are not properties for artifact %s", path_to_artifact)
+            properties = None
+        return properties
+
+    def get_folder_files_list(self, path_to_folder: str):
+        """"""
+        return self.artifactory.get_files_list(artifact_path=path_to_folder)
+
+    def get_artifact_content_by_url(self, path_to_file: str):
+        """"""
+        return self.artifactory.get_file_content(artifact_path=path_to_file)
+
+
+class ArtifactoryAPI:
+    def __init__(self, api_url: str, auth, verify=False):
+        self.api_url = api_url.rstrip('/')
+        self._session = requests.session()
+        self._session.verify = False
+        if auth:
+            self._session.auth = auth
+
+    def _get(self, url):
+        response = self._session.get(url)
+        response.raise_for_status()
+        return response
+
+    def get_artifact_info(self, artifact_path: str):
+        try:
+            response = self._get(f"{self.api_url}/api/storage/{artifact_path}").json()
+            artifact_info = ArtifactInfo(repo=response['repo'], path=response['path'])
+            return artifact_info
+        except requests.exceptions.HTTPError as error:
+            raise ArtifactoryError from error
+
+    def get_artifact_properties(self, artifact_path: str):
+        try:
+            response = self._get(f"{self.api_url}/api/storage/{artifact_path}?properties").json()
+            return ArtifactProperties(properties=response['properties'])
+        except requests.exceptions.HTTPError as error:
+            raise ArtifactoryError from error
+
+    def get_files_list(self, artifact_path: str):
+        try:
+            response = self._get(f"{self.api_url}/api/storage/{artifact_path}?list&deep=1&listFolders=1").json()
+            return [ArtifactListEntry(uri=f['uri'], size=int(f['size']), folder=f['folder'] == True) for f in
+                    response['files']]
+        except requests.exceptions.HTTPError as error:
+            raise ArtifactoryError from error
+
+    def get_file_content(self, artifact_path: str):
+        try:
+            info = self.get_artifact_info(artifact_path)
+            return self._get(f"{self.api_url}/{info.repo}{info.path}").content.decode("utf-8")
+        except requests.exceptions.HTTPError as error:
+            raise ArtifactoryError from error
+
+
+class ArtifactoryError(Exception):
+    pass
+
+
+@dataclass
+class ArtifactInfo:
+    repo: str
+    path: str
+
+
+@dataclass
+class ArtifactProperties:
+    properties: dict
+
+
+@dataclass
+class ArtifactListEntry:
+    uri: str
+    size: int
+    folder: bool

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/qubership_pipelines_common_library/v1/execution/exec_command.py

@@ -25,11 +25,11 @@ class ExecutionCommand:
     SUCCESS_MSG = "Status: SUCCESS"
     FAILURE_MSG = "Status: FAILURE"
 
-    def __init__(self, context_path: str = None, input_params: dict = None, folder_path: str = None,
-                 parent_context_to_reuse: ExecutionContext = None):
+    def __init__(self, context_path: str = None, input_params: dict = None, input_params_secure: dict = None,
+                 folder_path: str = None, parent_context_to_reuse: ExecutionContext = None):
         if not context_path:
-            context_path = create_execution_context(input_params=input_params, folder_path=folder_path,
-                                                    parent_context_to_reuse=parent_context_to_reuse)
+            context_path = create_execution_context(input_params=input_params, input_params_secure=input_params_secure,
+                                                    folder_path=folder_path, parent_context_to_reuse=parent_context_to_reuse)
         self.context = ExecutionContext(context_path)
 
     def run(self):

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/qubership_pipelines_common_library/v1/github_client.py

@@ -19,13 +19,10 @@ import uuid
 import zipfile
 import requests
 
+from ghapi.all import GhApi
 from datetime import datetime, timezone
 from pathlib import Path
-from github import Github, Auth
 from time import sleep
-from github.Artifact import Artifact
-from github.PaginatedList import PaginatedList
-from github.WorkflowRun import WorkflowRun
 
 from qubership_pipelines_common_library.v1.execution.exec_info import ExecutionInfo
 
@@ -59,11 +56,7 @@ class GithubClient:
             api_url (str): Optional Github Enterprise API URL, leave empty if using github.com
             **kwargs (Any): will be passed into Github API constructor
         """
-        self.auth = Auth.Token(token) if token else None
-        if api_url:
-            self.gh = Github(base_url=api_url, auth=self.auth, **kwargs)
-        else:
-            self.gh = Github(auth=self.auth, **kwargs)
+        self.gh = GhApi(token=token, gh_host=api_url, **kwargs)
         logging.info("Github Client configured")
 
     def trigger_workflow(self, owner: str, repo_name: str, workflow_file_name: str, branch: str, pipeline_params: dict,
@@ -84,24 +77,29 @@ class GithubClient:
             pipeline_params[uuid_param_name] = str(uuid.uuid4())
         if len(pipeline_params) > GithubClient.DISPATCH_PARAMS_LIMIT:
             logging.warning(f"Trying to dispatch workflow with more than {GithubClient.DISPATCH_PARAMS_LIMIT} pipeline_params, GitHub does not support it!")
-        workflow = self.gh.get_repo(f"{owner}/{repo_name}", lazy=True).get_workflow(workflow_file_name)
         dispatch_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
         execution = ExecutionInfo()
-        is_created = workflow.create_dispatch(ref=branch, inputs=pipeline_params)
+        try:
+            self.gh.actions.create_workflow_dispatch(owner, repo_name, workflow_file_name, branch, pipeline_params)
+            is_created = True
+        except Exception as ex:
+            logging.error(f"Exception when triggering workflow: {ex}")
+            is_created = False
         logging.info(f"Workflow Dispatch event for {workflow_file_name} is sent, workflow is created: {is_created}")
         if is_created:
             current_timeout = 0
             already_checked_runs = []
             while current_timeout < timeout_seconds:
-                runs_list = workflow.get_runs(event="workflow_dispatch", created=f">={dispatch_time}", branch=branch)
-                if runs_list.totalCount > 0:
+                runs_list = self.gh.actions.list_workflow_runs(owner, repo_name, workflow_file_name, event="workflow_dispatch", created=f">={dispatch_time}", branch=branch)
+                if runs_list.total_count > 0:
                     if find_via_uuid:
-                        created_run = self._find_run_via_uuid_input_param(runs_list, already_checked_runs,
+                        created_run = self._find_run_via_uuid_input_param(owner, repo_name, runs_list.workflow_runs,
+                                                                          already_checked_runs,
                                                                           uuid_artifact_name, uuid_file_name,
                                                                           uuid_param_name,
                                                                           pipeline_params[uuid_param_name])
                     else:
-                        created_run = runs_list.get_page(0).pop()
+                        created_run = runs_list.workflow_runs[0]
                     if created_run:
                         logging.info(f"Pipeline successfully started at {created_run.html_url}")
                         return execution.with_name(created_run.name).with_id(created_run.id) \
@@ -117,10 +115,10 @@ class GithubClient:
 
     def get_workflow_run_status(self, execution: ExecutionInfo):
         """"""
-        repo_full_name = self._get_repo_full_name(execution)
-        if not repo_full_name:
+        owner_and_repo_name = self._get_owner_and_repo(execution)
+        if not owner_and_repo_name:
             return execution.with_status(ExecutionInfo.STATUS_UNKNOWN)
-        run = self.gh.get_repo(repo_full_name).get_workflow_run(int(execution.get_id()))
+        run = self.gh.actions.get_workflow_run(owner_and_repo_name[0], owner_and_repo_name[1], execution.get_id())
         if run:
             execution.with_status(self._map_status_and_conclusion(run.status, run.conclusion, ExecutionInfo.STATUS_UNKNOWN))
         else:
@@ -133,13 +131,13 @@ class GithubClient:
         """"""
         if break_status_list is None:
             break_status_list = self.BREAK_STATUS_LIST
-        repo_full_name = self._get_repo_full_name(execution)
-        if not repo_full_name:
+        owner_and_repo_name = self._get_owner_and_repo(execution)
+        if not owner_and_repo_name:
             return execution.with_status(ExecutionInfo.STATUS_UNKNOWN)
         timeout = 0
         while timeout < timeout_seconds:
             try:
-                run = self.gh.get_repo(repo_full_name).get_workflow_run(int(execution.get_id()))
+                run = self.gh.actions.get_workflow_run(owner_and_repo_name[0], owner_and_repo_name[1], execution.get_id())
                 execution.with_status(self._map_status_and_conclusion(run.status, run.conclusion, ExecutionInfo.STATUS_UNKNOWN))
                 if run.status in break_status_list:
                     logging.info(f"Workflow Run status: '{run.status}' is present in input break statuses list. Stop waiting.")
@@ -155,46 +153,54 @@ class GithubClient:
 
     def cancel_workflow_run_execution(self, execution: ExecutionInfo, timeout: float = 1.0):
         """"""
-        repo_full_name = self._get_repo_full_name(execution)
-        if not repo_full_name:
+        owner_and_repo_name = self._get_owner_and_repo(execution)
+        if not owner_and_repo_name:
             return execution
-        run = self.gh.get_repo(repo_full_name).get_workflow_run(int(execution.get_id()))
         counter = 0
         while counter < timeout:
             counter += 1
             logging.info("Waiting pipeline execution timeout 1 second")
             sleep(1)
-        run.cancel()
+        self.gh.actions.cancel_workflow_run(owner_and_repo_name[0], owner_and_repo_name[1], execution.get_id())
         return execution.stop(ExecutionInfo.STATUS_ABORTED)
 
     def download_workflow_run_artifacts(self, execution: ExecutionInfo, local_dir: str):
         """"""
-        repo_full_name = self._get_repo_full_name(execution)
-        if not repo_full_name:
+        owner_and_repo_name = self._get_owner_and_repo(execution)
+        if not owner_and_repo_name:
             return execution
         local_dir_path = Path(local_dir)
         if not local_dir_path.exists():
             local_dir_path.mkdir(parents=True, exist_ok=True)
-        run = self.gh.get_repo(repo_full_name).get_workflow_run(int(execution.get_id()))
-        for artifact in run.get_artifacts():
+        artifacts = self.gh.actions.list_workflow_run_artifacts(owner_and_repo_name[0], owner_and_repo_name[1], execution.get_id(), 100)
+        for artifact in artifacts.artifacts:
             self._save_artifact_to_dir(artifact, local_dir_path)
 
-    def get_workflow_run_input_params(self, run: WorkflowRun, artifact_name: str = DEFAULT_UUID_ARTIFACT_NAME,
+    def get_workflow_run_input_params(self, execution: ExecutionInfo, artifact_name: str = DEFAULT_UUID_ARTIFACT_NAME,
                                       file_name: str = DEFAULT_UUID_FILE_NAME):
         """"""
-        for artifact in run.get_artifacts():
+        owner_and_repo_name = self._get_owner_and_repo(execution)
+        if not owner_and_repo_name:
+            return {}
+        artifacts = self.gh.actions.list_workflow_run_artifacts(owner_and_repo_name[0], owner_and_repo_name[1], execution.get_id(), 100)
+        for artifact in artifacts.artifacts:
             if artifact.name == artifact_name:
                 return self._get_input_params_from_artifact(artifact, file_name)
-        logging.info(f"Could not find input_params artifact for run {run.id}")
+        logging.info(f"Could not find input_params artifact for run {execution.get_id()}")
         return {}
 
-    def _find_run_via_uuid_input_param(self, runs_list: PaginatedList[WorkflowRun], already_checked_runs: list,
+    def get_repo_default_branch(self, owner: str, repo_name: str):
+        return self.gh.repos.get(owner, repo_name).default_branch
+
+    def _find_run_via_uuid_input_param(self, owner: str, repo_name: str,
+                                       workflow_runs: list, already_checked_runs: list,
                                        uuid_artifact_name: str, uuid_file_name: str,
                                        uuid_param_name: str, uuid_param_value: str):
-        for run in runs_list:
+        for run in workflow_runs:
             if run.id in already_checked_runs:
                 continue
-            for artifact in run.get_artifacts():
+            artifacts = self.gh.actions.list_workflow_run_artifacts(owner, repo_name, run.id, 100)
+            for artifact in artifacts.artifacts:
                 if artifact.name == uuid_artifact_name:
                     if self._check_input_params_uuid(artifact, uuid_file_name, uuid_param_name, uuid_param_value):
                         logging.info(f"Found workflow run with expected UUID: {run.id} with {uuid_param_name}={uuid_param_value}")
@@ -204,7 +210,7 @@ class GithubClient:
                         break
         return None
 
-    def _check_input_params_uuid(self, artifact: Artifact, uuid_file_name: str, uuid_param_name: str, uuid_param_value: str):
+    def _check_input_params_uuid(self, artifact, uuid_file_name: str, uuid_param_name: str, uuid_param_value: str):
         try:
             input_params = self._get_input_params_from_artifact(artifact, uuid_file_name)
             return input_params.get(uuid_param_name) == uuid_param_value
@@ -212,7 +218,7 @@ class GithubClient:
             logging.error(f"Exception when downloading and checking artifact ({artifact.name}): {ex}")
         return False
 
-    def _get_input_params_from_artifact(self, artifact: Artifact, file_name: str):
+    def _get_input_params_from_artifact(self, artifact, file_name: str):
         with tempfile.TemporaryDirectory() as temp_dirname:
             artifact_path = self._save_artifact_to_dir(artifact, temp_dirname)
             with zipfile.ZipFile(artifact_path) as zf:
@@ -220,13 +226,13 @@ class GithubClient:
                 with open(Path(temp_dirname, file_name)) as input_params_file:
                     return json.load(input_params_file)
 
-    def _save_artifact_to_dir(self, artifact: Artifact, dirname):
+    def _save_artifact_to_dir(self, artifact, dirname):
         local_path = Path(dirname, f"{artifact.name}.zip")
-        (status, headers, _) = artifact.requester.requestBlob("GET", artifact.archive_download_url)
-        if status != 302:
-            logging.error(f"Unexpected status while downloading run artifact {artifact.name}: expected 302, got {status}")
+        redirect_response = requests.get(artifact.archive_download_url, headers=self.gh.headers, allow_redirects=False)
+        if redirect_response.status_code != 302:
+            logging.error(f"Unexpected status while downloading run artifact {artifact.name}: expected 302, got {redirect_response.status_code}")
             return None
-        response = requests.get(headers["location"])
+        response = requests.get(redirect_response.headers["location"])
         with local_path.open('wb') as f:
             logging.info(f"saving {local_path}...")
             f.write(response.content)
@@ -258,3 +264,10 @@ class GithubClient:
             return None
         url_parts = execution.get_url().split("://")[1].split("/")
         return f"{url_parts[1]}/{url_parts[2]}"
+
+    def _get_owner_and_repo(self, execution: ExecutionInfo):
+        if not execution.get_id() or not execution.get_url():
+            logging.error("Can't get workflow run - empty run id or url in ExecutionInfo!")
+            return None
+        url_parts = execution.get_url().split("://")[1].split("/")
+        return url_parts[1], url_parts[2]

{qubership_pipelines_common_library-0.1.7 → qubership_pipelines_common_library-0.1.9}/qubership_pipelines_common_library/v1/utils/utils_context.py

@@ -38,9 +38,11 @@ def init_context(context_path):
     UtilsFile.write_yaml(context_path, context)
 
 
-def create_execution_context(input_params: dict = None, folder_path: str = None, parent_context_to_reuse: ExecutionContext = None):
+def create_execution_context(input_params: dict = None, input_params_secure: dict = None, folder_path: str = None,
+                             parent_context_to_reuse: ExecutionContext = None):
     """params:
         input_params: dict (will be merged into created input params)
+        input_params_secure: dict (will be merged into created secure input params)
         folder_path: str (optional, will generate new temp)
         parent_context_to_reuse: ExecutionContext (optional, to propagate existing input params)
     """
@@ -71,6 +73,7 @@ def create_execution_context(input_params: dict = None, folder_path: str = None,
 
     input_params_file.content = recursive_merge(input_params_file.content, input_params)
     input_params_file.save(context_file.get("paths.input.params"))
+    input_params_secure_file.content = recursive_merge(input_params_secure_file.content, input_params_secure)
     input_params_secure_file.save(context_file.get("paths.input.params_secure"))
 
     ExecutionContextFile().init_params().save(context_file.get("paths.output.params"))

qubership_pipelines_common_library-0.1.7/qubership_pipelines_common_library/v1/artifactory_client.py

@@ -1,52 +0,0 @@
-# Copyright 2024 NetCracker Technology Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from pyartifactory import Artifactory
-from pyartifactory.exception import PropertyNotFoundError
-
-
-class ArtifactoryClient:
-    def __init__(self, params: dict):
-        """
-        **`params`** is a dictionary with following mandatory params:
-
-        Arguments:
-            url (str): Artifactory host url
-            username (str): User used in auth request
-            password (str): Token used in auth request
-        """
-        self.url = params.get("url")
-        self.user = params.get("username")
-        self.token = params.get("password")
-        self.artifactory = Artifactory(url=self.url, auth=(self.user, self.token), api_version=1)
-        logging.info("Artifactory Client configured for %s", params.get("url"))
-
-    def get_artifact_properties(self, path_to_artifact: str):
-        """"""
-        try:
-            properties = self.artifactory.artifacts.properties(artifact_path=path_to_artifact)
-        except PropertyNotFoundError:
-            logging.error("There are not properties for artifact %s", path_to_artifact)
-            properties = None
-        return properties
-
-    def get_folder_files_list(self, path_to_folder: str):
-        """"""
-        return self.artifactory.artifacts.list(artifact_path=path_to_folder).files
-
-    def get_artifact_content_by_url(self, path_to_file: str):
-        """"""
-        file_content = self.artifactory.artifacts.download(artifact_path=path_to_file)
-        return file_content.read_text("utf-8")