benchmark-runner 1.0.806__tar.gz → 1.0.808__tar.gz

{benchmark_runner-1.0.806/benchmark_runner.egg-info → benchmark_runner-1.0.808}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmark-runner
-Version: 1.0.806
+Version: 1.0.808
 Summary: Benchmark Runner Tool
 Home-page: https://github.com/redhat-performance/benchmark-runner
 Author: Red Hat

benchmark_runner-1.0.808/benchmark_runner/common/google_drive/google_drive_exceptions.py

@@ -0,0 +1,13 @@
+class GoogleDrive(Exception):
+    """ Base class for all ElasticSearch error classes.
+        All exceptions raised by the google drive library should inherit from this class. """
+    pass
+
+
+class FolderNotCreated(GoogleDrive):
+    """Raised when a folder cannot be created in Google Drive after retries."""
+    def __init__(self, folder_path, message=None):
+        if message is None:
+            message = f"Failed to create folder: {folder_path}"
+        super().__init__(message)
+        self.folder_path = folder_path

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/benchmark_runner/common/google_drive/google_drive_operations.py

@@ -9,7 +9,7 @@ from googleapiclient.http import MediaFileUpload
 from google.auth.transport.requests import Request
 from google_auth_oauthlib.flow import InstalledAppFlow
 from benchmark_runner.common.logger.logger_time_stamp import logger  # Added logger import
-
+from benchmark_runner.common.google_drive.google_drive_exceptions import FolderNotCreated
 # Define the scope
 SCOPES = ['https://www.googleapis.com/auth/drive']
 
@@ -241,16 +241,44 @@ class GoogleDriveOperations:
             # Folder exists, return the folder URL
             folder_url = f"{self._google_drive_path}/{folder_id}"
             return folder_url
-        else:
-            # If folder does not exist, create it along the path
-            folder_id = self.create_folder_at_path(folder_path, parent_folder_id)
+
+    def create_drive_folder_url(self, folder_path, parent_folder_id, retries=3, delay=5):
+        """
+         Create folder if it doesn't exist. Retries in case of failure.
+        :param folder_path: The full folder path to retrieve or create.
+        :param parent_folder_id: The starting parent folder ID.
+        :param retries: Number of retry attempts on failure.
+        :param delay: Initial delay between retries in seconds.
+        :return: The Google Drive URL for the folder, or None on failure.
+        """
+        attempt = 1
+        while attempt <= retries:
+            folder_id = self.get_folder_id_by_path(folder_path, parent_folder_id)
             if folder_id:
-                # Return the newly created folder's URL
-                folder_url = f"{self._google_drive_path}/{folder_id}"
-                return folder_url
-            else:
-                logger.error(f"Unable to create or find the folder path: {folder_path}")
-                return None
+                logger.debug(f"Folder already exists: {folder_path}")
+                break
+
+            logger.warning(f"Folder not found. Attempting to create it (Attempt {attempt}/{retries}).")
+            try:
+                folder_id = self.create_folder_at_path(folder_path, parent_folder_id)
+                if folder_id:
+                    logger.info(f"Successfully created folder: {folder_path}")
+                    break
+            except Exception as e:
+                logger.error(f"Failed to create folder on attempt {attempt} due to: {e}")
+                raise FolderNotCreated(folder_path)
+
+            if attempt < retries:
+                wait_time = delay * (2 ** (attempt - 1))
+                logger.warning(f"Retrying in {wait_time} seconds...")
+                time.sleep(wait_time)
+
+            attempt += 1
+
+        if not folder_id:
+            error_msg = f"Unable to create or find the folder path after {retries} attempts: {folder_path}"
+            logger.error(error_msg)
+            raise FolderNotCreated(folder_path)
 
     def list_files_in_folder(self, folder_id, level=0):
         """

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/benchmark_runner/common/oc/oc.py

@@ -1229,6 +1229,10 @@ class OC(SSH):
         node_ips_list = node_ips.split()
         return dict([(k, v) for k, v in zip(node_ips_list[1::2], node_ips_list[::2])])
 
+    def get_vm_status(self, vm_name, namespace: str = environment_variables.environment_variables_dict['namespace']):
+        namespace = f'-n {namespace}' if namespace else ''
+        return self.run(f"{self._cli} get vm {vm_name} {namespace}")
+
     @logger_time_stamp
     def wait_for_vm_status(self, vm_name: str = '', status: VMStatus = VMStatus.Stopped,
                            namespace: str = environment_variables.environment_variables_dict['namespace'],
@@ -1586,8 +1590,26 @@ class OC(SSH):
         :param output_dir:
         :return:
         """
-        self.run(f"oc get vmi {vm_name} -n {namespace} -o yaml > '{output_dir}/{vm_name}.yaml'")
+        self.run(f"oc get vmi '{vm_name}' -n '{namespace}' -o yaml > '{output_dir}/{vm_name}.yaml'")
         pod_name = self.run(
-            f'oc get pod -n {namespace} -o jsonpath="{{.items[?(@.metadata.generateName==\'virt-launcher-{vm_name}-\')].metadata.name}}"'
+            f"oc get pod -n '{namespace}' -o jsonpath=\"{{.items[?(@.metadata.generateName=='virt-launcher-{vm_name}-')].metadata.name}}\""
         )
-        self.run(f"oc get pod {pod_name} -n {namespace} -o yaml > '{output_dir}/{pod_name}.yaml'")
+        self.run(f"oc get pod '{pod_name}' -n '{namespace}' -o yaml > '{output_dir}/{pod_name}.yaml'")
+
+    def save_describe_yml(self, vm_name, vm_access, output_dir='/tmp', namespace: str = environment_variables.environment_variables_dict['namespace']):
+        """
+        This method save pod and vm into yaml per namespace
+        :param vm_name:
+        :param vm_access: True is vm accessible
+        :param namespace:
+        :param output_dir:
+        :return:
+        """
+        if vm_access:
+            self.run(f"oc describe vmi '{vm_name}' -n '{namespace}' > '{output_dir}/describe_vmi_{vm_name}.yaml'")
+            pod_name = self.run(
+                f'oc get pod -n {namespace} -o jsonpath="{{.items[?(@.metadata.generateName==\'virt-launcher-{vm_name}-\')].metadata.name}}"'
+            )
+            self.run(f"oc describe pod '{pod_name}' -n '{namespace}' > '{output_dir}/describe_pod_{pod_name}.yaml'")
+        else:
+            self.run(f"oc describe vm '{vm_name}' -n '{namespace}' > '{output_dir}/describe_vm_{vm_name}.yaml'")

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/benchmark_runner/workloads/bootstorm_vm.py

@@ -69,7 +69,7 @@ class BootstormVM(WorkloadsOperations):
     @logger_time_stamp
     def _wait_vm_access(self, vm_name: str):
         """
-        This method verifies virtctl into VM and return vm node in success or False if failed
+        This method waits for VM access and returns the VM node on success, or False if it fails
         @return:
         """
         if self._oc.get_vm_node(vm_name=vm_name):
@@ -137,7 +137,7 @@ class BootstormVM(WorkloadsOperations):
             total_run_time = self._get_bootstorm_vm_total_run_time()
             self._data_dict.update({'total_run_time': total_run_time})
         # Google drive run_artifacts_url folder path
-        if self._google_drive_path and self.get_run_artifacts_google_drive():
+        if self._google_drive_path:
             self._data_dict.update({'run_artifacts_url': self.get_run_artifacts_google_drive()})
         if self._es_host:
             # upload several run results
@@ -165,9 +165,9 @@ class BootstormVM(WorkloadsOperations):
             yaml=os.path.join(f'{self._run_artifacts_path}', f'{self._name}.yaml'),
             vm_name=self._vm_name)
 
-    def _verify_single_vm(self, vm_name, retries=5, delay=10):
+    def _verify_single_vm_access(self, vm_name, retries=5, delay=10):
         """
-        This method verifies the virtctl status of a single VM using a retry mechanism
+        This method verifies single vm access using a retry mechanism
         :param vm_name: The name of the VM to verify.
         :param retries: Number of retry attempts.
         :param delay: Time to wait (in seconds) between retries.
@@ -192,7 +192,6 @@ class BootstormVM(WorkloadsOperations):
             if attempt < retries - 1:
                 time.sleep(delay)
 
-
         # Final update to self._data_dict after all attempts
         vm_node = self._oc.get_vm_node(vm_name)  # Get the node again in case it changed
         self._data_dict = {
@@ -217,27 +216,29 @@ class BootstormVM(WorkloadsOperations):
 
             try:
                 with open(error_log_path, "w") as error_log_file:
-                    error_log_file.write(str(status_message))
+                    error_log_file.write(self._oc.get_vm_status(vm_name=vm_name) + "\n\n")
+                    error_log_file.write(str(status_message) + "\n")
             except Exception as write_err:
                 logger.error(f"Failed to write error log for {vm_name}: {write_err}")
 
         self._finalize_vm()
         return access_status
 
-    def _verify_single_vm_wrapper(self, vm_name, return_dict):
+    def _verify_single_vm_access_wrapper(self, vm_name, return_dict):
         """
-        This method verifies single vm and update vm status in return_dict
+        This method verifies access to a single VM, saves its YAML files, and updates the VM status in return_dict
         :param vm_name:
         :param return_dict:
         :return:
         """
         self._oc.save_to_yaml(vm_name, output_dir=self._run_artifacts_path)
-        status = self._verify_single_vm(vm_name)
-        return_dict[vm_name] = status
+        vm_access = self._verify_single_vm_access(vm_name)
+        self._oc.save_describe_yml(vm_name, str(vm_access).lower() == 'true', output_dir=self._run_artifacts_path)
+        return_dict[vm_name] = vm_access
 
-    def _verify_vms_in_parallel(self, vm_names):
+    def _verify_vms_access_in_parallel(self, vm_names):
         """
-        This method verifies vms in parallel
+        This method verifies VM access in parallel
         :param vm_names:
         :return:
         """
@@ -251,7 +252,7 @@ class BootstormVM(WorkloadsOperations):
             processes = []
 
             for vm_name in bulk:
-                p = Process(target=self._verify_single_vm_wrapper, args=(vm_name, return_dict))
+                p = Process(target=self._verify_single_vm_access_wrapper, args=(vm_name, return_dict))
                 p.start()
                 processes.append(p)
 
@@ -265,64 +266,62 @@ class BootstormVM(WorkloadsOperations):
         return failure_vms
 
     def _verify_vms_access(self, delay=10):
-            """
-            This method verifies the virtctl SSH login for each VM, either during the upgrade or once for each VM.
-            It prepares the data for ElasticSearch, generates a must-gather in case of an error, and uploads it to Google Drive.
-            :param delay: delay between each iteration
-            """
-            try:
-                vm_names = self._oc._get_all_vm_names()
-                if not vm_names:
-                    raise MissingVMs("No VM names were retrieved from the cluster.")
+        """
+        This method verifies access for each VM
+        It prepares the data for ElasticSearch, generates a must-gather in case of an error, and uploads it to Google Drive.
+        :param delay: delay between each iteration
+        """
+        try:
+            vm_names = self._oc._get_all_vm_names()
+            if not vm_names:
+                raise MissingVMs()
+
+            upgrade_done = True
+            failure_vms = []  # List to store failed VM names
 
-                upgrade_done = True
-                failure_vms = []  # List to store failed VM names
+            if self._wait_for_upgrade_version:
+                logger.info(f"wait for ocp upgrade version: {self._wait_for_upgrade_version}")
+                upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
+                start_time = time.time()
 
-                if self._wait_for_upgrade_version:
-                    logger.info(f"wait for ocp upgrade version: {self._wait_for_upgrade_version}")
+                while (self._timeout <= 0 or time.time() - start_time <= self._timeout) and not upgrade_done:
+                    failure_vms = self._verify_vms_access_in_parallel(vm_names)
                     upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
-                    start_time = time.time()
 
-                    while (self._timeout <= 0 or time.time() - start_time <= self._timeout) and not upgrade_done:
-                        failure_vms = self._verify_vms_in_parallel(vm_names)
-                        upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
+                    if upgrade_done:
+                        break
 
-                        if upgrade_done:
-                            break
+                    # Sleep between each cycle
+                    time.sleep(delay)
+            else:
+                failure_vms = self._verify_vms_access_in_parallel(vm_names)
+
+            if self._wait_for_upgrade_version:
+                logger.info(f'Cluster is upgraded to: {self._wait_for_upgrade_version}')
+
+            if failure_vms:
+                self._oc.generate_cnv_must_gather(destination_path=self._run_artifacts_path, cnv_version=self._cnv_version)
+                self._oc.generate_odf_must_gather(destination_path=self._run_artifacts_path, odf_version=self._odf_version)
+                # Error log with details of failed VM, for catching all vm errors
+                logger.error(f"Failed to verify virtctl SSH login for the following VMs: {', '.join(failure_vms)}")
+            # Upload artifacts in validation
+            if self._google_drive_shared_drive_id:
+                self.upload_run_artifacts_to_google_drive()
+            elif self._endpoint_url:
+                self.upload_run_artifacts_to_s3()
+            else:
+                self._save_artifacts_local = True
+            if self._es_host:
+                self._data_dict.update({'run_artifacts_url': self.get_run_artifacts_google_drive(), 'failure_vms': failure_vms, 'verification_failure': True})
+                # upload several run results
+                self._upload_to_elasticsearch(index=self._es_index, kind=self._kind, status=self._status,result=self._data_dict)
+                # verify that data upload to elastic search according to unique uuid
+                self._verify_elasticsearch_data_uploaded(index=self._es_index, uuid=self._uuid)
 
-                        # Sleep between each cycle
-                        time.sleep(delay)
-                else:
-                    failure_vms = self._verify_vms_in_parallel(vm_names)
-
-                if self._wait_for_upgrade_version:
-                    logger.info(f'Cluster is upgraded to: {self._wait_for_upgrade_version}')
-
-                if failure_vms:
-                    self._oc.generate_cnv_must_gather(destination_path=self._run_artifacts_path,
-                                                      cnv_version=self._cnv_version)
-                    self._oc.generate_odf_must_gather(destination_path=self._run_artifacts_path,
-                                                      odf_version=self._odf_version)
-                    # Error log with details of failed VM, for catching all vm errors
-                    logger.error(f"Failed to verify virtctl SSH login for the following VMs: {', '.join(failure_vms)}")
-                    # Upload artifacts
-                    if self._google_drive_shared_drive_id:
-                        self.upload_run_artifacts_to_google_drive()
-                    elif self._endpoint_url:
-                        self.upload_run_artifacts_to_s3()
-                    else:
-                        self._save_artifacts_local = True
-                    if self._es_host:
-                        self._data_dict.update({'run_artifacts_url': self.get_run_artifacts_google_drive(), 'failure_vms': failure_vms, 'verification_failure': True})
-                        # upload several run results
-                        self._upload_to_elasticsearch(index=self._es_index, kind=self._kind, status=self._status,result=self._data_dict)
-                        # verify that data upload to elastic search according to unique uuid
-                        self._verify_elasticsearch_data_uploaded(index=self._es_index, uuid=self._uuid)
-
-            except Exception as err:
-                # Save run artifacts logs
-                self.save_error_logs()
-                raise err
+        except Exception as err:
+            # Save run artifacts logs
+            self.save_error_logs()
+            raise err
 
     def _run_vm_scale(self, vm_num: str):
         """

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/benchmark_runner/workloads/workloads_operations.py

@@ -373,6 +373,15 @@ class WorkloadsOperations:
         run_artifacts_hierarchy = self._get_run_artifacts_hierarchy(workload_name=workload)
         return self._google_drive_operation.get_drive_folder_url(folder_path=run_artifacts_hierarchy, parent_folder_id=self._google_drive_shared_drive_id)
 
+    def create_run_artifacts_google_drive(self):
+        """
+        This method creates google drive run artifacts folder path
+        :return:
+        """
+        workload = self._workload.replace('_', '-')
+        run_artifacts_hierarchy = self._get_run_artifacts_hierarchy(workload_name=workload)
+        self._google_drive_operation.create_drive_folder_url(folder_path=run_artifacts_hierarchy, parent_folder_id=self._google_drive_shared_drive_id)
+
     @logger_time_stamp
     def upload_run_artifacts_to_google_drive(self):
         """
@@ -524,6 +533,8 @@ class WorkloadsOperations:
         self._template.generate_yamls(scale=str(self._scale), scale_nodes=self._scale_node_list, redis=self._redis, thread_limit=self._threads_limit)
         if self._enable_prometheus_snapshot:
             self.start_prometheus()
+        if self._google_drive_path:
+            self.create_run_artifacts_google_drive()
 
     def finalize_workload(self):
         """
@@ -535,6 +546,8 @@ class WorkloadsOperations:
             self.end_prometheus()
         if self._endpoint_url:
             self.upload_run_artifacts_to_s3()
+        elif self._google_drive_path:
+            self.upload_run_artifacts_to_google_drive()
         if not self._save_artifacts_local:
             self.delete_local_artifacts()
         if self._delete_all:

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808/benchmark_runner.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmark-runner
-Version: 1.0.806
+Version: 1.0.808
 Summary: Benchmark Runner Tool
 Home-page: https://github.com/redhat-performance/benchmark-runner
 Author: Red Hat

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/benchmark_runner.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ benchmark_runner/common/elasticsearch/elasticsearch_operations.py
 benchmark_runner/common/github/__init__.py
 benchmark_runner/common/github/github_operations.py
 benchmark_runner/common/google_drive/__init__.py
+benchmark_runner/common/google_drive/google_drive_exceptions.py
 benchmark_runner/common/google_drive/google_drive_operations.py
 benchmark_runner/common/grafana/__init__.py
 benchmark_runner/common/grafana/grafana_operations.py

{benchmark_runner-1.0.806 → benchmark_runner-1.0.808}/setup.py

@@ -4,7 +4,7 @@ from os import path
 from setuptools import setup, find_packages
 
 
-__version__ = '1.0.806'  # update also .bumpversion.cfg
+__version__ = '1.0.808'  # update also .bumpversion.cfg
 
 
 here = path.abspath(path.dirname(__file__))