benchmark-runner 1.0.787__py3-none-any.whl → 1.0.789__py3-none-any.whl

benchmark_runner/main/environment_variables.py

@@ -90,6 +90,8 @@ class EnvironmentVariables:
         self._environment_variables_dict['run_strategy'] = EnvironmentVariables.get_boolean_from_environment('RUN_STRATEGY', False)
         # Verification only, without running or deleting any resources, default False
         self._environment_variables_dict['verification_only'] = EnvironmentVariables.get_boolean_from_environment('VERIFICATION_ONLY', False)
+        # verify after test
+        self._environment_variables_dict['verify_after_test'] = EnvironmentVariables.get_env('VERIFY_AFTER_TEST', '')
 
         # default parameter - change only if needed
         # Parameters below related to 'run_workload()'

benchmark_runner/workloads/bootstorm_vm.py

@@ -1,7 +1,7 @@
 
 import os
 import time
-from multiprocessing import Process
+from multiprocessing import Process, Manager
 
 from benchmark_runner.common.logger.logger_time_stamp import logger_time_stamp, logger
 from benchmark_runner.common.elasticsearch.elasticsearch_exceptions import ElasticSearchDataNotUploaded
@@ -162,7 +162,7 @@ class BootstormVM(WorkloadsOperations):
             try:
                 virtctl_status = self._oc.get_virtctl_vm_status(vm_name)
 
-                if virtctl_status == "True":
+                if str(virtctl_status).lower() == "true":
                     # Log success and store relevant data
                     logger.info(f"VM {vm_name} verified successfully (virtctl SSH status: { virtctl_status}).")
                     break  # Exit loop on success
@@ -173,22 +173,25 @@ class BootstormVM(WorkloadsOperations):
                 logger.info(f"Attempt {attempt + 1}/{retries} failed for Virtctl SSH VM {vm_name}: {e}")
 
             # Sleep before retrying
-            time.sleep(delay)
+            if attempt < retries - 1:
+                time.sleep(delay)
+
 
         # Final update to self._data_dict after all attempts
         vm_node = self._oc.get_vm_node(vm_name)  # Get the node again in case it changed
         self._data_dict = {
             'vm_name': vm_name,
             'node': vm_node,
-            'virtctl_vm': 1 if virtctl_status == 'True' else 0, # int value for Grafana
+            'virtctl_vm': 1 if str(virtctl_status).lower() == "true" else 0, # int value for Grafana
             'virtctl_status': virtctl_status,
+            'verify_after_test': self._verify_after_test,
             'run_artifacts_url': os.path.join(
                 self._run_artifacts_url,
                 f"{self._get_run_artifacts_hierarchy(self._workload_name, True)}-{self._time_stamp_format}.tar.gz"
             )
         }
 
-        if virtctl_status != "True":
+        if str(virtctl_status).lower() != "true":
             logger.info(
                 f"All attempts failed for VM {vm_name}. Final SSH status: {self._data_dict.get('virtctl_status', 'No status available')}")
             error_log_path = f"{self._run_artifacts_path}/{vm_name}_error.log"
@@ -196,76 +199,109 @@ class BootstormVM(WorkloadsOperations):
             # Retrieve the status or use a default message
             status_message = self._data_dict.get('virtctl_status') or "No status available"
 
-            with open(error_log_path, "w") as error_log_file:
-                error_log_file.write(str(status_message))  # Convert to string just in case
+            try:
+                with open(error_log_path, "w") as error_log_file:
+                    error_log_file.write(str(status_message))
+            except Exception as write_err:
+                logger.error(f"Failed to write error log for {vm_name}: {write_err}")
 
         self._finalize_vm()
         return virtctl_status
 
-    def _verify_virtctl_vm(self):
+    def _verify_single_vm_wrapper(self, vm_name, return_dict):
         """
-        This method verifies the virtctl SSH login for each VM, either during the upgrade or once for each VM.
-        It prepares the data for ElasticSearch, generates a must-gather in case of an error, and uploads it to Google Drive.
+        This method verifies single vm and update vm status in return_dict
+        :param vm_name:
+        :param return_dict:
+        :return:
         """
-        try:
-            vm_names = self._oc._get_all_vm_names()
-            if not vm_names:
-                raise MissingVMs("No VM names were retrieved from the cluster.")
-
-            upgrade_done = True
-            failure = False
-            failure_vms = []  # List to store failed VM names
-
-            if self._wait_for_upgrade_version:
-                logger.info(f"wait for ocp upgrade version: {self._wait_for_upgrade_version}")
-                upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
-                current_wait_time = 0
-
-                while (self._timeout <= 0 or current_wait_time <= self._timeout) and not upgrade_done:
-                    for vm_name in vm_names:
-                        virtctl_status = self._verify_single_vm(vm_name)
-                        if virtctl_status != 'True':
-                            failure = True
-                            if vm_name not in failure_vms:
-                                failure_vms.append(vm_name)
+        status = self._verify_single_vm(vm_name)
+        return_dict[vm_name] = status
+
+    def _verify_vms_in_parallel(self, vm_names):
+        """
+        This method verifies vms in parallel
+        :param vm_names:
+        :return:
+        """
+        failure_vms = []
+        manager = Manager()
+        return_dict = manager.dict()
+
+        # Split vm_names according to self._threads_limit
+        for i in range(0, len(vm_names), self._threads_limit):
+            bulk = vm_names[i:i + self._threads_limit]
+            processes = []
+
+            for vm_name in bulk:
+                p = Process(target=self._verify_single_vm_wrapper, args=(vm_name, return_dict))
+                p.start()
+                processes.append(p)
+
+            for p in processes:
+                p.join()
+
+        # After all processes are done, collect failures
+        for vm_name, status in return_dict.items():
+            if str(status).lower() != 'true':
+                failure_vms.append(vm_name)
+        return failure_vms
+
+    def _verify_virtctl_vms(self, delay=10):
+            """
+            This method verifies the virtctl SSH login for each VM, either during the upgrade or once for each VM.
+            It prepares the data for ElasticSearch, generates a must-gather in case of an error, and uploads it to Google Drive.
+            :param delay: delay between each iteration
+            """
+            try:
+                vm_names = self._oc._get_all_vm_names()
+                if not vm_names:
+                    raise MissingVMs("No VM names were retrieved from the cluster.")
+
+                upgrade_done = True
+                failure_vms = []  # List to store failed VM names
+
+                if self._wait_for_upgrade_version:
+                    logger.info(f"wait for ocp upgrade version: {self._wait_for_upgrade_version}")
                     upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
+                    start_time = time.time()
 
-                    # Sleep 1 sec between each cycle
-                    time.sleep(1)
-                    current_wait_time += 1  # Increment the wait time
-            else:
-                # If _wait_for_upgrade_version is empty, verify VM SSH without waiting for upgrade
-                for vm_name in vm_names:
-                    virtctl_status = self._verify_single_vm(vm_name)
-                    if virtctl_status != 'True':
-                        failure = True
-                        if vm_name not in failure_vms:
-                            failure_vms.append(vm_name)
-
-            if self._wait_for_upgrade_version:
-                logger.info(f'Cluster is upgraded to: {self._wait_for_upgrade_version}')
-
-            if failure:
-                self._oc.generate_cnv_must_gather(destination_path=self._run_artifacts_path,
-                                                  cnv_version=self._cnv_version)
-                self._oc.generate_odf_must_gather(destination_path=self._run_artifacts_path,
-                                                  odf_version=self._odf_version)
-                # Upload artifacts
-                if self._google_drive_shared_drive_id:
-                    self.upload_run_artifacts_to_google_drive()
-                elif self._endpoint_url and not self._google_drive_shared_drive_id:
-                    self.upload_run_artifacts_to_s3()
-                else:
-                    self._save_artifacts_local = True
+                    while (self._timeout <= 0 or time.time() - start_time <= self._timeout) and not upgrade_done:
+                        failure_vms = self._verify_vms_in_parallel(vm_names)
+                        upgrade_done = self._oc.get_cluster_status() == f'Cluster version is {self._wait_for_upgrade_version}'
 
-                # Error log with details of failed VM, for catching all vm errors
-                logger.error(
-                    f"Failed to verify virtctl SSH login for the following VMs: {', '.join(failure_vms)}")
+                        if upgrade_done:
+                            break
 
-        except Exception as err:
-            # Save run artifacts logs
-            self.save_error_logs()
-            raise err
+                        # Sleep between each cycle
+                        time.sleep(delay)
+                else:
+                    failure_vms = self._verify_vms_in_parallel(vm_names)
+
+                if self._wait_for_upgrade_version:
+                    logger.info(f'Cluster is upgraded to: {self._wait_for_upgrade_version}')
+
+                if failure_vms:
+                    self._oc.generate_cnv_must_gather(destination_path=self._run_artifacts_path,
+                                                      cnv_version=self._cnv_version)
+                    self._oc.generate_odf_must_gather(destination_path=self._run_artifacts_path,
+                                                      odf_version=self._odf_version)
+                    # Upload artifacts
+                    if self._google_drive_shared_drive_id:
+                        self.upload_run_artifacts_to_google_drive()
+                    elif self._endpoint_url and not self._google_drive_shared_drive_id:
+                        self.upload_run_artifacts_to_s3()
+                    else:
+                        self._save_artifacts_local = True
+
+                    # Error log with details of failed VM, for catching all vm errors
+                    logger.error(
+                        f"Failed to verify virtctl SSH login for the following VMs: {', '.join(failure_vms)}")
+
+            except Exception as err:
+                # Save run artifacts logs
+                self.save_error_logs()
+                raise err
 
     def _run_vm_scale(self, vm_num: str):
         """
@@ -349,7 +385,7 @@ class BootstormVM(WorkloadsOperations):
     def run_vm_workload(self):
         # verification only w/o running or deleting any resource
         if self._verification_only:
-            self._verify_virtctl_vm()
+            self._verify_virtctl_vms()
         else:
             if not self._scale:
                 self._run_vm()

benchmark_runner/workloads/workloads_operations.py

@@ -17,7 +17,7 @@ from benchmark_runner.common.clouds.shared.s3.s3_operations import S3Operations
 from benchmark_runner.common.prometheus.prometheus_snapshot import PrometheusSnapshot
 from benchmark_runner.common.prometheus.prometheus_snapshot_exceptions import PrometheusSnapshotError
 from benchmark_runner.common.template_operations.template_operations import TemplateOperations
-from benchmark_runner.common.clouds.IBM.ibm_operations import IBMOperations
+from benchmark_runner.common.clouds.BareMetal.bare_metal_operations import BareMetalOperations
 from benchmark_runner.common.prometheus.prometheus_metrics_operations import PrometheusMetricsOperation
 from benchmark_runner.common.google_drive.google_drive_operations import GoogleDriveOperations
 
@@ -103,6 +103,7 @@ class WorkloadsOperations:
         self._windows_url = self._environment_variables_dict.get('windows_url', '')
         self._delete_all = self._environment_variables_dict.get('delete_all', '')
         self._verification_only = self._environment_variables_dict.get('verification_only', '')
+        self._verify_after_test = self._environment_variables_dict.get('verify_after_test', '')
         self._wait_for_upgrade_version = self._environment_variables_dict.get('wait_for_upgrade_version', '')
         if self._windows_url:
             file_name = os.path.basename(self._windows_url)
@@ -390,7 +391,7 @@ class WorkloadsOperations:
         """
         date_format = '%Y_%m_%d'
         metadata = {'ocp_version': self._oc.get_ocp_server_version(),
-                    'previous_ocp_version': self._oc.get_previous_ocp_version() if self._upgrade_ocp_version else '',
+                    'previous_ocp_version': '' if len(self._oc.get_previous_ocp_version()) > 10 else self._oc.get_previous_ocp_version(),
                     'cnv_version': self._oc.get_cnv_version(),
                     'kata_version': self._oc.get_kata_operator_version(),
                     'kata_rpm_version': self._oc.get_kata_rpm_version(node=self._pin_node1),
@@ -455,15 +456,15 @@ class WorkloadsOperations:
         """
         if self._run_type == 'test_ci':
             es_index = 'ci-status-test'
+        elif self._run_type == 'chaos_ci':
+            es_index = 'ci-status-chaos'
         else:
             es_index = 'ci-status'
         status_dict = {'failed': 0, 'pass': 1}
         metadata = self.__get_metadata()
         if ocp_resource_install_minutes_time != 0:
-            ibm_operations = IBMOperations(user=self._environment_variables_dict.get('provision_user', ''))
-            ibm_operations.ibm_connect()
-            ocp_install_minutes_time = ibm_operations.get_ocp_install_time()
-            ibm_operations.ibm_disconnect()
+            bm_operations = BareMetalOperations(user=self._environment_variables_dict.get('provision_user', ''))
+            ocp_install_minutes_time = bm_operations.get_ocp_install_time()
         metadata.update({'status': status, 'status#': status_dict[status], 'ci_minutes_time': ci_minutes_time, 'benchmark_runner_id': benchmark_runner_id, 'benchmark_operator_id': benchmark_operator_id, 'benchmark_wrapper_id': benchmark_wrapper_id, 'ocp_install_minutes_time': ocp_install_minutes_time, 'ocp_resource_install_minutes_time': ocp_resource_install_minutes_time})
         self._es_operations.upload_to_elasticsearch(index=es_index, data=metadata)
 

{benchmark_runner-1.0.787.dist-info → benchmark_runner-1.0.789.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmark-runner
-Version: 1.0.787
+Version: 1.0.789
 Summary: Benchmark Runner Tool
 Home-page: https://github.com/redhat-performance/benchmark-runner
 Author: Red Hat

{benchmark_runner-1.0.787.dist-info → benchmark_runner-1.0.789.dist-info}/RECORD

@@ -161,20 +161,20 @@ benchmark_runner/krkn_hub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 benchmark_runner/krkn_hub/krknhub_exceptions.py,sha256=Hk7Co6zZ0u2RSmBmS4ZAi21ffZaQ2ITTfl6tGLtiAdY,180
 benchmark_runner/krkn_hub/krknhub_workloads.py,sha256=vUEw5y1rkhspxPdRFTHF_jKo1XwVphtkL-oSQQULVyI,3032
 benchmark_runner/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-benchmark_runner/main/environment_variables.py,sha256=4kJx_e6vrI4ThBvIiCEkvmwbSfQit2EuUHiKmVllYe8,28356
+benchmark_runner/main/environment_variables.py,sha256=ebp3VjLHzeprjZbhs8rGOHASfW3nKK-7BR-J7yBOYFg,28502
 benchmark_runner/main/environment_variables_exceptions.py,sha256=UR0Ith0P0oshsDZdJRlRq8ZUTt0h8jFvUtrnP4m4AIY,437
 benchmark_runner/main/main.py,sha256=A744O550wQh37hhk10H0HlT28LZ_2EOaRlJyWG6Pras,14083
 benchmark_runner/main/temporary_environment_variables.py,sha256=ODSHkfhgvdr_b2e3XyvykW21MVjSdyqimREyMc2klRE,957
 benchmark_runner/workloads/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-benchmark_runner/workloads/bootstorm_vm.py,sha256=04hjGxfDcqpkJcarz-u56GqkrErQMUBC0c0vDQWn0h8,18754
+benchmark_runner/workloads/bootstorm_vm.py,sha256=C0V4hfWSW_-kXLkC3MeW_Ie5myLfhDk3myu-e8eGhNs,19900
 benchmark_runner/workloads/vdbench_pod.py,sha256=OcmxVr5QlbkhAgY37wsZ7xJUjs9HI-qMVDex1HdL2P0,9778
 benchmark_runner/workloads/vdbench_vm.py,sha256=Yhoz-GbvZwA8q6qGIeSUsYhEIERj8SmJB1yjetwsGow,9449
 benchmark_runner/workloads/windows_vm.py,sha256=qFVD3qBFMnVpYXnrpam-7H5-0Yzvx6qtaEEZx4T-ex4,2415
 benchmark_runner/workloads/workloads.py,sha256=F9fnk4h715tq7ANSCbDH0jktB8fpr_u3YG61Kdi5_os,1422
 benchmark_runner/workloads/workloads_exceptions.py,sha256=u7VII95iPRF_YhfpGH1U1RmgiIYESMOtbSF1dz7_ToE,1858
-benchmark_runner/workloads/workloads_operations.py,sha256=EEIZjNYGjEkyUafxFKSu4X9jaN_wgYDbMeCi1hZBZ-0,25186
-benchmark_runner-1.0.787.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-benchmark_runner-1.0.787.dist-info/METADATA,sha256=bsMnUcGVMnQTr5F5Uqf2drMDe5VgSKiozbvu585JRLA,11520
-benchmark_runner-1.0.787.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-benchmark_runner-1.0.787.dist-info/top_level.txt,sha256=MP7UbTCzu59D53uKCZl5VsQeM_vheyMc7FmryczJQbk,17
-benchmark_runner-1.0.787.dist-info/RECORD,,
+benchmark_runner/workloads/workloads_operations.py,sha256=nHwy61d-CyyW03HsgPJZ73SKuq2rY0aLaRL6R4V0eLc,25324
+benchmark_runner-1.0.789.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+benchmark_runner-1.0.789.dist-info/METADATA,sha256=WliGrtbb6nGLtemvwvmitKavx6ywUJjCowiKrHFJ_mg,11520
+benchmark_runner-1.0.789.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+benchmark_runner-1.0.789.dist-info/top_level.txt,sha256=MP7UbTCzu59D53uKCZl5VsQeM_vheyMc7FmryczJQbk,17
+benchmark_runner-1.0.789.dist-info/RECORD,,