benchmark-runner 1.0.813__py3-none-any.whl → 1.0.909__py3-none-any.whl

benchmark_runner/benchmark_operator/benchmark_operator_workloads_operations.py

@@ -462,7 +462,7 @@ class BenchmarkOperatorWorkloadsOperations:
         :return:
         """
         workload_name = self._workload.split('_')
-        if workload_name[0] in self._workloads_odf_pvc:
+        if workload_name[0] in self._workloads_odf_pvc and '_lso' not in self._workload:
             if not self._oc.is_odf_installed():
                 raise ODFNotInstalled(workload=self._workload)
 

benchmark_runner/common/clouds/BareMetal/bare_metal_operations.py

@@ -42,6 +42,8 @@ class BareMetalOperations:
         self._upgrade_ocp_version = self._environment_variables_dict.get('upgrade_ocp_version', '')
         self._upgrade_channel = self._environment_variables_dict.get('upgrade_channel', '')
         self._timeout = int(self._environment_variables_dict.get('timeout', ''))
+        self._must_gather_log = self._environment_variables_dict.get('must_gather_log', '')
+        self._run_artifacts_path = self._environment_variables_dict.get('run_artifacts_path', '')
         if user:
             self._user = user
             self._provision_kubeadmin_password_path = self._environment_variables_dict.get('provision_kubeadmin_password_path', '')
@@ -107,14 +109,17 @@ class BareMetalOperations:
         current_wait_time = 0
         logger.info(f'Waiting until the upgrade to version {self._upgrade_ocp_version} starts...')
         oc.wait_for_ocp_upgrade_start(upgrade_version=self._upgrade_ocp_version)
-        while self._timeout <= 0 or current_wait_time <= self._timeout and oc.upgrade_in_progress():
+        while (self._timeout <= 0 or current_wait_time <= self._timeout) and oc.upgrade_in_progress():
             logger.info(f'Waiting till OCP upgrade complete, waiting {int(current_wait_time / 60)} minutes')
             # sleep for x seconds
             time.sleep(sleep_time)
             current_wait_time += sleep_time
-        if f'Cluster version is {self._upgrade_ocp_version}' == oc.get_cluster_status():
+        if oc.get_upgrade_version() == self._upgrade_ocp_version:
+            logger.info(f"OCP successfully upgraded to {oc.get_upgrade_version()}")
             return True
         else:
+            if self._must_gather_log:
+                oc.generate_ocp_must_gather(destination_path=self._run_artifacts_path)
             raise OCPUpgradeFailed(status=oc.get_cluster_status())
 
     def _install_ocp_cmd(self):
@@ -283,24 +288,34 @@ class BareMetalOperations:
         else:
             return True
 
-    def is_cluster_upgraded(self, oc: OC, cnv_version: str, odf_version: str, lso_version: str):
+    def is_cluster_upgraded(self, oc: OC, cnv_version: str = None, odf_version: str = None,
+                            lso_version: str = None) -> bool:
         """
-        This method checks if cluster/operators upgraded successfully
-        @param oc:
-        @param cnv_version:
-        @param odf_version:
-        @param lso_version:
-        @return:
+        Checks if the cluster and the specified operators were successfully upgraded.
+
+        :param oc: OC client instance
+        :param cnv_version: (Optional) Target CNV operator version
+        :param odf_version: (Optional) Target ODF operator version
+        :param lso_version: (Optional) Target LSO operator version
+        :return: True if the upgrade (cluster + specified operators) is successful, otherwise False
         """
-        # check cluster version upgrade
-        if oc.get_upgrade_version() == self._upgrade_ocp_version:
-            lso_upgraded = oc.wait_for_operator_installation(operator='lso', version=lso_version, namespace='openshift-local-storage')
-            odf_upgraded = oc.wait_for_operator_installation(operator='odf', version=odf_version, namespace='openshift-storage')
-            cnv_upgraded = oc.wait_for_operator_installation(operator='cnv', version=cnv_version, namespace='openshift-cnv')
-            # check operators versions upgrade
-            if cnv_upgraded and lso_upgraded and odf_upgraded:
-                return True
-        return False
+        if oc.get_upgrade_version() != self._upgrade_ocp_version:
+            return False
+
+        # Operator details mapping: short_key -> (version, namespace, operator_name_prefix)
+        operators_to_check = {
+            "cnv": (cnv_version, "openshift-cnv", "kubevirt-hyperconverged-operator"),
+            "odf": (odf_version, "openshift-storage", "odf-operator"),
+            "lso": (lso_version, "openshift-local-storage", "local-storage-operator"),
+        }
+
+        upgrade_checks = [
+            oc.wait_for_operator_installation(operator_name=operator_name, version=ver, namespace=ns)
+            for _, (ver, ns, operator_name) in operators_to_check.items()
+            if ver  # only include if a version was specified
+        ]
+
+        return all(upgrade_checks) if upgrade_checks else True
 
     @logger_time_stamp
     def verify_upgrade_complete(self, oc: OC):

benchmark_runner/common/grafana/update_grafana_latest_value_mappings.py

@@ -26,6 +26,30 @@ class UpdateGrafanaLatestValueMappings:
         self.main_libsonnet_path = main_libsonnet_path
         self.value_mappings = self.get_value_mappings()
 
+    @staticmethod
+    def _normalize_version_for_grafana(version: str) -> str:
+        """
+        Normalizes a version string for Grafana, then strips all non-digit characters.
+
+        Examples:
+            '4.14.0-ec.2' -> '414002'
+            '4.15.0-rc.3' -> '415013'
+            '4.15.5' -> '4155'
+
+        Returns:
+            A normalized numeric string, replacing '-ec.' with '0' and '-rc.' with '1' if present;
+            otherwise returns the version string with all non-digit characters removed.
+        """
+        qualifiers = {'-ec.': '0', '-rc.': '1'}
+        for qualifier, replacement in qualifiers.items():
+            if qualifier in version:
+                replaced = version.replace(qualifier, replacement)
+                digits_only = ''.join(c for c in replaced if c.isdigit())
+                return digits_only
+
+        # No qualifier found; return only digits from the original version
+        return ''.join(c for c in version if c.isdigit())
+
     def get_last_elasticsearch_versions(self, last_es_fetch_days=LAST_ES_FETCH_DAYS):
         """
         This method fetches new versions from ElasticSearch
@@ -42,10 +66,11 @@ class UpdateGrafanaLatestValueMappings:
         display_versions = ['ocp_version', 'cnv_version', 'kata_version', 'kata_rpm_version', 'odf_version']
         for id in ids:
             data = self.elasticsearch.get_elasticsearch_index_by_id(index='ci-status', id=id)
-            for version, value in data['_source'].items():
-                if version in display_versions and value not in new_versions.values() and len(value) < self.MAX_VERSION_LEN:
-                    # Display only numbers in the Grafana panel, removing characters
-                    new_versions[value.translate(str.maketrans('', '', '.-rcef'))] = value
+            for resource, version in data['_source'].items():
+                if resource in display_versions and version not in new_versions.values() and len(version) < self.MAX_VERSION_LEN:
+                    # Normalize version for Grafana panel if it contains non-digit characters
+                    normalized = self._normalize_version_for_grafana(version)
+                    new_versions[normalized] = version
 
         return new_versions
 
@@ -88,9 +113,11 @@ class UpdateGrafanaLatestValueMappings:
         max_index, max_key = sorted_mapping[-1][1]['index'], sorted_mapping[-1][0]
 
         num = 1
-        for key, value in last_versions.items():
-            if not self.value_mappings.get(key):
-                self.value_mappings[key] = {"index": int(max_index) + num, "text": value}
+        for version_key, version in last_versions.items():
+            if not self.value_mappings.get(version_key) or self.value_mappings[version_key]['text'] != version:
+                # Normalize version for Grafana panel
+                normalized = self._normalize_version_for_grafana(version)
+                self.value_mappings[normalized] = {"index": int(max_index) + num, "text": version}
                 num += 1
 
     def update_main_libsonnet(self):

benchmark_runner/common/oc/oc.py

@@ -42,6 +42,8 @@ class OC(SSH):
         self.__kata_csv = self.__environment_variables_dict.get('kata_csv', '')
         self.__worker_disk_prefix = self.__environment_variables_dict.get('worker_disk_prefix', '')
         self.__worker_disk_ids = self.__environment_variables_dict.get('worker_disk_ids', '')
+        self.__skip_all_namespaces_operators_list = ast.literal_eval(self.__environment_variables_dict.get('skip_all_namespaces_operators_list', ''))
+        self.__skip_operators_pattern_list = "|".join(self.__skip_all_namespaces_operators_list)
         if self.__worker_disk_ids:
             self.__worker_disk_ids = ast.literal_eval(self.__worker_disk_ids)
         self._cli = self.__environment_variables_dict.get('cli', '')
@@ -50,6 +52,10 @@ class OC(SSH):
         else:
             self._kubeadmin_password = self.__environment_variables_dict.get('kubeadmin_password', '')
         self._kubeconfig_path = self.__environment_variables_dict.get('kubeconfig_path', '')
+        self._must_gather_log = self.__environment_variables_dict.get('must_gather_log', '')
+        self._run_artifacts_path = self.__environment_variables_dict.get('run_artifacts_path', '')
+        self._cnv_version = self.__environment_variables_dict.get('cnv_version', '')
+        self._odf_version = self.__environment_variables_dict.get('odf_version', '')
         # Singleton Login class
         SingletonOCLogin(self)
 
@@ -89,8 +95,9 @@ class OC(SSH):
         @param upgrade_channel: upgrade channel candidate or stable, default stable
         @return:
         """
+        # default stable
+        upgrade_channel = upgrade_channel or 'stable'
         ocp_channel = '.'.join(upgrade_ocp_version.split('.')[:2])
-
         # see: https://access.redhat.com/articles/7031404
         if ocp_channel == "4.16":
             patch_command = f"{self._cli} -n openshift-config patch cm admin-acks --patch '{{\"data\":{{\"ack-4.15-kube-1.29-api-removals-in-4.16\":\"true\"}}}}' --type=merge"
@@ -101,25 +108,37 @@ class OC(SSH):
 
     def upgrade_in_progress(self):
         """
-        This method returns True when an upgrade is in progress and False when it is not.
-        @return: bool
+        Checks whether an OpenShift upgrade is in progress by inspecting the 'Progressing' status condition.
+        Retries on transient failures.
+
+        @return: bool - True if upgrade is in progress, False otherwise.
         """
-        status = self.run(f"{self._cli} get clusterversion version -o jsonpath='{{.status.conditions[?(@.type==\"Progressing\")].status}}'")
-        return status == 'True'
+        for i in range(1, self.RETRIES + 1):
+            try:
+                status = self.run(f"{self._cli} get clusterversion version -o jsonpath='{{.status.conditions[?(@.type==\"Progressing\")].status}}'").strip()
+                return status == 'True'
+            except Exception as e:
+                logger.warning(f"[upgrade_in_progress] attempt {i}/{self.RETRIES} failed: {e}")
+                time.sleep(self.DELAY)
+
+        return False
 
     @logger_time_stamp
     def wait_for_ocp_upgrade_start(self, upgrade_version: str, timeout: int = SHORT_TIMEOUT):
         """
-        This method waits for ocp upgrade to start
-        :param upgrade_version:
-        :param timeout:
-        :return:
+        This method waits for the OCP upgrade to start.
+        :param upgrade_version: The target OCP version.
+        :param timeout: Timeout in seconds (waits indefinitely if <= 0).
+        :return: True if upgrade starts within timeout, else raises UpgradeNotStartTimeout.
         """
         current_wait_time = 0
-        while timeout <= 0 or current_wait_time <= timeout and not self.upgrade_in_progress():
-            # sleep for x seconds
+        while timeout <= 0 or current_wait_time <= timeout:
+            if self.upgrade_in_progress():
+                return True
             time.sleep(OC.SLEEP_TIME)
             current_wait_time += OC.SLEEP_TIME
+
+        # Final check after timeout
         if self.upgrade_in_progress():
             return True
         else:
@@ -139,34 +158,36 @@ class OC(SSH):
         """
         return self.run(f"{self._cli} get clusterversion version -o jsonpath='{{.status.conditions[?(@.type==\"Progressing\")].message}}'")
 
-    def get_operator_version(self, namespace):
+    def get_operator_version(self, namespace: str, operator_name: str = None):
         """
         This method returns the operator version from the specified namespace.
         @param namespace: str - The namespace to search for the operator version.
+        @param operator_name: str - The name of the operator to search for.
         @return: major version
         """
-        version = self.run(f"{self._cli} get csv -n {namespace} -o jsonpath='{{.items[0].spec.version}}'")
+        cmd = f"""{self._cli} get csv -n {namespace} -o json | jq -r '.items[] | select(.metadata.name | startswith("{operator_name}")) | .spec.version'"""
+        version = self.run(cmd)
         return '.'.join(version.split('.')[:2])
 
-    def wait_for_operator_installation(self, operator: str, version: str, namespace: str, timeout: int =  SHORT_TIMEOUT):
+    def wait_for_operator_installation(self, operator_name: str, version: str, namespace: str, timeout: int = int(environment_variables.environment_variables_dict['timeout'])):
         """
         This method waits till operator version is installed successfully
-        @param operator:
+        @param operator_name:
         @param version:
         @param timeout:
         @param namespace:
         @return:
         """
         current_wait_time = 0
-        while timeout <= 0 or current_wait_time <= timeout and not self.get_operator_version(namespace) == version:
+        while timeout <= 0 or current_wait_time <= timeout and not self.get_operator_version(namespace, operator_name) == version:
             # sleep for x seconds
             time.sleep(OC.SLEEP_TIME)
             current_wait_time += OC.SLEEP_TIME
-        if self.get_operator_version(namespace) == version:
-            logger.info(f'{operator} operator version: {version} in namespace: {namespace} has been installed successfully')
+        if self.get_operator_version(namespace, operator_name) == version:
+            logger.info(f'{operator_name} operator version: {version} in namespace: {namespace} has been installed successfully')
             return True
         else:
-            raise OperatorInstallationTimeout(operator=operator, version=version, namespace=namespace)
+            raise OperatorInstallationTimeout(operator=operator_name, version=version, namespace=namespace)
 
     def healthcheck(self, action: str):
         """
@@ -186,6 +207,20 @@ class OC(SSH):
         """
         return self.run(f"{self._cli} get csv -n openshift-cnv -o json | jq -r '.items[] | select(.metadata.name | startswith(\"kubevirt-hyperconverged-operator\")) | .spec.version'")
 
+    def get_nhc_version(self):
+        """
+        This method returns Node Health Check Operator version
+        :return:
+        """
+        return self.run(f"""{self._cli} get csv -n openshift-workload-availability -o json | jq -r '.items[] | select(.metadata.name | startswith("node-healthcheck-operator")) | .spec.version'""")
+
+    def get_far_version(self):
+        """
+        This method returns Fence Agents Remediation Operator version
+        :return:
+        """
+        return self.run(f"""{self._cli} get csv -n openshift-workload-availability -o json | jq -r '.items[] | select(.metadata.name | startswith("fence-agents-remediation")) | .spec.version'""")
+
     def get_odf_version(self):
         """
         This method returns the ODF version by extracting it from the csv name.
@@ -517,6 +552,9 @@ class OC(SSH):
             time.sleep(wait_time)
             current_wait_time += wait_time
         logger.info(f"oc get nodes:\n{self.run('oc get nodes')}")
+        if self._must_gather_log:
+            self.generate_must_gather(run_artifacts_path=self._run_artifacts_path, cnv_version=self._cnv_version,
+                                          odf_version=self._odf_version)
         raise NodeNotReady(nodes_status=nodes_status)
 
     @typechecked
@@ -944,8 +982,8 @@ class OC(SSH):
         current_wait_time = 0
 
         while timeout <= 0 or current_wait_time <= timeout:
-            #  Filter out node-healthcheck-operator and self-node-remediation during CSV verification because they exist in all namespaces
-            upgrade_versions = self.run(f"{self._cli} get csv -n {namespace} -o json | jq -r '.items[] | select(.metadata.name | test(\"node-healthcheck-operator|self-node-remediation\") | not) | .spec.version'".splitlines()
+            #  Skip all namespaces operators during CSV verification because they exist in all namespaces
+            upgrade_versions = self.run(f"{self._cli} get csv -n {namespace} -o json | jq -r '.items[] | select(.metadata.name | test(\"{self.__skip_operators_pattern_list}\") | not) | .spec.version'".splitlines()
 ).splitlines()
             count_upgrade_version = sum(1 for actual_upgrade_version in upgrade_versions if
                                         '.'.join(actual_upgrade_version.split('.')[0:2]) == upgrade_version)
@@ -1521,6 +1559,29 @@ class OC(SSH):
                             results_list.append(line.strip().split(':')[data_index:])
         return results_list
 
+    @typechecked
+    @logger_time_stamp
+    def generate_ocp_must_gather(self, destination_path: str = '/tmp'):
+        """
+        Generates OpenShift must-gather and stores it in the destination path.
+
+        :param destination_path: The directory where the must-gather logs will be stored. Default is '/tmp'.
+        :return: The result of the run command.
+        :raises: RuntimeError if the command fails.
+        """
+        folder_path = os.path.join(destination_path, "must-gather")
+
+        try:
+            command = f'oc adm must-gather --dest-dir={folder_path}'
+            self.run(command)
+        except Exception as e:
+            if os.path.exists(folder_path):
+                try:
+                    shutil.rmtree(folder_path)
+                except Exception as remove_error:
+                    raise RuntimeError(f"Failed to remove folder {folder_path}: {remove_error}")
+            raise RuntimeError(f"Failed to generate OCP must-gather logs for version: {e}")
+
     @typechecked
     @logger_time_stamp
     def generate_odf_must_gather(self, destination_path: str = '/tmp', odf_version: str = None):
@@ -1582,21 +1643,32 @@ class OC(SSH):
                     raise RuntimeError(f"Failed to remove folder {folder_path}: {remove_error}")
             raise RuntimeError(f"Failed to generate CNV must-gather logs for version {cnv_version}: {e}")
 
-    def save_to_yaml(self, vm_name, output_dir='/tmp', namespace: str = environment_variables.environment_variables_dict['namespace']):
+    def generate_must_gather(self, run_artifacts_path, cnv_version, odf_version):
+        """
+        This method generates must gather and saves it in the run artifacts path
+        """
+        self.generate_ocp_must_gather(destination_path=run_artifacts_path)
+        self.generate_cnv_must_gather(destination_path=run_artifacts_path, cnv_version=cnv_version)
+        self.generate_odf_must_gather(destination_path=run_artifacts_path, odf_version=odf_version)
+
+    def save_to_yaml(self, vm_name, vm_access, output_dir='/tmp', namespace: str = environment_variables.environment_variables_dict['namespace']):
         """
         This method save pod and vm into yaml per namespace
         :param vm_name:
+        :param vm_access: True is vm accessible, if not there is no pod launcher
         :param namespace:
         :param output_dir:
         :return:
         """
-        self.run(f"oc get vmi '{vm_name}' -n '{namespace}' -o yaml > '{output_dir}/{vm_name}.yaml'")
-        pod_name = self.run(
-            f"oc get pod -n '{namespace}' -o jsonpath=\"{{.items[?(@.metadata.generateName=='virt-launcher-{vm_name}-')].metadata.name}}\""
-        )
-        self.run(f"oc get pod '{pod_name}' -n '{namespace}' -o yaml > '{output_dir}/{pod_name}.yaml'")
+        self.run(f"oc get vm '{vm_name}' -n '{namespace}' -o yaml > '{output_dir}/vm-{vm_name}.yaml'")
+        if vm_access:
+            self.run(f"oc get vmi '{vm_name}' -n '{namespace}' -o yaml > '{output_dir}/vmi-{vm_name}.yaml'")
+            pod_name = self.run(
+                f"oc get pod -n '{namespace}' -o jsonpath=\"{{.items[?(@.metadata.generateName=='virt-launcher-{vm_name}-')].metadata.name}}\""
+            )
+            self.run(f"oc get pod '{pod_name}' -n '{namespace}' -o yaml > '{output_dir}/{pod_name}.yaml'")
 
-    def save_describe_yml(self, vm_name, vm_access, output_dir='/tmp', namespace: str = environment_variables.environment_variables_dict['namespace']):
+    def save_describe_yaml(self, vm_name, vm_access, output_dir='/tmp', namespace: str = environment_variables.environment_variables_dict['namespace']):
         """
         This method save pod and vm into yaml per namespace
         :param vm_name:
@@ -1605,11 +1677,10 @@ class OC(SSH):
         :param output_dir:
         :return:
         """
+        self.run(f"oc describe vm '{vm_name}' -n '{namespace}' > '{output_dir}/describe-vm-{vm_name}.yaml'")
         if vm_access:
-            self.run(f"oc describe vmi '{vm_name}' -n '{namespace}' > '{output_dir}/describe_vmi_{vm_name}.yaml'")
+            self.run(f"oc describe vmi '{vm_name}' -n '{namespace}' > '{output_dir}/describe-vmi-{vm_name}.yaml'")
             pod_name = self.run(
                 f'oc get pod -n {namespace} -o jsonpath="{{.items[?(@.metadata.generateName==\'virt-launcher-{vm_name}-\')].metadata.name}}"'
             )
-            self.run(f"oc describe pod '{pod_name}' -n '{namespace}' > '{output_dir}/describe_pod_{pod_name}.yaml'")
-        else:
-            self.run(f"oc describe vm '{vm_name}' -n '{namespace}' > '{output_dir}/describe_vm_{vm_name}.yaml'")
+            self.run(f"oc describe pod '{pod_name}' -n '{namespace}' > '{output_dir}/describe-pod-{pod_name}.yaml'")

benchmark_runner/common/ocp_resources/create_cnv.py

@@ -42,18 +42,18 @@ class CreateCNV(CreateOCPResourceOperations):
                     if cnv_nightly_channel:
                         # wait till get the patch
                         self.wait_for_ocp_resource_create(operator=resource,
-                                                          verify_cmd="oc get InstallPlan -n openshift-cnv -o jsonpath={.items[0].metadata.name}",
+                                                          verify_cmd="oc get installplan -n openshift-cnv -o json | jq -r '.items[] | select(.spec.clusterServiceVersionNames[] | startswith(\"kubevirt-hyperconverged-operator\")) | .metadata.name'",
                                                           status="install-")
                         self.apply_patch(namespace='openshift-cnv', operator='cnv')
                     # stable channel
                     else:
                         self.wait_for_ocp_resource_create(operator='cnv',
-                                                          verify_cmd="oc get csv -n openshift-cnv -o jsonpath='{.items[0].status.phase}'",
+                                                          verify_cmd="""oc get csv -n openshift-cnv -o json | jq -r '.items[] | select(.metadata.name | startswith("kubevirt-hyperconverged-operator")) | .status.phase'""",
                                                           status='Succeeded')
                 # for second script wait for refresh status
                 if '02_hyperconverge.yaml' in resource:
                     # Wait that till succeeded
                     self.wait_for_ocp_resource_create(operator='cnv',
-                                                      verify_cmd="oc get csv -n openshift-cnv -o jsonpath='{.items[0].status.phase}'",
+                                                      verify_cmd = """oc get csv -n openshift-cnv -o json | jq -r '.items[] | select(.metadata.name | startswith("kubevirt-hyperconverged-operator")) | .status.phase'""",
                                                       status='Succeeded')
         return True

benchmark_runner/common/ocp_resources/create_lso.py

@@ -10,11 +10,14 @@ class CreateLSO(CreateOCPResourceOperations):
     """
     This class is created Local Storage operator
     """
-    def __init__(self, oc: OC, path: str, resource_list: list):
+    def __init__(self, oc: OC, path: str, resource_list: list, lso_node: str, lso_disk_id: str, ceph_version: str):
         super().__init__(oc)
         self.__oc = oc
         self.__path = path
         self.__resource_list = resource_list
+        self.__lso_node = lso_node
+        self.__lso_disk_id = lso_disk_id
+        self.__ceph_version = ceph_version
 
     @logger_time_stamp
     def create_lso(self, upgrade_version: str):
@@ -29,7 +32,15 @@ class CreateLSO(CreateOCPResourceOperations):
         else:
             for resource in self.__resource_list:
                 logger.info(f'run {resource}')
-                self.__oc.create_async(yaml=os.path.join(self.__path, resource))
+                if resource.endswith('.sh'):
+                    if '01_delete_disks.sh' == resource and self.__lso_disk_id:
+                        disk = f'/dev/disk/by-id/{self.__lso_disk_id}'
+                        delete_node_disk = f""" podman run --authfile /var/lib/kubelet/config.json --rm --privileged --device "$(readlink -e '{disk}')" --entrypoint ceph-bluestore-tool quay.io/ceph/ceph:v{self.__ceph_version} zap-device --dev "$(readlink -e '{disk}')" --yes-i-really-really-mean-it """
+                        logger.info(f'{self.__lso_node}: {delete_node_disk}')
+                        self.__oc.run(cmd=f'chmod +x {os.path.join(self.__path, resource)}; {self.__path}/{resource} "{self.__lso_node}" "{delete_node_disk}"')
+                else:
+                    logger.info(f'run {resource}')
+                    self.__oc.create_async(yaml=os.path.join(self.__path, resource))
 
             # verify once after create all resource files
             self.wait_for_ocp_resource_create(operator='lso',

benchmark_runner/common/ocp_resources/create_nhc_far.py

@@ -0,0 +1,51 @@
+import os
+
+from benchmark_runner.common.oc.oc import OC
+from benchmark_runner.common.logger.logger_time_stamp import logger_time_stamp, logger
+from benchmark_runner.common.ocp_resources.create_ocp_resource_operations import CreateOCPResourceOperations
+
+
+class CreateNHCFAR(CreateOCPResourceOperations):
+    """
+    This class is created node-health-check and fence-agents-remediation operators
+    """
+
+    def __init__(self, oc: OC, path: str, resource_list: list):
+        super().__init__(oc)
+        self.__oc = oc
+        self.__path = path
+        self.__resource_list = resource_list
+
+    @logger_time_stamp
+    def create_nhc_far(self):
+        """
+        This method creates node-health-check and fence-agents-remediation operators
+        :return:
+        """
+        for resource in self.__resource_list:
+            logger.info(f'run {resource}')
+            self.__oc.create_async(yaml=os.path.join(self.__path, resource))
+
+            if '03_nhc_subscription.yaml' in resource:
+                self.wait_for_ocp_resource_create(operator='nhc',
+                                                  verify_cmd=f"oc get csv -n openshift-workload-availability -o json | jq -r '.items[] | select(.metadata.name | startswith(\"node-healthcheck-operator\")) | .status.phase'",
+                                                  status='Succeeded')
+            if '04_far_subscription.yaml' in resource:
+                self.wait_for_ocp_resource_create(operator='far',
+                                                  verify_cmd=f"oc get csv -n openshift-workload-availability -o json | jq -r '.items[] | select(.metadata.name | startswith(\"fence-agents-remediation\")) | .status.phase'",
+                                                  status='Succeeded')
+            if '05_nhc_far.yaml' in resource:
+                # Wait for NHC to be enabled using a retries mechanism
+                for _ in range(self.__oc.RETRIES):
+                    try:
+                        self.wait_for_ocp_resource_create(
+                            operator='nhc_far',
+                            verify_cmd="oc get nhc -A -o jsonpath='{range .items[*]}{.status.phase}{\"\\n\"}{end}'",
+                            status='Enabled',
+                            timeout=600
+                        )
+                        break
+                    except:
+                        # rerun latest resource creation, if nhc is not enabled
+                        self.__oc.create_async(yaml=os.path.join(self.__path, resource))
+        return True

benchmark_runner/common/ocp_resources/create_ocp_resource.py

@@ -8,7 +8,7 @@ from benchmark_runner.common.ocp_resources.create_lso import CreateLSO
 from benchmark_runner.common.ocp_resources.create_odf import CreateODF
 from benchmark_runner.common.ocp_resources.create_kata import CreateKata
 from benchmark_runner.common.ocp_resources.create_cnv import CreateCNV
-from benchmark_runner.common.ocp_resources.create_nhc_snr import CreateNHCSNR
+from benchmark_runner.common.ocp_resources.create_nhc_far import CreateNHCFAR
 from benchmark_runner.common.ocp_resources.create_custom import CreateCustom
 from benchmark_runner.common.ocp_resources.migrate_infra import MigrateInfra
 
@@ -24,6 +24,10 @@ class CreateOCPResource:
         self.__oc.populate_additional_template_variables(self.__environment_variables_dict)
         self.__worker_disk_prefix = self.__environment_variables_dict.get('worker_disk_prefix', '')
         self.__worker_disk_ids = self.__environment_variables_dict.get('worker_disk_ids', '')
+        self.__worker_disk_ids = self.__environment_variables_dict.get('worker_disk_ids', '')
+        self.__lso_disk_id = self.__environment_variables_dict.get('lso_disk_id', '')
+        self.__lso_node = self.__environment_variables_dict.get('lso_node', '')
+        self.__ceph_version = self.__environment_variables_dict.get('ceph_version', '')
         if self.__worker_disk_ids:
             if self.__worker_disk_ids:
                 # Solved GitHub Actions issue that env variable detect as string instead of dict/ list -skip for Jenkins
@@ -80,7 +84,7 @@ class CreateOCPResource:
         self.remove_resource_files(path=os.path.join(self.__dir_path, resource))
         resource_files = self.get_sorted_resources(resource=resource)
         if 'lso' == resource:
-            create_lso = CreateLSO(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files)
+            create_lso = CreateLSO(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files, lso_node=self.__lso_node, lso_disk_id=self.__lso_disk_id, ceph_version=self.__ceph_version)
             create_lso.create_lso(upgrade_version)
         elif 'odf' == resource:
             create_odf = CreateODF(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files, worker_disk_ids=self.__worker_disk_ids, worker_disk_prefix=self.__worker_disk_prefix)
@@ -91,9 +95,9 @@ class CreateOCPResource:
         elif 'cnv' == resource:
             create_cnv = CreateCNV(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files)
             create_cnv.create_cnv()
-        elif 'nhc_snr' == resource:
-            create_nhc_snr = CreateNHCSNR(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files)
-            create_nhc_snr.create_nhc_snr()
+        elif 'nhc_far' == resource:
+            create_nhc_far = CreateNHCFAR(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files)
+            create_nhc_far.create_nhc_far()
         elif 'infra' == resource:
             create_infra = MigrateInfra(self.__oc, path=os.path.join(self.__dir_path, resource), resource_list=resource_files)
             create_infra.migrate_infra()

benchmark_runner/common/ocp_resources/lso/template/01_delete_disks_template.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+while (($# >= 2)); do
+    node="$1"
+    delete_disk_command="$2"
+
+    echo "Executing deletion on node: $node"
+
+    oc debug "node/$node" -- chroot /host sh -c "$delete_disk_command"
+
+    shift 2
+done

benchmark_runner/common/ocp_resources/lso/template/04_subscription_template.yaml

@@ -7,6 +7,10 @@ spec:
   channel: "stable"
   installPlanApproval: Automatic
   name: local-storage-operator
+  {%- if upgrade_ocp_version %}
+  source: redhat-operators
+  {%- else %}
   source: redhat-operators-v{{ lso_version | replace('.', '') }}   # <-- Modify the name of the redhat-operators catalogsource if not default
+  {%- endif %}
   sourceNamespace: openshift-marketplace
 

benchmark_runner/common/ocp_resources/nhc_far/template/04_far_subscription_template.yaml

@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+    name: fence-agents-remediation-subscription
+    namespace: openshift-workload-availability
+spec:
+    channel: stable
+    name: fence-agents-remediation
+    source: redhat-operators
+    sourceNamespace: openshift-marketplace
+    package: fence-agents-remediation