sbcli-pre 25.6.0__tar.gz → 25.6.2__tar.gz

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 25.6.0
+Version: 25.6.2
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 25.6.0
+Version: 25.6.2
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/controllers/health_controller.py

@@ -2,7 +2,7 @@
 import time
 
 from typing import Any
-from logging import DEBUG, ERROR
+from logging import DEBUG, ERROR, INFO
 
 import jc
 
@@ -22,7 +22,7 @@ def check_bdev(name, *, rpc_client=None, bdev_names=None):
             ((bdev_names is not None) and (name in bdev_names)) or
             (rpc_client is not None and (rpc_client.get_bdevs(name) is not None))
     )
-    logger.log(DEBUG if present else ERROR, f"Checking bdev: {name} ... " + ('ok' if present else 'failed'))
+    logger.log(INFO if present else ERROR, f"Checking bdev: {name} ... " + ('ok' if present else 'failed'))
     return present
 
 
@@ -38,9 +38,7 @@ def check_subsystem(nqn, *, rpc_client=None, nqns=None, ns_uuid=None):
         logger.error(f"Checking subsystem {nqn} ... not found")
         return False
 
-    logger.debug(f"Checking subsystem {nqn} ... ok")
-
-    listeners = len(subsystem['listen_addresses'])
+    logger.info(f"Checking subsystem {nqn} ... ok")
 
     if ns_uuid:
         for ns in subsystem['namespaces']:
@@ -51,9 +49,15 @@ def check_subsystem(nqn, *, rpc_client=None, nqns=None, ns_uuid=None):
             namespaces = 0
     else:
         namespaces = len(subsystem['namespaces'])
+    logger.log(INFO if namespaces else ERROR, f"Checking namespaces: {namespaces} ... " + ('ok' if namespaces else 'not found'))
+
+    listeners = subsystem['listen_addresses']
+    if not listeners:
+        logger.error(f"Checking listener for {nqn} ... not found")
+    else:
+        for listener in listeners:
+            logger.info(f"Checking listener {listener['traddr']}:{listener['trsvcid']} ... ok")
 
-    logger.log(DEBUG if listeners else ERROR, f"Checking listener: {listeners} ... " + ('ok' if listeners else 'not found'))
-    logger.log(DEBUG if namespaces else ERROR, f"Checking namespaces: {namespaces} ... " + ('ok' if namespaces else 'not found'))
     return listeners and namespaces
 
 
@@ -224,8 +228,7 @@ def _check_node_hublvol(node: StorageNode, node_bdev_names=None, node_lvols_nqns
     return passed
 
 
-
-def _check_sec_node_hublvol(node: StorageNode, node_bdev=None, node_lvols_nqns=None):
+def _check_sec_node_hublvol(node: StorageNode, node_bdev=None, node_lvols_nqns=None, auto_fix=False):
     db_controller = DBController()
     primary_node = db_controller.get_storage_node_by_id(node.lvstore_stack_secondary_1)
     if primary_node is None:
@@ -257,11 +260,18 @@ def _check_sec_node_hublvol(node: StorageNode, node_bdev=None, node_lvols_nqns=N
 
 
         ret = rpc_client.bdev_nvme_controller_list(primary_node.hublvol.bdev_name)
-        if ret:
-            logger.info(f"Checking controller: {primary_node.hublvol.bdev_name} ... ok")
-        else:
-            logger.info(f"Checking controller: {primary_node.hublvol.bdev_name} ... failed")
-            passed = False
+        passed = bool(ret)
+        logger.info(f"Checking controller: {primary_node.hublvol.bdev_name} ... {passed}")
+
+        if not passed and auto_fix and primary_node.lvstore_status == "ready" \
+                and primary_node.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN]:
+            try:
+                node.connect_to_hublvol(primary_node)
+            except Exception as e:
+                logger.error("Error establishing hublvol: %s", e)
+            ret = rpc_client.bdev_nvme_controller_list(primary_node.hublvol.bdev_name)
+            passed = bool(ret)
+            logger.info(f"Checking controller: {primary_node.hublvol.bdev_name} ... {passed}")
 
         passed &= check_bdev(primary_node.hublvol.get_remote_bdev_name(), bdev_names=node_bdev)
         cl = db_controller.get_cluster_by_id(node.cluster_id)
@@ -467,6 +477,7 @@ def check_node(node_id, with_devices=True):
         logger.info("Skipping devices checks because RPC check failed")
     else:
         logger.info(f"Node device count: {len(snode.nvme_devices)}")
+        print("*" * 100)
         for dev in snode.nvme_devices:
             if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]:
                 ret = check_device(dev.get_id())
@@ -476,15 +487,16 @@ def check_node(node_id, with_devices=True):
             print("*" * 100)
 
         logger.info(f"Node remote device: {len(snode.remote_devices)}")
+        print("*" * 100)
         rpc_client = RPCClient(
             snode.mgmt_ip, snode.rpc_port,
             snode.rpc_username, snode.rpc_password,
             timeout=5, retry=1)
         for remote_device in snode.remote_devices:
-            node_remote_devices_check &= check_bdev(remote_device.remote_bdev, rpc_client=rpc_client)
+            node_remote_devices_check &= check_remote_device(remote_device.get_id(), snode)
+            print("*" * 100)
 
         if snode.jm_device:
-            print("*" * 100)
             jm_device = snode.jm_device
             logger.info(f"Node JM: {jm_device.get_id()}")
             ret = check_jm_device(jm_device.get_id())
@@ -498,7 +510,27 @@ def check_node(node_id, with_devices=True):
             print("*" * 100)
             logger.info(f"Node remote JMs: {len(snode.remote_jm_devices)}")
             for remote_device in snode.remote_jm_devices:
-                node_remote_devices_check &= check_bdev(remote_device.remote_bdev, rpc_client=rpc_client)
+
+                name = f'remote_{remote_device.jm_bdev}n1'
+                bdev_info = rpc_client.get_bdevs(name)
+                logger.log(INFO if bdev_info else ERROR,
+                           f"Checking bdev: {name} ... " + ('ok' if bdev_info else 'failed'))
+                node_remote_devices_check &= bool(bdev_info)
+
+                controller_info = rpc_client.bdev_nvme_controller_list(f'remote_{remote_device.jm_bdev}')
+                if controller_info:
+                    addr = controller_info[0]['ctrlrs'][0]['trid']['traddr']
+                    port = controller_info[0]['ctrlrs'][0]['trid']['trsvcid']
+                    logger.info(f"IP Address: {addr}:{port}")
+
+                if remote_device.nvmf_multipath:
+                    if controller_info and "alternate_trids" in controller_info[0]['ctrlrs'][0]:
+                        addr = controller_info[0]['ctrlrs'][0]['alternate_trids'][0]['traddr']
+                        port = controller_info[0]['ctrlrs'][0]['alternate_trids'][0]['trsvcid']
+                        logger.info(f"IP Address: {addr}:{port}")
+
+                    if bdev_info:
+                        logger.info(f"multipath policy: {bdev_info[0]['driver_specific']['mp_policy']}")
 
         print("*" * 100)
         if snode.lvstore_stack:
@@ -570,12 +602,11 @@ def check_device(device_id):
 
         passed &= check_subsystem(device.nvmf_nqn, rpc_client=rpc_client)
 
-        if device.status == NVMeDevice.STATUS_ONLINE:
-            logger.info("Checking other node's connection to this device...")
-            ret = check_remote_device(device_id)
-            if not ret:
-                logger.warning(f"Remote device {device_id} is not accessible from other nodes")
-            # passed &= ret
+        # if device.status == NVMeDevice.STATUS_ONLINE:
+        #     logger.info("Checking other node's connection to this device...")
+        #     ret = check_remote_device(device_id)
+        # if not ret:
+        #         logger.warning(f"Remote device {device_id} is not accessible from other nodes")#     # passed &= ret
 
     except Exception as e:
         logger.error(f"Failed to connect to node's SPDK: {e}")
@@ -584,7 +615,7 @@ def check_device(device_id):
     return passed
 
 
-def check_remote_device(device_id):
+def check_remote_device(device_id, target_node=None):
     db_controller = DBController()
     device = db_controller.get_storage_device_by_id(device_id)
     if not device:
@@ -596,13 +627,34 @@ def check_remote_device(device_id):
         return False
 
     result = True
-    for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
+    if target_node:
+        nodes = [target_node]
+    else:
+        nodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    for node in nodes:
         if node.status == StorageNode.STATUS_ONLINE:
             if node.get_id() == snode.get_id():
                 continue
-            logger.info(f"Connecting to node: {node.get_id()}")
+            logger.info(f"Checking device: {device_id}")
             rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
-            result &= check_bdev(f'remote_{device.alceml_bdev}n1', rpc_client=rpc_client)
+            name = f'remote_{device.alceml_bdev}n1'
+            bdev_info = rpc_client.get_bdevs(name)
+            logger.log(DEBUG if bdev_info else ERROR, f"Checking bdev: {name} ... " + ('ok' if bdev_info else 'failed'))
+            result &= bool(bdev_info)
+            controller_info = rpc_client.bdev_nvme_controller_list(f'remote_{device.alceml_bdev}')
+            if controller_info:
+                addr = controller_info[0]['ctrlrs'][0]['trid']['traddr']
+                port = controller_info[0]['ctrlrs'][0]['trid']['trsvcid']
+                logger.info(f"IP Address: {addr}:{port}")
+
+            if device.nvmf_multipath:
+                if controller_info and "alternate_trids" in controller_info[0]['ctrlrs'][0]:
+                    addr = controller_info[0]['ctrlrs'][0]['alternate_trids'][0]['traddr']
+                    port = controller_info[0]['ctrlrs'][0]['alternate_trids'][0]['trsvcid']
+                    logger.info(f"IP Address: {addr}:{port}")
+
+                if bdev_info:
+                    logger.info(f"multipath policy: {bdev_info[0]['driver_specific']['mp_policy']}")
 
     return result
 
@@ -725,6 +777,8 @@ def check_jm_device(device_id):
             snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
 
         passed &= check_bdev(jm_device.jm_bdev, rpc_client=rpc_client)
+        if snode.enable_ha_jm:
+            passed &= check_subsystem(jm_device.nvmf_nqn, rpc_client=rpc_client)
 
     except Exception as e:
         logger.error(f"Failed to connect to node's SPDK: {e}")

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=25.6.0
+SIMPLY_BLOCK_VERSION=25.6.2
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:R25.6-PRE

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/models/nvme_device.py

@@ -46,6 +46,7 @@ class NVMeDevice(BaseModel):
     nvmf_ip: str = ""
     nvmf_nqn: str = ""
     nvmf_port: int = 0
+    nvmf_multipath: bool = False
     overload_percentage: int = 0  # Unused
     partition_jm_bdev: str = ""  # Unused
     partition_jm_size: int = 0  # Unused

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/models/storage_node.py

@@ -241,10 +241,7 @@ class StorageNode(BaseNodeObject):
                 ret = rpc_client.bdev_nvme_attach_controller_tcp(
                         primary_node.hublvol.bdev_name, primary_node.hublvol.nqn,
                         ip, primary_node.hublvol.nvmf_port)
-                if ret:
-                    remote_bdev = ret[0]
-                    break
-                else:
+                if not ret:
                     logger.warning(f'Failed to connect to hublvol on {ip}')
 
         if not rpc_client.bdev_lvol_set_lvs_opts(

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/rpc_client.py

@@ -562,7 +562,7 @@ class RPCClient:
         # ultra/DISTR_v2/src_code_app_spdk/specs/message_format_rpcs__distrib__v5.txt#L396C1-L396C27
         return self._request("distr_status_events_update", params)
 
-    def bdev_nvme_attach_controller_tcp(self, name, nqn, ip, port):
+    def bdev_nvme_attach_controller_tcp(self, name, nqn, ip, port, multipath=False):
         params = {
             "name": name,
             "trtype": "tcp",
@@ -571,11 +571,12 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "num_io_queues": 128,
-            #"ctrlr_loss_timeout_sec": 3,
-            "multipath":"disable",
-            # "reconnect_delay_sec":1
+            "num_io_queues": 128
         }
+        if multipath:
+            params["multipath"] = "failover"
+        else:
+            params["multipath"] = "disable"
         return self._request("bdev_nvme_attach_controller", params)
 
     def bdev_nvme_attach_controller_tcp_caching(self, name, nqn, ip, port):
@@ -632,7 +633,7 @@ class RPCClient:
     def bdev_nvme_set_options(self):
         params = {
             # "action_on_timeout": "abort",
-            "bdev_retry_count": 0,
+            "bdev_retry_count": 1,
             "transport_retry_count": 3,
             "ctrlr_loss_timeout_sec": 1,
             "fast_io_fail_timeout_sec" : 0,
@@ -1071,3 +1072,10 @@ class RPCClient:
         if ana_state:
             params["ana_state"] = ana_state
         return self._request2("nvmf_subsystem_add_listener", params)
+
+    def bdev_nvme_set_multipath_policy(self, name, policy):  # policy: active_active or active_passive
+        params = {
+            "name": name,
+            "policy": policy,
+        }
+        return self._request("bdev_nvme_set_multipath_policy", params)

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/services/health_check_service.py

@@ -225,7 +225,7 @@ while True:
                         if second_node_1 and second_node_1.status == StorageNode.STATUS_ONLINE:
                             lvstore_check &= health_controller._check_node_lvstore(
                                 lvstore_stack, second_node_1, auto_fix=True, stack_src_node=snode)
-                            lvstore_check &= health_controller._check_sec_node_hublvol(second_node_1)
+                            lvstore_check &= health_controller._check_sec_node_hublvol(second_node_1, auto_fix=True)
 
                     lvol_port_check = False
                     # if node_api_check:

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/services/storage_node_monitor.py

@@ -301,12 +301,14 @@ while True:
                     logger.info(f"Check: node port {snode.mgmt_ip}, {port} ... {ret}")
                     node_port_check &= ret
 
+                node_data_nic_ping_check = False
                 for data_nic in snode.data_nics:
                     if data_nic.ip4_address:
                         data_ping_check = health_controller._check_node_ping(data_nic.ip4_address)
                         logger.info(f"Check: ping data nic {data_nic.ip4_address} ... {data_ping_check}")
-                        if not data_ping_check:
-                            node_port_check = False
+                        node_data_nic_ping_check |= data_ping_check
+
+                node_port_check &= node_data_nic_ping_check
 
             cluster = db.get_cluster_by_id(cluster.get_id())
 

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_core/storage_node_ops.py

@@ -112,7 +112,7 @@ def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
 
     pt_name = ""
     subsystem_nqn = ""
-    IP = ""
+    ip_list = []
     if snode.enable_ha_jm:
         # add pass through
         pt_name = f"{jm_bdev}_PT"
@@ -125,19 +125,24 @@ def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
         logger.info("creating subsystem %s", subsystem_nqn)
         ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
         logger.info(f"add {pt_name} to subsystem")
-        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, alceml_id)
         if not ret:
             logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
             return False
 
-        IP = None
         for iface in snode.data_nics:
             if iface.ip4_address:
                 tr_type = iface.get_transport_type()
                 logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
                 ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, snode.nvmf_port)
-                IP = iface.ip4_address
-                break
+                ip_list.append(iface.ip4_address)
+
+    if len(ip_list) > 1:
+        IP = ",".join(ip_list)
+        multipath = True
+    else:
+        IP = ip_list[0]
+        multipath = False
 
     ret = rpc_client.get_bdevs(raid_bdev)
 
@@ -155,6 +160,7 @@ def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
         'nvmf_nqn': subsystem_nqn,
         'nvmf_ip': IP,
         'nvmf_port': snode.nvmf_port,
+        'nvmf_multipath': multipath,
     })
 
 
@@ -192,7 +198,7 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
 
     pt_name = ""
     subsystem_nqn = ""
-    IP = ""
+    ip_list = []
     if snode.enable_ha_jm:
         # add pass through
         pt_name = f"{jm_bdev}_PT"
@@ -205,19 +211,24 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
         logger.info("creating subsystem %s", subsystem_nqn)
         ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
         logger.info(f"add {pt_name} to subsystem")
-        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, alceml_id)
         if not ret:
             logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
             return False
 
-        IP = None
         for iface in snode.data_nics:
             if iface.ip4_address:
                 tr_type = iface.get_transport_type()
                 logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
                 ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, snode.nvmf_port)
-                IP = iface.ip4_address
-                break
+                ip_list.append(iface.ip4_address)
+
+    if len(ip_list) > 1:
+        IP = ",".join(ip_list)
+        multipath = True
+    else:
+        IP = ip_list[0]
+        multipath = False
 
     return JMDevice({
         'uuid': alceml_id,
@@ -235,6 +246,7 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
         'nvmf_nqn': subsystem_nqn,
         'nvmf_ip': IP,
         'nvmf_port': snode.nvmf_port,
+        'nvmf_multipath': multipath,
     })
 
 
@@ -286,21 +298,27 @@ def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
     subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
     logger.info("creating subsystem %s", subsystem_nqn)
     ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
-    IP = None
+    ip_list = []
     for iface in snode.data_nics:
         if iface.ip4_address:
             tr_type = iface.get_transport_type()
             logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
             ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, snode.nvmf_port)
-            IP = iface.ip4_address
-            break
+            ip_list.append(iface.ip4_address)
+
     logger.info(f"add {pt_name} to subsystem")
-    ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+    ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, alceml_id)
     if not ret:
         logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
         return None
-    # if snode.enable_test_device:
-    #     nvme.testing_bdev = test_name
+
+    if len(ip_list) > 1:
+        IP = ",".join(ip_list)
+        multipath = True
+    else:
+        IP = ip_list[0]
+        multipath = False
+
     nvme.alceml_bdev = alceml_bdev
     nvme.pt_bdev = pt_name
     nvme.qos_bdev = qos_bdev
@@ -309,6 +327,7 @@ def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
     nvme.nvmf_ip = IP
     nvme.nvmf_port = snode.nvmf_port
     nvme.io_error = False
+    nvme.nvmf_multipath = multipath
     # if nvme.status != NVMeDevice.STATUS_NEW:
     #     nvme.status = NVMeDevice.STATUS_ONLINE
     return nvme
@@ -560,7 +579,7 @@ def _prepare_cluster_devices_on_restart(snode, clear_data=False):
             logger.info("creating subsystem %s", subsystem_nqn)
             ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
             logger.info(f"add {pt_name} to subsystem")
-            ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+            ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, snode.get_id())
             if not ret:
                 logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
                 return False
@@ -570,7 +589,6 @@ def _prepare_cluster_devices_on_restart(snode, clear_data=False):
                     tr_type = iface.get_transport_type()
                     logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
                     ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, snode.nvmf_port)
-                    break
 
     return True
 
@@ -621,16 +639,23 @@ def _connect_to_remote_devs(this_node, force_conect_restarting_nodes=False):
                 #     logger.info(f"detaching {name} from {this_node.get_id()}")
                 #     rpc_client.bdev_nvme_detach_controller(name)
                 #     time.sleep(1)
-
-                logger.info(f"Connecting {name} to {this_node.get_id()}")
-                rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
+                if dev.nvmf_multipath:
+                    for ip in dev.nvmf_ip.split(","):
+                        logger.info(f"Connecting {name} to {this_node.get_id()}")
+                        rpc_client.bdev_nvme_attach_controller_tcp(
+                            name, dev.nvmf_nqn, ip, dev.nvmf_port, multipath=True)
+                        rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_active")
+                else:
+                    logger.info(f"Connecting {name} to {this_node.get_id()}")
+                    rpc_client.bdev_nvme_attach_controller_tcp(
+                        name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port, multipath=False)
                 ret = rpc_client.get_bdevs(bdev_name)
                 if not ret:
                     logger.error(f"Failed to connect to device: {dev.get_id()}")
                     continue
             dev.remote_bdev = bdev_name
             remote_devices.append(dev)
-            # distr_controller.send_dev_status_event(dev, dev.status, this_node)
+
     return remote_devices
 
 
@@ -700,14 +725,27 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=None):
                     rpc_client.bdev_nvme_detach_controller(name)
                     time.sleep(1)
 
-                logger.info(f"Connecting {name} to {this_node.get_id()}")
-                ret = rpc_client.bdev_nvme_attach_controller_tcp(
-                    name, org_dev.nvmf_nqn, org_dev.nvmf_ip, org_dev.nvmf_port)
-                if ret:
+                if org_dev.nvmf_multipath:
+                    for ip in org_dev.nvmf_ip.split(","):
+                        logger.info(f"Connecting {name} to {this_node.get_id()}")
+                        ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                            name, org_dev.nvmf_nqn, ip, org_dev.nvmf_port, multipath=True)
+                        rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_active")
+
+                    # if ret:
                     org_dev.status = JMDevice.STATUS_ONLINE
+                    # else:
+                    #     logger.error(f"failed to connect to remote JM {name}")
+                    #     org_dev.status = JMDevice.STATUS_UNAVAILABLE
                 else:
-                    logger.error(f"failed to connect to remote JM {name}")
-                    org_dev.status = JMDevice.STATUS_UNAVAILABLE
+                    logger.info(f"Connecting {name} to {this_node.get_id()}")
+                    ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                        name, org_dev.nvmf_nqn, org_dev.nvmf_ip, org_dev.nvmf_port, multipath=False)
+                    if ret:
+                        org_dev.status = JMDevice.STATUS_ONLINE
+                    else:
+                        logger.error(f"failed to connect to remote JM {name}")
+                        org_dev.status = JMDevice.STATUS_UNAVAILABLE
                 new_devs.append(org_dev)
             else:
                 org_dev.status = JMDevice.STATUS_UNAVAILABLE

{sbcli-pre-25.6.0 → sbcli-pre-25.6.2}/simplyblock_web/blueprints/web_api_metrics.py

@@ -169,32 +169,32 @@ def get_data():
                         ng[g].labels(cluster=cl.get_id(), snode=node.get_id()).set(node.get_status_code())
                     elif v == "health_check":
                         ng[g].labels(cluster=cl.get_id(), snode=node.get_id()).set(node.health_check)
-
-                    for reactor in reactor_data.get("reactors", []):
-                        lcore = reactor.get("lcore")
-                        core_idle = reactor.get("idle", 0)
-                        core_busy = reactor.get("busy", 0)
-                        irq = reactor.get("irq", 0)
-                        sys = reactor.get("sys", 0)
-
-                        thread_names = ", ".join(thread["name"] for thread in reactor.get("lw_threads", []))
-                        if v == "cpu_busy_percentage":
-                            for thread in reactor.get("lw_threads", []):
-                                thread_name = thread.get("name")
-                                thread_id = thread.get("id")
-                                thread_busy = thread_busy_map.get(thread_id, 0)
-
-                                total_core_cycles = core_busy + core_idle
-                                cpu_usage_percent = (thread_busy / total_core_cycles) * 100 if total_core_cycles > 0 else 0
-
-                                ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), thread_name=thread_name).set(cpu_usage_percent)
-
-                        elif v == "cpu_core_utilization":
-
-                            total_cycle = core_busy + irq + sys
-                            total_with_idle = total_cycle + core_idle
-                            core_utilization = (total_cycle / total_with_idle) * 100 if total_with_idle > 0 else 0
-                            ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), core_id=str(lcore), thread_names=thread_names).set(core_utilization)
+                    if reactor_data and "reactors" in reactor_data:
+                        for reactor in reactor_data.get("reactors", []):
+                            lcore = reactor.get("lcore")
+                            core_idle = reactor.get("idle", 0)
+                            core_busy = reactor.get("busy", 0)
+                            irq = reactor.get("irq", 0)
+                            sys = reactor.get("sys", 0)
+
+                            thread_names = ", ".join(thread["name"] for thread in reactor.get("lw_threads", []))
+                            if v == "cpu_busy_percentage":
+                                for thread in reactor.get("lw_threads", []):
+                                    thread_name = thread.get("name")
+                                    thread_id = thread.get("id")
+                                    thread_busy = thread_busy_map.get(thread_id, 0)
+
+                                    total_core_cycles = core_busy + core_idle
+                                    cpu_usage_percent = (thread_busy / total_core_cycles) * 100 if total_core_cycles > 0 else 0
+
+                                    ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), thread_name=thread_name).set(cpu_usage_percent)
+
+                            elif v == "cpu_core_utilization":
+
+                                total_cycle = core_busy + irq + sys
+                                total_with_idle = total_cycle + core_idle
+                                core_utilization = (total_cycle / total_with_idle) * 100 if total_with_idle > 0 else 0
+                                ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), core_id=str(lcore), thread_names=thread_names).set(core_utilization)
 
 
             for device in node.nvme_devices: