sbcli-dev 3.8.65__zip → 3.8.67__zip

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.65
+Version: 3.8.67
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.65
+SIMPLY_BLOCK_VERSION=3.8.67
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.65
+Version: 3.8.67
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/simplyblock_cli/cli.py

@@ -230,6 +230,14 @@ class CLIWrapper:
         sub_command = self.add_sub_command(subparser, "info-spdk", 'Get SPDK memory information')
         sub_command.add_argument("id", help='Node UUID')
 
+        sub_command = self.add_sub_command(subparser, 'remove-jm-device', 'Remove JM device')
+        sub_command.add_argument("jm_device_id", help='JM device ID')
+        sub_command.add_argument("--force", help='Force device remove', required=False, action='store_true')
+
+        sub_command = self.add_sub_command(subparser, 'restart-jm-device', 'Restart JM device')
+        sub_command.add_argument("jm_device_id", help='JM device ID')
+        sub_command.add_argument("--force", help='Force device remove', required=False, action='store_true')
+
         #
         # ----------------- cluster -----------------
         #
@@ -879,6 +887,12 @@ class CLIWrapper:
             elif sub_command == "get":
                 ret = storage_ops.get(args.id)
 
+            elif sub_command == "remove-jm-device":
+                ret = device_controller.remove_jm_device(args.jm_device_id, args.force)
+
+            elif sub_command == "restart-jm-device":
+                ret = device_controller.restart_jm_device(args.jm_device_id, args.force)
+
             else:
                 self.parser.print_help()
 

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/simplyblock_core/controllers/device_controller.py

@@ -258,43 +258,13 @@ def restart_device(device_id, force=False):
             if part.startswith(nvme_controller):
                 jm_part = part
                 break
-        if snode.jm_device.status == NVMeDevice.STATUS_ONLINE:
-            set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
-            # delete jm stack
-            if snode.enable_ha_jm:
-                ret = rpc_client.subsystem_delete(snode.jm_device.nvmf_nqn)
-
-            if snode.jm_device.pt_bdev:
-                ret = rpc_client.bdev_PT_NoExcl_delete(snode.jm_device.pt_bdev)
-
-            ret = rpc_client.bdev_jm_delete(snode.jm_device.jm_bdev)
-            ret = rpc_client.bdev_alceml_delete(snode.jm_device.alceml_bdev)
-            if snode.jm_device.testing_bdev:
-                ret = rpc_client.bdev_passtest_delete(snode.jm_device.testing_bdev)
-            if len(snode.jm_device.jm_nvme_bdev_list) == 2:
-                ret = rpc_client.bdev_raid_delete(snode.jm_device.raid_bdev)
-
-        # create jm
-        jm_nvme_bdevs = snode.jm_device.jm_nvme_bdev_list
-        if jm_part not in jm_nvme_bdevs:
-            jm_nvme_bdevs.append(jm_part)
-        new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
-        if new_jm:
-            snode = db_controller.get_storage_node_by_id(snode.get_id())
-            snode.jm_device = new_jm
-            snode.write_to_db(db_controller.kv_store)
-            set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
-
-        # make other nodes connect to new jm
-        if snode.enable_ha_jm:
-            logger.info("Make other nodes connect to the new devices")
-            snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
-            for node_index, node in enumerate(snodes):
-                if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
-                    continue
-                node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
-                node.write_to_db(db_controller.kv_store)
-                logger.info(f"connected to devices count: {len(node.remote_devices)}")
+
+        if not jm_part:
+            if snode.jm_device.status == NVMeDevice.STATUS_ONLINE:
+                remove_jm_device(snode.jm_device.get_id(), force=True)
+                time.sleep(3)
+
+            restart_jm_device(snode.jm_device.get_id(), force=True)
 
     return "Done"
 
@@ -373,8 +343,8 @@ def device_remove(device_id, force=True):
         if force is False:
             return False
 
-    logger.info("Sending device event")
-    distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_REMOVED)
+    logger.info("Setting device unavailable")
+    device_set_unavailable(device_id)
 
     logger.info("Disconnecting device from all nodes")
     distr_controller.disconnect_device(device)
@@ -408,12 +378,7 @@ def device_remove(device_id, force=True):
             if not force:
                 return False
 
-    device.status = 'removed'
-    snode.write_to_db(db_controller.kv_store)
-    device_events.device_delete(device)
-
-    for lvol in db_controller.get_lvols():
-        lvol_controller.send_cluster_map(lvol.get_id())
+    device_set_state(device_id, NVMeDevice.STATUS_REMOVED)
 
     # remove device from jm raid
     if snode.jm_device.raid_bdev:
@@ -425,53 +390,11 @@ def device_remove(device_id, force=True):
                 break
 
         if dev_to_remove:
-            if len(snode.jm_device.jm_nvme_bdev_list) <= 2:
-
-                set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
-
-                # delete jm stack
-                if snode.enable_ha_jm:
-                    ret = rpc_client.subsystem_delete(snode.jm_device.nvmf_nqn)
-
-                if snode.jm_device.pt_bdev:
-                    ret = rpc_client.bdev_PT_NoExcl_delete(snode.jm_device.pt_bdev)
-
-                ret = rpc_client.bdev_jm_delete(snode.jm_device.jm_bdev)
-                ret = rpc_client.bdev_alceml_delete(snode.jm_device.alceml_bdev)
-                if snode.jm_device.testing_bdev:
-                    ret = rpc_client.bdev_passtest_delete(snode.jm_device.testing_bdev)
-                if len(snode.jm_device.jm_nvme_bdev_list) == 2:
-                    ret = rpc_client.bdev_raid_delete(snode.jm_device.raid_bdev)
-
-
-                # create jm
-                jm_nvme_bdevs = snode.jm_device.jm_nvme_bdev_list
-                jm_nvme_bdevs.remove(dev_to_remove)
-                if len(jm_nvme_bdevs) > 0:
-                    new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
-                    if new_jm:
-                        snode = db_controller.get_storage_node_by_id(snode.get_id())
-                        snode.jm_device = new_jm
-                        snode.write_to_db(db_controller.kv_store)
-                        set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
-
-                    # make other nodes connect to new jm
-                    if snode.enable_ha_jm:
-                        logger.info("Make other nodes connect to the new devices")
-                        snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
-                        for node_index, node in enumerate(snodes):
-                            if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
-                                continue
-                            node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
-                            node.write_to_db(db_controller.kv_store)
-                            logger.info(f"connected to devices count: {len(node.remote_devices)}")
-                            time.sleep(3)
-
-            elif len(snode.jm_device.jm_nvme_bdev_list) > 2:
-                ret = rpc_client.bdev_raid_remove_base_bdev(snode.jm_device.raid_bdev, dev_to_remove)
-                if ret:
-                    snode.jm_device.jm_nvme_bdev_list.remove(dev_to_remove)
-                    snode.write_to_db(db_controller.kv_store)
+            if snode.jm_device.status == NVMeDevice.STATUS_ONLINE:
+                remove_jm_device(snode.jm_device.get_id(), force=True)
+                time.sleep(3)
+
+            restart_jm_device(snode.jm_device.get_id(), force=True)
 
     return True
 
@@ -795,9 +718,109 @@ def set_jm_device_state(device_id, state):
         logger.error("device not found")
         return False
 
-    if jm_device.status == state:
-        return True
+    if jm_device.status != state:
+        jm_device.status = state
+        snode.write_to_db(db_controller.kv_store)
+
+    if snode.enable_ha_jm and state in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]:
+        # make other nodes connect to the new devices
+        snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+        for node_index, node in enumerate(snodes):
+            if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+                continue
+            logger.info(f"Connecting to node: {node.get_id()}")
+            node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
+            node.write_to_db(db_controller.kv_store)
+            logger.info(f"connected to devices count: {len(node.remote_jm_devices)}")
+
+    return True
+
+
+def remove_jm_device(device_id, force=False):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if jm_device.status != JMDevice.STATUS_ONLINE:
+        logger.warning("device is not online")
+        if not force:
+            return False
+
+    set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+
+    # delete jm stack
+    if snode.enable_ha_jm:
+        ret = rpc_client.subsystem_delete(snode.jm_device.nvmf_nqn)
+        if not ret:
+            logger.error("device not found")
+
+    if snode.jm_device.pt_bdev:
+        ret = rpc_client.bdev_PT_NoExcl_delete(snode.jm_device.pt_bdev)
+
+    ret = rpc_client.bdev_jm_delete(snode.jm_device.jm_bdev)
+
+    ret = rpc_client.bdev_alceml_delete(snode.jm_device.alceml_bdev)
+
+    if snode.jm_device.testing_bdev:
+        ret = rpc_client.bdev_passtest_delete(snode.jm_device.testing_bdev)
+
+    if len(snode.jm_device.jm_nvme_bdev_list) == 2:
+        ret = rpc_client.bdev_raid_delete(snode.jm_device.raid_bdev)
+
+    return True
+
+
+def restart_jm_device(device_id, force=False):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if jm_device.status == JMDevice.STATUS_ONLINE:
+        logger.warning("device is online")
+        if not force:
+            return False
+
+    # add to jm raid
+    if snode.jm_device and snode.jm_device.raid_bdev:
+        rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+        bdevs_names = [d['name'] for d in rpc_client.get_bdevs()]
+        jm_nvme_bdevs = []
+        for dev in snode.nvme_devices:
+            if dev.status != NVMeDevice.STATUS_ONLINE:
+                continue
+            dev_part = f"{dev.nvme_bdev[:-1]}1"
+            if dev_part in bdevs_names:
+                if dev_part not in jm_nvme_bdevs:
+                    jm_nvme_bdevs.append(dev_part)
+
+        if len(jm_nvme_bdevs) > 0:
+            new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
+            if not new_jm:
+                logger.error("failed to create jm stack")
+                return False
+
+            else:
+                snode = db_controller.get_storage_node_by_id(snode.get_id())
+                snode.jm_device = new_jm
+                snode.write_to_db(db_controller.kv_store)
+                set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
 
-    jm_device.status = state
-    snode.write_to_db(db_controller.kv_store)
     return True

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/simplyblock_core/services/storage_node_monitor.py

@@ -136,36 +136,33 @@ while True:
                 logger.info(f"Check 2: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
             # 2- check node API
-            node_api_check = False
-            spdk_process = False
-            node_rpc_check = False
-            if ping_check:
-                node_api_check = health_controller._check_node_api(snode.mgmt_ip)
-                logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
-
-                # 3- check spdk_process
-                spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
-                logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
-
-                # node_rpc_check = True
-                # 3- check node RPC
-                node_rpc_check = health_controller._check_node_rpc(
-                    snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=1)
-                logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
-
-                # check JM device
-                if snode.jm_device:
-                    if snode.jm_device.status in [JMDevice.STATUS_ONLINE, JMDevice.STATUS_UNAVAILABLE]:
-                        ret = health_controller.check_jm_device(snode.jm_device.get_id())
-                        if ret:
-                            logger.info(f"JM bdev is online: {snode.jm_device.get_id()}")
-                            if snode.jm_device.status != JMDevice.STATUS_ONLINE:
-                                device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
-                        else:
-                            logger.error(f"JM bdev is offline: {snode.jm_device.get_id()}")
-                            if snode.jm_device.status != JMDevice.STATUS_UNAVAILABLE:
-                                device_controller.set_jm_device_state(snode.jm_device.get_id(),
-                                                                      JMDevice.STATUS_UNAVAILABLE)
+
+            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+            # 3- check spdk_process
+            spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
+            logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
+
+            # node_rpc_check = True
+            # 3- check node RPC
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=1)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            # check JM device
+            if snode.jm_device:
+                if snode.jm_device.status in [JMDevice.STATUS_ONLINE, JMDevice.STATUS_UNAVAILABLE]:
+                    ret = health_controller.check_jm_device(snode.jm_device.get_id())
+                    if ret:
+                        logger.info(f"JM bdev is online: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_ONLINE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+                    else:
+                        logger.error(f"JM bdev is offline: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_UNAVAILABLE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(),
+                                                                  JMDevice.STATUS_UNAVAILABLE)
 
             is_node_online = ping_check and node_api_check and node_rpc_check and spdk_process
             if is_node_online:
@@ -178,7 +175,7 @@ while True:
 
                 elif ping_check and node_api_check and not spdk_process:
                     # add node to auto restart
-                    if cluster.status != Cluster.STATUS_UNREADY:
+                    if cluster.status == Cluster.STATUS_ACTIVE:
                         tasks_controller.add_node_to_auto_restart(snode)
 
         update_cluster_status(cluster_id)

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/simplyblock_core/storage_node_ops.py

@@ -792,8 +792,8 @@ def _connect_to_remote_jm_devs(this_node):
         if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
 
-        logger.info(f"Connecting to node {node.get_id()}")
-        if node.jm_device:
+        if node.jm_device and node.jm_device.status == NVMeDevice.STATUS_ONLINE:
+            logger.info(f"Connecting to JM on node: {node.get_id()}")
             name = f"remote_{node.jm_device.jm_bdev}"
             bdev_name = f"{name}n1"
             ret = rpc_client.get_bdevs(bdev_name)

{sbcli_dev-3.8.65 → sbcli_dev-3.8.67}/simplyblock_core/utils.py

@@ -261,7 +261,7 @@ def process_records(records, records_count):
 
 def ping_host(ip):
     logger.debug(f"Pinging ip ... {ip}")
-    response = os.system(f"ping -c 3 -W 3 {ip} > /dev/null")
+    response = os.system(f"ping -c 1 -W 3 {ip} > /dev/null")
     if response == 0:
         logger.debug(f"{ip} is UP")
         return True