sbcli-dev 3.8.57__zip → 3.8.59__zip

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.57
+Version: 3.8.59
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.57
+SIMPLY_BLOCK_VERSION=3.8.59
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.57
+Version: 3.8.59
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/simplyblock_core/controllers/device_controller.py

@@ -4,7 +4,8 @@ import logging
 from simplyblock_core import distr_controller, utils, storage_node_ops
 from simplyblock_core.controllers import device_events, lvol_controller, tasks_controller
 from simplyblock_core.kv_store import DBController
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
+from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 
 
@@ -248,6 +249,53 @@ def restart_device(device_id, force=False):
     logger.info("Setting device online")
     device_set_online(device_id)
     device_events.device_restarted(device_obj)
+
+    # add to jm raid
+    if snode.jm_device and snode.jm_device.raid_bdev:
+        nvme_controller = device_obj.nvme_controller
+        jm_part = None
+        for part in snode.jm_device.jm_nvme_bdev_list:
+            if part.startswith(nvme_controller):
+                jm_part = part
+                break
+        if snode.jm_device.status == NVMeDevice.STATUS_ONLINE:
+            set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+            # delete jm stack
+            if snode.enable_ha_jm:
+                ret = rpc_client.subsystem_delete(snode.jm_device.nvmf_nqn)
+
+            if snode.jm_device.pt_bdev:
+                ret = rpc_client.bdev_PT_NoExcl_delete(snode.jm_device.pt_bdev)
+
+            ret = rpc_client.bdev_jm_delete(snode.jm_device.jm_bdev)
+            if snode.jm_device.testing_bdev:
+                ret = rpc_client.bdev_passtest_delete(snode.jm_device.testing_bdev)
+            ret = rpc_client.bdev_alceml_delete(snode.jm_device.alceml_bdev)
+            if len(snode.jm_device.jm_nvme_bdev_list) == 2:
+                ret = rpc_client.bdev_raid_delete(snode.jm_device.raid_bdev)
+
+        # create jm
+        jm_nvme_bdevs = snode.jm_device.jm_nvme_bdev_list
+        if jm_part not in jm_nvme_bdevs:
+            jm_nvme_bdevs.append(jm_part)
+        new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
+        if new_jm:
+            snode = db_controller.get_storage_node_by_id(snode.get_id())
+            snode.jm_device = new_jm
+            snode.write_to_db(db_controller.kv_store)
+            set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+
+        # make other nodes connect to new jm
+        if snode.enable_ha_jm:
+            logger.info("Make other nodes connect to the new devices")
+            snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+            for node_index, node in enumerate(snodes):
+                if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+                    continue
+                node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
+                node.write_to_db(db_controller.kv_store)
+                logger.info(f"connected to devices count: {len(node.remote_devices)}")
+
     return "Done"
 
 
@@ -377,10 +425,51 @@ def device_remove(device_id, force=True):
                 break
 
         if dev_to_remove:
-            ret = rpc_client.bdev_raid_remove_base_bdev(snode.jm_device.raid_bdev, dev_to_remove)
-            if ret:
-                snode.jm_device.jm_nvme_bdev_list.remove(dev_to_remove)
-                snode.write_to_db(db_controller.kv_store)
+            if len(snode.jm_device.jm_nvme_bdev_list) <= 2:
+
+                set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+
+                # delete jm stack
+                if snode.enable_ha_jm:
+                    ret = rpc_client.subsystem_delete(snode.jm_device.nvmf_nqn)
+
+                if snode.jm_device.pt_bdev:
+                    ret = rpc_client.bdev_PT_NoExcl_delete(snode.jm_device.pt_bdev)
+
+                ret = rpc_client.bdev_jm_delete(snode.jm_device.jm_bdev)
+                if snode.jm_device.testing_bdev:
+                    ret = rpc_client.bdev_passtest_delete(snode.jm_device.testing_bdev)
+                ret = rpc_client.bdev_alceml_delete(snode.jm_device.alceml_bdev)
+                if len(snode.jm_device.jm_nvme_bdev_list) == 2:
+                    ret = rpc_client.bdev_raid_delete(snode.jm_device.raid_bdev)
+
+
+                # create jm
+                jm_nvme_bdevs = snode.jm_device.jm_nvme_bdev_list
+                jm_nvme_bdevs.remove(dev_to_remove)
+                if len(jm_nvme_bdevs) > 0:
+                    new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
+                    snode.jm_device = new_jm
+                    snode.write_to_db(db_controller.kv_store)
+                    set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+
+                    # make other nodes connect to new jm
+                    if snode.enable_ha_jm:
+                        logger.info("Make other nodes connect to the new devices")
+                        snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+                        for node_index, node in enumerate(snodes):
+                            if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+                                continue
+                            node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
+                            node.write_to_db(db_controller.kv_store)
+                            logger.info(f"connected to devices count: {len(node.remote_devices)}")
+                            time.sleep(3)
+
+            elif len(snode.jm_device.jm_nvme_bdev_list) > 2:
+                ret = rpc_client.bdev_raid_remove_base_bdev(snode.jm_device.raid_bdev, dev_to_remove)
+                if ret:
+                    snode.jm_device.jm_nvme_bdev_list.remove(dev_to_remove)
+                    snode.write_to_db(db_controller.kv_store)
 
     return True
 

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/simplyblock_core/controllers/health_controller.py

@@ -160,13 +160,12 @@ def check_node(node_id, with_devices=True):
     else:
         logger.info(f"Node device count: {len(snode.nvme_devices)}")
         for dev in snode.nvme_devices:
-            ret = check_device(dev.get_id())
             if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]:
+                ret = check_device(dev.get_id())
                 node_devices_check &= ret
             print("*" * 100)
 
         logger.info(f"Node remote device: {len(snode.remote_devices)}")
-
         rpc_client = RPCClient(
             snode.mgmt_ip, snode.rpc_port,
             snode.rpc_username, snode.rpc_password,
@@ -177,14 +176,28 @@ def check_node(node_id, with_devices=True):
                 logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
             else:
                 logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-            # node_remote_devices_check &= bool(ret)
+            node_remote_devices_check &= bool(ret)
+
         if snode.jm_device:
             jm_device = snode.jm_device
+            logger.info(f"Node node jm: {jm_device}")
             ret = check_jm_device(jm_device.get_id())
             if ret:
                 logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
             else:
                 logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
+            node_devices_check &= ret
+
+        if snode.enable_ha_jm:
+            logger.info(f"Node remote JMs: {len(snode.remote_jm_devices)}")
+            for remote_device in snode.remote_jm_devices:
+                ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                if ret:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                else:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                node_remote_devices_check &= bool(ret)
+
         lvstore_check = True
         if snode.lvstore and snode.lvstore_stack:
             distribs_list = []
@@ -196,7 +209,7 @@ def check_node(node_id, with_devices=True):
             for distr in distribs_list:
                 ret = rpc_client.get_bdevs(distr)
                 if ret:
-                    logger.info("Checking distr bdev : {distr} ... ok")
+                    logger.info(f"Checking distr bdev : {distr} ... ok")
                     logger.info("Checking Distr map ...")
                     ret = rpc_client.distr_get_cluster_map(distr)
                     if not ret:
@@ -211,19 +224,19 @@ def check_node(node_id, with_devices=True):
                             logger.error("Failed to parse distr cluster map")
                         lvstore_check &= is_passed
                 else:
-                    logger.info("Checking distr bdev : {distr} ... not found")
+                    logger.info(f"Checking distr bdev : {distr} ... not found")
                     lvstore_check = False
             ret = rpc_client.get_bdevs(snode.raid)
             if ret:
-                logger.info("Checking raid bdev: {snode.raid} ... ok")
+                logger.info(f"Checking raid bdev: {snode.raid} ... ok")
             else:
-                logger.info("Checking raid bdev: {snode.raid} ... not found")
+                logger.info(f"Checking raid bdev: {snode.raid} ... not found")
                 lvstore_check = False
             ret = rpc_client.bdev_lvol_get_lvstores(snode.lvstore)
             if ret:
-                logger.info("Checking lvstore: {snode.lvstore} ... ok")
+                logger.info(f"Checking lvstore: {snode.lvstore} ... ok")
             else:
-                logger.info("Checking lvstore: {snode.lvstore} ... not found")
+                logger.info(f"Checking lvstore: {snode.lvstore} ... not found")
                 lvstore_check = False
 
     return is_node_online and node_devices_check and node_remote_devices_check and lvstore_check

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/simplyblock_core/services/health_check_service.py

@@ -114,7 +114,27 @@ while True:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                     else:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    # node_remote_devices_check &= bool(ret)
+                    node_remote_devices_check &= bool(ret)
+
+                if snode.jm_device:
+                    jm_device = snode.jm_device
+                    logger.info(f"Node node jm: {jm_device}")
+                    ret = health_controller.check_jm_device(jm_device.get_id())
+                    if ret:
+                        logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
+                    else:
+                        logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
+                    node_devices_check &= ret
+
+                if snode.enable_ha_jm:
+                    logger.info(f"Node remote JMs: {len(snode.remote_jm_devices)}")
+                    for remote_device in snode.remote_jm_devices:
+                        ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                        if ret:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                        else:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                        node_remote_devices_check &= bool(ret)
 
                 health_check_status = is_node_online and node_devices_check and node_remote_devices_check
             set_node_health_check(snode, health_check_status)

{sbcli_dev-3.8.57 → sbcli_dev-3.8.59}/simplyblock_core/storage_node_ops.py

@@ -244,13 +244,15 @@ def _search_for_partitions(rpc_client, nvme_device):
 
 def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
     raid_bdev = f"raid_jm_{snode.get_id()}"
-    raid_level = "0"
     if len(jm_nvme_bdevs) > 1:
         raid_level = "1"
-    ret = rpc_client.bdev_raid_create(raid_bdev, jm_nvme_bdevs, raid_level)
-    if not ret:
-        logger.error(f"Failed to create raid_jm_{snode.get_id()}")
-        return False
+        ret = rpc_client.bdev_raid_create(raid_bdev, jm_nvme_bdevs, raid_level)
+        if not ret:
+            logger.error(f"Failed to create raid_jm_{snode.get_id()}")
+            return False
+    else:
+        raid_bdev = jm_nvme_bdevs[0]
+
     alceml_name = f"alceml_jm_{snode.get_id()}"
 
     nvme_bdev = raid_bdev
@@ -323,7 +325,7 @@ def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
             logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
             return False
 
-    ret = rpc_client.get_bdevs(raid_bdev)
+    ret = rpc_client.get_bdevs(jm_bdev)
 
     return JMDevice({
         'uuid': alceml_id,