sbcli-pre 1.2.5__zip → 1.2.7__zip

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/storage_node_ops.py

@@ -14,11 +14,11 @@ import docker
 from simplyblock_core import constants, scripts, distr_controller
 from simplyblock_core import utils
 from simplyblock_core.controllers import lvol_controller, storage_events, snapshot_controller, device_events, \
-    device_controller
+    device_controller, tasks_controller
 from simplyblock_core.kv_store import DBController
 from simplyblock_core import shell_utils
 from simplyblock_core.models.iface import IFace
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.pci_utils import get_nvme_devices, bind_spdk_driver
 from simplyblock_core.rpc_client import RPCClient
@@ -81,55 +81,50 @@ def _get_if_ip_address(ifname):
 
 
 def addNvmeDevices(cluster, rpc_client, devs, snode):
-    sequential_number = 0
     devices = []
     ret = rpc_client.bdev_nvme_controller_list()
-    if ret:
-        ctr_map = {i["ctrlrs"][0]['trid']['traddr']: i["name"] for i in ret}
-    else:
-        ctr_map = {}
+    ctr_map = {}
+    try:
+        if ret:
+            ctr_map = {i["ctrlrs"][0]['trid']['traddr']: i["name"] for i in ret}
+    except:
+        pass
 
+    next_physical_label = get_next_physical_device_order()
     for index, pcie in enumerate(devs):
 
         if pcie in ctr_map:
-            nvme_bdev = ctr_map[pcie] + "n1"
+            nvme_controller = ctr_map[pcie]
         else:
-            name = "nvme_%s" % index
-            ret, err = rpc_client.bdev_nvme_controller_attach(name, pcie)
+            nvme_controller = "nvme_%s" % index
+            ret, err = rpc_client.bdev_nvme_controller_attach(nvme_controller, pcie)
             time.sleep(2)
-            nvme_bdev = f"{name}n1"
 
+        nvme_bdev = f"{nvme_controller}n1"
+        rpc_client.bdev_examine(nvme_bdev)
+        time.sleep(5)
         ret = rpc_client.get_bdevs(nvme_bdev)
-        if ret:
-            nvme_dict = ret[0]
-            nvme_driver_data = nvme_dict['driver_specific']['nvme'][0]
-            model_number = nvme_driver_data['ctrlr_data']['model_number']
+        nvme_dict = ret[0]
+        nvme_driver_data = nvme_dict['driver_specific']['nvme'][0]
+        model_number = nvme_driver_data['ctrlr_data']['model_number']
+        total_size = nvme_dict['block_size'] * nvme_dict['num_blocks']
 
-            size = nvme_dict['block_size'] * nvme_dict['num_blocks']
-            device_partitions_count = int(size / (cluster.blk_size * cluster.page_size_in_blocks))
-            devices.append(
-                NVMeDevice({
-                    'uuid': str(uuid.uuid4()),
-                    'device_name': nvme_dict['name'],
-                    'sequential_number': sequential_number,
-                    'partitions_count': device_partitions_count,
-                    'capacity': size,
-                    'size': size,
-                    'pcie_address': nvme_driver_data['pci_address'],
-                    'model_id': model_number,
-                    'serial_number': nvme_driver_data['ctrlr_data']['serial_number'],
-                    'nvme_bdev': nvme_bdev,
-                    'alloc_bdev': nvme_bdev,
-                    'node_id': snode.get_id(),
-                    'cluster_id': snode.cluster_id,
-
-                    # 'nvmf_nqn': subsystem_nqn,
-                    # 'nvmf_ip': IP,
-                    # 'nvmf_port': 4420,
-
-                    'status': 'online'
-                }))
-            sequential_number += device_partitions_count
+        devices.append(
+            NVMeDevice({
+                'uuid': str(uuid.uuid4()),
+                'device_name': nvme_dict['name'],
+                'size': total_size,
+                'physical_label': next_physical_label,
+                'pcie_address': nvme_driver_data['pci_address'],
+                'model_id': model_number,
+                'serial_number': nvme_driver_data['ctrlr_data']['serial_number'],
+                'nvme_bdev': nvme_bdev,
+                'nvme_controller': nvme_controller,
+                'node_id': snode.get_id(),
+                'cluster_id': snode.cluster_id,
+                'status': NVMeDevice.STATUS_ONLINE
+        }))
+        next_physical_label += 1
     return devices
 
 
@@ -197,10 +192,10 @@ def _run_nvme_smart_log_add(dev_name):
     return data
 
 
-def get_next_cluster_device_order(db_controller):
+def get_next_cluster_device_order(db_controller, cluster_id):
     max_order = 0
     found = False
-    for node in db_controller.get_storage_nodes():
+    for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id):
         for dev in node.nvme_devices:
             found = True
             max_order = max(max_order, dev.cluster_device_order)
@@ -209,91 +204,319 @@ def get_next_cluster_device_order(db_controller):
     return 0
 
 
-def _prepare_cluster_devices(snode, after_restart=False):
+def get_next_physical_device_order():
     db_controller = DBController()
+    max_order = 0
+    found = False
+    for node in db_controller.get_storage_nodes():
+        for dev in node.nvme_devices:
+            found = True
+            max_order = max(max_order, dev.physical_label)
+    if found:
+        return max_order + 1
+    return 0
+
+
+def _search_for_partitions(rpc_client, nvme_device):
+    partitioned_devices = []
+    for bdev in rpc_client.get_bdevs():
+        name = bdev['name']
+        if name.startswith(f"{nvme_device.nvme_bdev}p"):
+            new_dev = NVMeDevice(nvme_device.to_dict())
+            new_dev.uuid = str(uuid.uuid4())
+            new_dev.device_name = name
+            new_dev.nvme_bdev = name
+            new_dev.size = bdev['block_size'] * bdev['num_blocks']
+            partitioned_devices.append(new_dev)
+    return partitioned_devices
+
+
+def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
+    raid_bdev = f"raid_jm_{snode.get_id()}"
+    ret = rpc_client.bdev_raid_create(raid_bdev, jm_nvme_bdevs)
+    if not ret:
+        logger.error(f"Failed to create raid_jm_{snode.get_id()}")
+        return False
+    alceml_name = f"alceml_jm_{snode.get_id()}"
+    pba_init_mode = 3
+    if after_restart:
+        pba_init_mode = 2
+    ret = rpc_client.bdev_alceml_create(alceml_name, raid_bdev, str(uuid.uuid4()), pba_init_mode=pba_init_mode)
+    if not ret:
+        logger.error(f"Failed to create alceml bdev: {alceml_name}")
+        return False
+
+    jm_bdev = f"jm_{snode.get_id()}"
+    ret = rpc_client.bdev_jm_create(jm_bdev, alceml_name)
+    if not ret:
+        logger.error(f"Failed to create {jm_bdev}")
+        return False
+    ret = rpc_client.get_bdevs(raid_bdev)
+
+    return JMDevice({
+        'uuid': str(uuid.uuid4()),
+        'device_name': jm_bdev,
+        'size': ret[0]["block_size"] * ret[0]["num_blocks"],
+        'status': JMDevice.STATUS_ONLINE,
+        'jm_nvme_bdev_list': jm_nvme_bdevs,
+        'raid_bdev': raid_bdev,
+        'alceml_bdev': alceml_name,
+        'jm_bdev': jm_bdev
+    })
+
+
+def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
+
+    alceml_id = nvme.get_id()
+    alceml_name = device_controller.get_alceml_name(alceml_id)
+    logger.info(f"adding {alceml_name}")
+
+    pba_init_mode = 3
+    if after_restart:
+        pba_init_mode = 2
+    ret = rpc_client.bdev_alceml_create(alceml_name, nvme.nvme_bdev, alceml_id, pba_init_mode=pba_init_mode)
+    if not ret:
+        logger.error(f"Failed to create alceml bdev: {alceml_name}")
+        return False
+
+    jm_bdev = f"jm_{snode.get_id()}"
+    ret = rpc_client.bdev_jm_create(jm_bdev, alceml_name)
+    if not ret:
+        logger.error(f"Failed to create {jm_bdev}")
+        return False
 
+    return JMDevice({
+        'uuid': alceml_id,
+        'device_name': jm_bdev,
+        'size': nvme.size,
+        'status': JMDevice.STATUS_ONLINE,
+        'alceml_bdev': alceml_name,
+        'nvme_bdev': nvme.nvme_bdev,
+        'jm_bdev': jm_bdev
+    })
+
+
+def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
+    test_name = f"{nvme.nvme_bdev}_test"
+    ret = rpc_client.bdev_passtest_create(test_name, nvme.nvme_bdev)
+    if not ret:
+        logger.error(f"Failed to create passtest bdev {test_name}")
+        return False
+    alceml_id = nvme.get_id()
+    alceml_name = device_controller.get_alceml_name(alceml_id)
+    logger.info(f"adding {alceml_name}")
+    pba_init_mode = 3
+    if after_restart:
+        pba_init_mode = 2
+    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode,
+                                            dev_cpu_mask=snode.dev_cpu_mask)
+    if not ret:
+        logger.error(f"Failed to create alceml bdev: {alceml_name}")
+        return False
+
+    # add pass through
+    pt_name = f"{alceml_name}_PT"
+    ret = rpc_client.bdev_PT_NoExcl_create(pt_name, alceml_name)
+    if not ret:
+        logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
+        return False
+
+    subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
+    logger.info("creating subsystem %s", subsystem_nqn)
+    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
+    IP = None
+    for iface in snode.data_nics:
+        if iface.ip4_address:
+            tr_type = iface.get_transport_type()
+            ret = rpc_client.transport_list()
+            found = False
+            if ret:
+                for ty in ret:
+                    if ty['trtype'] == tr_type:
+                        found = True
+            if found is False:
+                ret = rpc_client.transport_create(tr_type)
+            logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
+            ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, "4420")
+            IP = iface.ip4_address
+            break
+    logger.info(f"add {pt_name} to subsystem")
+    ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+    if not ret:
+        logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
+        return False
+
+    nvme.testing_bdev = test_name
+    nvme.alceml_bdev = alceml_name
+    nvme.pt_bdev = pt_name
+    nvme.nvmf_nqn = subsystem_nqn
+    nvme.nvmf_ip = IP
+    nvme.nvmf_port = 4420
+    nvme.io_error = False
+    nvme.status = NVMeDevice.STATUS_ONLINE
+    return nvme
+
+
+def _create_device_partitions(rpc_client, nvme, snode):
+    nbd_device = rpc_client.nbd_start_disk(nvme.nvme_bdev)
+    time.sleep(3)
+    if not nbd_device:
+        logger.error(f"Failed to start nbd dev")
+        return False
+    snode_api = SNodeClient(snode.api_endpoint)
+    result, error = snode_api.make_gpt_partitions(
+        nbd_device, snode.jm_percent, snode.num_partitions_per_dev)
+    if error:
+        logger.error(f"Failed to make partitions")
+        logger.error(error)
+        return False
+    time.sleep(3)
+    rpc_client.nbd_stop_disk(nbd_device)
+    time.sleep(1)
+    rpc_client.bdev_nvme_detach_controller(nvme.nvme_controller)
+    time.sleep(1)
+    rpc_client.bdev_nvme_controller_attach(nvme.nvme_controller, nvme.pcie_address)
+    time.sleep(1)
+    rpc_client.bdev_examine(nvme.nvme_bdev)
+    time.sleep(1)
+    return True
+
+
+def _prepare_cluster_devices_partitions(snode, devices):
+    db_controller = DBController()
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)
 
-    for index, nvme in enumerate(snode.nvme_devices):
+    new_devices = []
+    jm_devices = []
+    dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
+    for index, nvme in enumerate(devices):
+        if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY]:
+            logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}")
+            continue
+
+        # look for partitions
+        partitioned_devices = _search_for_partitions(rpc_client, nvme)
+        logger.debug("partitioned_devices")
+        logger.debug(partitioned_devices)
+        if len(partitioned_devices) == (1 + snode.num_partitions_per_dev):
+            logger.info("Partitioned devices found")
+        else:
+            logger.info(f"Creating partitions for {nvme.nvme_bdev}")
+            _create_device_partitions(rpc_client, nvme, snode)
+            partitioned_devices = _search_for_partitions(rpc_client, nvme)
+            if len(partitioned_devices) == (1 + snode.num_partitions_per_dev):
+                logger.info("Device partitions created")
+            else:
+                logger.error("Failed to create partitions")
+                return False
+
+        jm_devices.append(partitioned_devices.pop(0))
+
+        for dev in partitioned_devices:
+            new_device = _create_storage_device_stack(rpc_client, dev, snode, after_restart=False)
+            if not new_device:
+                logger.error("failed to create dev stack")
+                return False
+            new_device.cluster_device_order = dev_order
+            dev_order += 1
+            new_devices.append(new_device)
+            device_events.device_create(new_device)
+
+    snode.nvme_devices = new_devices
+
+    if jm_devices:
+        jm_nvme_bdevs = [dev.nvme_bdev for dev in jm_devices]
+        jm_device = _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=False)
+        if not jm_device:
+            logger.error(f"Failed to create JM device")
+            return False
+        snode.jm_device = jm_device
+
+    return True
+
+
+def _prepare_cluster_devices_jm_on_dev(snode, devices):
+    db_controller = DBController()
+
+    jm_device = devices[0]
+    # Set device cluster order
+    dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
+    for index, nvme in enumerate(devices):
+        nvme.cluster_device_order = dev_order
+        dev_order += 1
+        if nvme.size < jm_device.size:
+            jm_device = nvme
+        device_events.device_create(nvme)
+    jm_device.status = NVMeDevice.STATUS_JM
+
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+
+    new_devices = []
+    for index, nvme in enumerate(devices):
         if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE,
                                NVMeDevice.STATUS_JM, NVMeDevice.STATUS_READONLY]:
             logger.debug(f"Device is not online or unavailable: {nvme.get_id()}, status: {nvme.status}")
             continue
 
-        test_name = f"{nvme.nvme_bdev}_test"
-        # create testing bdev
-        ret = rpc_client.bdev_passtest_create(test_name, nvme.nvme_bdev)
-        if not ret:
-            logger.error(f"Failed to create bdev: {test_name}")
-            return False
-        alceml_id = nvme.get_id()
-        alceml_name = device_controller.get_alceml_name(alceml_id)
-        logger.info(f"adding {alceml_name}")
-        pba_init_mode = 3
-        if after_restart:
-            pba_init_mode = 2
-        ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode)
-        if not ret:
-            logger.error(f"Failed to create alceml bdev: {alceml_name}")
-            return False
-
-        # create jm
-        if nvme.jm_bdev:
-            ret = rpc_client.bdev_jm_create(nvme.jm_bdev, alceml_name)
-            if not ret:
-                logger.error(f"Failed to create JM bdev: {nvme.jm_bdev}")
+        if nvme.status == NVMeDevice.STATUS_JM:
+            jm_device = _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart=False)
+            if not jm_device:
+                logger.error(f"Failed to create JM device")
                 return False
-            nvme.testing_bdev = test_name
-            nvme.alceml_bdev = alceml_name
-            nvme.io_error = True
-            nvme.status = NVMeDevice.STATUS_JM
+            snode.jm_device = jm_device
+        else:
+            new_device = _create_storage_device_stack(rpc_client, nvme, snode, after_restart=False)
+            if not new_device:
+                logger.error("failed to create dev stack")
+                return False
+            new_device.cluster_device_order = dev_order
+            dev_order += 1
+            new_devices.append(new_device)
+            device_events.device_create(new_device)
+
+    snode.nvme_devices = new_devices
+    return True
+
+
+def _prepare_cluster_devices_on_restart(snode):
+    db_controller = DBController()
+
+    rpc_client = RPCClient(
+        snode.mgmt_ip, snode.rpc_port,
+        snode.rpc_username, snode.rpc_password)
+
+    for index, nvme in enumerate(snode.nvme_devices):
+        if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY]:
+            logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}")
             continue
 
-        # add pass through
-        pt_name = f"{alceml_name}_PT"
-        ret = rpc_client.bdev_PT_NoExcl_create(pt_name, alceml_name)
+        dev = _create_storage_device_stack(rpc_client, nvme, snode, after_restart=True)
+        if not dev:
+            logger.error(f"Failed to create dev stack {nvme.get_id()}")
+            return False
+        device_events.device_restarted(dev)
+
+    # prepare JM device
+    jm_device = snode.jm_device
+    if jm_device.jm_nvme_bdev_list:
+        ret = _create_jm_stack_on_raid(rpc_client, jm_device.jm_nvme_bdev_list, snode, after_restart=False)
         if not ret:
-            logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
+            logger.error(f"Failed to create JM device")
             return False
+    else:
 
-        subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
-        logger.info("creating subsystem %s", subsystem_nqn)
-        ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
-        IP = None
-        for iface in snode.data_nics:
-            if iface.ip4_address:
-                tr_type = iface.get_transport_type()
-                ret = rpc_client.transport_list()
-                found = False
-                if ret:
-                    for ty in ret:
-                        if ty['trtype'] == tr_type:
-                            found = True
-                if found is False:
-                    ret = rpc_client.transport_create(tr_type)
-                logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
-                ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, "4420")
-                IP = iface.ip4_address
-                break
-        logger.info(f"add {pt_name} to subsystem")
-        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
+        ret = rpc_client.bdev_alceml_create(jm_device.alceml_bdev, jm_device.nvme_bdev, jm_device.get_id(), pba_init_mode=2)
         if not ret:
-            logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
+            logger.error(f"Failed to create alceml bdev: {jm_device.alceml_bdev}")
             return False
 
-        nvme.testing_bdev = test_name
-        nvme.alceml_bdev = alceml_name
-        nvme.pt_bdev = pt_name
-        nvme.nvmf_nqn = subsystem_nqn
-        nvme.nvmf_ip = IP
-        nvme.nvmf_port = 4420
-        nvme.io_error = False
-        old_status = nvme.status
-        nvme.status = NVMeDevice.STATUS_ONLINE
-        device_events.device_status_change(nvme, nvme.status, old_status)
-        snode.write_to_db(db_controller.kv_store)
+        jm_bdev = f"jm_{snode.get_id()}"
+        ret = rpc_client.bdev_jm_create(jm_bdev, jm_device.alceml_bdev)
+        if not ret:
+            logger.error(f"Failed to create {jm_bdev}")
+            return False
 
     return True
 
@@ -307,7 +530,7 @@ def _connect_to_remote_devs(this_node):
 
     remote_devices = []
     # connect to remote devs
-    snodes = db_controller.get_storage_nodes()
+    snodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id)
     for node_index, node in enumerate(snodes):
         if node.get_id() == this_node.get_id() or node.status == node.STATUS_OFFLINE:
             continue
@@ -326,9 +549,10 @@ def _connect_to_remote_devs(this_node):
     return remote_devices
 
 
-def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
-             spdk_mem, spdk_image=None, spdk_debug=False,
-             small_pool_count=0, large_pool_count=0, small_bufsize=0, large_bufsize=0, jm_device_pcie=None):
+def add_node(cluster_id, node_ip, iface_name, data_nics_list,
+             max_lvol, max_snap, max_prov, spdk_image=None, spdk_debug=False,
+             small_bufsize=0, large_bufsize=0,
+             num_partitions_per_dev=0, jm_percent=0, number_of_devices=0):
     db_controller = DBController()
     kv_store = db_controller.kv_store
 
@@ -384,20 +608,71 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
                 logger.error(f"Node already exists, try remove it first: {ec2_metadata['instanceId']}")
                 return False
 
+    # Tune cpu maks parameters
+    cpu_count = node_info["cpu_count"]
+    pollers_mask = ""
+    app_thread_mask = ""
+    dev_cpu_mask = ""
+    nvme_pollers_cores = []
+    if cpu_count < 8:
+        mask = (1 << (cpu_count - 1)) - 1
+        mask <<= 1
+        spdk_cpu_mask = f'0x{mask:X}'
+        os_cores = [0]
+    else:
+        os_cores, nvme_pollers_cores, app_thread_core, dev_cpu_cores = \
+            utils.calculate_core_allocation(cpu_count)
+        spdk_cores = nvme_pollers_cores + app_thread_core + dev_cpu_cores
+
+        pollers_mask = utils.generate_mask(nvme_pollers_cores)
+        app_thread_mask = utils.generate_mask(app_thread_core)
+        spdk_cpu_mask = utils.generate_mask(spdk_cores)
+        dev_cpu_mask = utils.generate_mask(dev_cpu_cores)
+
+    # Calculate pool count
+    if ec2_metadata and ec2_metadata.get('instanceType'):
+        supported_type, storage_devices, device_size = utils.get_total_size_per_instance_type(ec2_metadata["instanceType"])
+        if not supported_type:
+            logger.warning(f"Unsupported ec2 instance-type {ec2_metadata['instanceType']} for deployment")
+            if not number_of_devices:
+                logger.error(f"Unsupported ec2 instance-type {ec2_metadata['instanceType']} "
+                             "for deployment, please specify --number-of-devices")
+                return False
+        number_of_devices = storage_devices
+    else:
+        logger.warning("Can not get ec2 instance type for this instance.")
+        if not number_of_devices:
+            logger.error("Unsupported instance type please specify --number-of-devices.")
+            return False
+
+    number_of_split = num_partitions_per_dev if num_partitions_per_dev else num_partitions_per_dev + 1
+    number_of_alceml_devices = number_of_devices * number_of_split
+    small_pool_count, large_pool_count = utils.calculate_pool_count(
+        number_of_alceml_devices, max_lvol, max_snap, cpu_count, len(nvme_pollers_cores) or cpu_count)
+
+    # Calculate minimum huge page memory
+    minimum_hp_memory = utils.calculate_minimum_hp_memory(small_pool_count, large_pool_count, max_lvol, max_snap, cpu_count)
+
+    # Calculate minimum sys memory
+    minimum_sys_memory = utils.calculate_minimum_sys_memory(max_prov)
+
     # check for memory
     if "memory_details" in node_info and node_info['memory_details']:
         memory_details = node_info['memory_details']
         logger.info("Node Memory info")
         logger.info(f"Total: {utils.humanbytes(memory_details['total'])}")
         logger.info(f"Free: {utils.humanbytes(memory_details['free'])}")
-        logger.info(f"Hugepages Total: {utils.humanbytes(memory_details['huge_total'])}")
-        huge_free = memory_details['huge_free']
-        logger.info(f"Hugepages Free: {utils.humanbytes(huge_free)}")
-        if huge_free < 1 * 1024 * 1024:
-            logger.warning(f"Free hugepages are less than 1G: {utils.humanbytes(huge_free)}")
-        if not spdk_mem:
-            spdk_mem = huge_free
-            logger.info(f"Using the free hugepages for spdk memory: {utils.humanbytes(huge_free)}")
+    else:
+        logger.error(f"Cannot get memory info from the ec2 instance.. Exiting")
+        return False
+
+    satisfied, spdk_mem = utils.calculate_spdk_memory(minimum_hp_memory,
+                                                      minimum_sys_memory,
+                                                      int(memory_details['free']),
+                                                      int(memory_details['huge_total']))
+    if not satisfied:
+        logger.error(f"Not enough memory for the provided max_lvo: {max_lvol}, max_snap: {max_snap}, max_prov: {max_prov}.. Exiting")
+        return False
 
     logger.info("Joining docker swarm...")
     cluster_docker = utils.get_docker_client(cluster_id)
@@ -472,16 +747,28 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
         snode.hugepages = node_info['hugepages']
 
     snode.spdk_cpu_mask = spdk_cpu_mask or ""
-    snode.spdk_mem = spdk_mem or 0
+    snode.spdk_mem = spdk_mem
+    snode.max_lvol = max_lvol
+    snode.max_snap = max_snap
+    snode.max_prov = max_prov
+    snode.number_of_devices = number_of_devices
     snode.spdk_image = spdk_image or ""
     snode.spdk_debug = spdk_debug or 0
     snode.write_to_db(kv_store)
+    snode.app_thread_mask = app_thread_mask or ""
+    snode.pollers_mask = pollers_mask or ""
+    snode.nvme_pollers_cores = nvme_pollers_cores or []
+    snode.dev_cpu_mask = dev_cpu_mask or ""
+    snode.os_cores = os_cores or []
 
     snode.iobuf_small_pool_count = small_pool_count or 0
     snode.iobuf_large_pool_count = large_pool_count or 0
     snode.iobuf_small_bufsize = small_bufsize or 0
     snode.iobuf_large_bufsize = large_bufsize or 0
 
+    snode.num_partitions_per_dev = num_partitions_per_dev
+    snode.jm_percent = jm_percent
+
     snode.write_to_db(kv_store)
 
     # creating RPCClient instance
@@ -499,13 +786,41 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- start spdk framework
+    # 2- set socket implementation options
+    ret = rpc_client.sock_impl_set_options()
+    if not ret:
+        logger.error("Failed socket implement set options")
+        return False
+
+    # 3- set nvme config
+    if snode.pollers_mask:
+        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
+        if not ret:
+            logger.error("Failed to set pollers mask")
+            return False
+
+    # 4- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 3- set nvme bdev options
+    # 5- set app_thread cpu mask
+    if snode.app_thread_mask:
+        ret = rpc_client.thread_get_stats()
+        app_thread_process_id = 0
+        if ret.get("threads"):
+            for entry in ret["threads"]:
+                if entry['name'] == 'app_thread':
+                    app_thread_process_id = entry['id']
+                    break
+
+        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
+        if not ret:
+            logger.error("Failed to set app thread mask")
+            return False
+
+    # 6- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")
@@ -513,36 +828,18 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
 
     # get new node info after starting spdk
     node_info, _ = snode_api.info()
-    # adding devices
+
+    # discover devices
     nvme_devs = addNvmeDevices(cluster, rpc_client, node_info['spdk_pcie_list'], snode)
     if not nvme_devs:
         logger.error("No NVMe devices was found!")
         return False
 
-    snode.nvme_devices = nvme_devs
-
-    jm_device = snode.nvme_devices[0]
-    # Set device cluster order
-    dev_order = get_next_cluster_device_order(db_controller)
-    for index, nvme in enumerate(snode.nvme_devices):
-        nvme.cluster_device_order = dev_order
-        dev_order += 1
-        if jm_device_pcie:
-            if nvme.pcie_address == jm_device_pcie:
-                jm_device = nvme
-        elif nvme.size < jm_device.size:
-            jm_device = nvme
-        device_events.device_create(nvme)
-
-    # create jm
-    logger.info(f"Using device for JM: {jm_device.get_id()}")
-    jm_device.jm_bdev = f"jm_{snode.get_id()}"
-
-    # save object
-    snode.write_to_db(db_controller.kv_store)
-
     # prepare devices
-    ret = _prepare_cluster_devices(snode)
+    if snode.num_partitions_per_dev == 0 or snode.jm_percent == 0:
+        ret = _prepare_cluster_devices_jm_on_dev(snode, nvme_devs)
+    else:
+        ret = _prepare_cluster_devices_partitions(snode, nvme_devs)
     if not ret:
         logger.error("Failed to prepare cluster devices")
         return False
@@ -557,7 +854,7 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
 
     # make other nodes connect to the new devices
     logger.info("Make other nodes connect to the new devices")
-    snodes = db_controller.get_storage_nodes()
+    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     for node_index, node in enumerate(snodes):
         if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
@@ -599,150 +896,16 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
     time.sleep(3)
 
     logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode.get_id(), "online")
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
 
     for dev in snode.nvme_devices:
-        distr_controller.send_dev_status_event(dev.cluster_device_order, "online")
+        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
 
     storage_events.snode_add(snode)
     logger.info("Done")
     return "Success"
 
 
-# Deprecated
-def add_storage_node(cluster_id, iface_name, data_nics):
-    db_controller = DBController()
-    kv_store = db_controller.kv_store
-
-    cluster = db_controller.get_cluster_by_id(cluster_id)
-    if not cluster:
-        logger.error("Cluster not found: %s", cluster_id)
-        return False
-
-    logger.info("Add Storage node")
-
-    hostname = utils.get_hostname()
-    snode = db_controller.get_storage_node_by_hostname(hostname)
-    if snode:
-        logger.error("Node already exists, try remove it first.")
-        exit(1)
-    else:
-        snode = StorageNode()
-        snode.uuid = str(uuid.uuid4())
-
-    mgmt_ip = _get_if_ip_address(iface_name)
-    system_id = utils.get_system_id()
-
-    BASE_NQN = cluster.nqn.split(":")[0]
-    subsystem_nqn = f"{BASE_NQN}:{hostname}"
-
-    if data_nics:
-        data_nics = _get_data_nics(data_nics)
-    else:
-        data_nics = _get_data_nics([iface_name])
-
-    rpc_user, rpc_pass = utils.generate_rpc_user_and_pass()
-
-    # creating storage node object
-    snode.status = StorageNode.STATUS_IN_CREATION
-    snode.baseboard_sn = utils.get_baseboard_sn()
-    snode.system_uuid = system_id
-    snode.hostname = hostname
-    snode.host_nqn = subsystem_nqn
-    snode.subsystem = subsystem_nqn
-    snode.data_nics = data_nics
-    snode.mgmt_ip = mgmt_ip
-    snode.rpc_port = constants.RPC_HTTP_PROXY_PORT
-    snode.rpc_username = rpc_user
-    snode.rpc_password = rpc_pass
-    snode.cluster_id = cluster_id
-    snode.write_to_db(kv_store)
-
-    # creating RPCClient instance
-    rpc_client = RPCClient(
-        snode.mgmt_ip,
-        snode.rpc_port,
-        snode.rpc_username,
-        snode.rpc_password)
-
-    logger.info("Getting nvme devices")
-    devs = get_nvme_devices()
-    logger.debug(devs)
-    pcies = [d[0] for d in devs]
-    nvme_devs = addNvmeDevices(cluster, rpc_client, pcies, snode)
-    if not nvme_devs:
-        logger.error("No NVMe devices was found!")
-
-    logger.debug(nvme_devs)
-    snode.nvme_devices = nvme_devs
-
-    # Set device cluster order
-    dev_order = get_next_cluster_device_order(db_controller)
-    for index, nvme in enumerate(snode.nvme_devices):
-        nvme.cluster_device_order = dev_order
-        dev_order += 1
-    snode.write_to_db(db_controller.kv_store)
-
-    # prepare devices
-    _prepare_cluster_devices(snode)
-
-    logger.info("Connecting to remote devices")
-    remote_devices = _connect_to_remote_devs(snode)
-    snode.remote_devices = remote_devices
-
-    logger.info("Setting node status to Active")
-    snode.status = StorageNode.STATUS_ONLINE
-    snode.write_to_db(kv_store)
-
-    # make other nodes connect to the new devices
-    logger.info("Make other nodes connect to the new devices")
-    snodes = db_controller.get_storage_nodes()
-    for node_index, node in enumerate(snodes):
-        if node.get_id() == snode.get_id():
-            continue
-        logger.info(f"Connecting to node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
-        count = 0
-        for dev in snode.nvme_devices:
-            name = f"remote_{dev.alceml_bdev}"
-            ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
-            if not ret:
-                logger.error(f"Failed to connect to device: {name}")
-                continue
-
-            dev.remote_bdev = f"{name}n1"
-            idx = -1
-            for i, d in enumerate(node.remote_devices):
-                if d.get_id() == dev.get_id():
-                    idx = i
-                    break
-            if idx >= 0:
-                node.remote_devices[idx] = dev
-            else:
-                node.remote_devices.append(dev)
-            count += 1
-        node.write_to_db(kv_store)
-        logger.info(f"connected to devices count: {count}")
-
-    logger.info("Sending cluster map")
-    ret = distr_controller.send_cluster_map_to_node(snode)
-    if not ret:
-        return False, "Failed to send cluster map"
-    ret = distr_controller.send_cluster_map_add_node(snode)
-    if not ret:
-        return False, "Failed to send cluster map add node"
-    time.sleep(3)
-
-    logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode.get_id(), "online")
-
-    for dev in snode.nvme_devices:
-        distr_controller.send_dev_status_event(dev.cluster_device_order, "online")
-
-    logger.info("Done")
-    return "Success"
-
-
 def delete_storage_node(node_id):
     db_controller = DBController()
     snode = db_controller.get_storage_node_by_id(node_id)
@@ -756,7 +919,7 @@ def delete_storage_node(node_id):
 
     snode.remove(db_controller.kv_store)
 
-    for lvol in db_controller.get_lvols():
+    for lvol in db_controller.get_lvols(snode.cluster_id):
         logger.info(f"Sending cluster map to LVol: {lvol.get_id()}")
         lvol_controller.send_cluster_map(lvol.get_id())
 
@@ -764,7 +927,7 @@ def delete_storage_node(node_id):
     logger.info("done")
 
 
-def remove_storage_node(node_id, force_remove=False):
+def remove_storage_node(node_id, force_remove=False, force_migrate=False):
     db_controller = DBController()
     snode = db_controller.get_storage_node_by_id(node_id)
     if not snode:
@@ -811,7 +974,7 @@ def remove_storage_node(node_id, force_remove=False):
                 distr_controller.disconnect_device(dev)
             old_status = dev.status
             dev.status = NVMeDevice.STATUS_FAILED
-            distr_controller.send_dev_status_event(dev.cluster_device_order, NVMeDevice.STATUS_FAILED)
+            distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_FAILED)
             device_events.device_status_change(dev, NVMeDevice.STATUS_FAILED, old_status)
 
     logger.info("Removing storage node")
@@ -825,29 +988,32 @@ def remove_storage_node(node_id, force_remove=False):
         pass
 
     try:
-        snode_api = SNodeClient(snode.api_endpoint)
+        snode_api = SNodeClient(snode.api_endpoint, timeout=20)
         snode_api.spdk_process_kill()
         snode_api.leave_swarm()
+        pci_address = []
+        for dev in snode.nvme_devices:
+            if dev.pcie_address not in pci_address:
+                ret = snode_api.delete_dev_gpt_partitions(dev.pcie_address)
+                logger.debug(ret)
+                pci_address.append(dev.pcie_address)
     except Exception as e:
-        logger.warning(f"Failed to remove SPDK process: {e}")
+        logger.exception(e)
 
     old_status = snode.status
     snode.status = StorageNode.STATUS_REMOVED
     snode.write_to_db(db_controller.kv_store)
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode.get_id(), snode.status)
+    distr_controller.send_node_status_event(snode, snode.status)
     storage_events.snode_status_change(snode, StorageNode.STATUS_REMOVED, old_status)
     logger.info("done")
 
 
 def restart_storage_node(
-        node_id,
-        spdk_cpu_mask=None,
-        spdk_mem=None,
+        node_id, max_lvol=0, max_snap=0, max_prov="",
         spdk_image=None,
         set_spdk_debug=None,
-        small_pool_count=0, large_pool_count=0,
-        small_bufsize=0, large_bufsize=0):
+        small_bufsize=0, large_bufsize=0, number_of_devices=0):
 
     db_controller = DBController()
     kv_store = db_controller.kv_store
@@ -868,7 +1034,7 @@ def restart_storage_node(
     snode.status = StorageNode.STATUS_RESTARTING
     snode.write_to_db(kv_store)
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode.get_id(), snode.status)
+    distr_controller.send_node_status_event(snode, snode.status)
     storage_events.snode_status_change(snode, snode.status, old_status)
 
     logger.info(f"Restarting Storage node: {snode.mgmt_ip}")
@@ -878,18 +1044,67 @@ def restart_storage_node(
     logger.info(f"Node info: {node_info}")
 
     logger.info("Restarting SPDK")
-    cpu = snode.spdk_cpu_mask
-    if spdk_cpu_mask:
-        cpu = spdk_cpu_mask
-        snode.spdk_cpu_mask = cpu
-    mem = snode.spdk_mem
-    if spdk_mem:
-        mem = spdk_mem
-        snode.spdk_mem = mem
+
     img = snode.spdk_image
+    if max_lvol:
+        snode.max_lvol = max_lvol
+    if max_snap:
+        snode.max_snap = max_snap
+    if max_prov:
+        snode.max_prov = max_prov
     if spdk_image:
         img = spdk_image
         snode.spdk_image = img
+
+    # Calculate pool count
+    if snode.ec2_metadata and snode.ec2_metadata.get('instanceType'):
+        supported_type, storage_devices, device_size = utils.get_total_size_per_instance_type(snode.ec2_metadata["instanceType"])
+        if not supported_type:
+            logger.warning(f"Unsupported ec2 instance-type {snode.ec2_metadata['instanceType']} for deployment")
+            if not number_of_devices:
+                logger.error(f"Unsupported ec2 instance-type {snode.ec2_metadata['instanceType']} "
+                             "for deployment, please specify --number-of-devices")
+                return False
+        number_of_devices = storage_devices
+    else:
+        logger.warning("Can not get ec2 instance type for this instance..")
+        if not number_of_devices:
+            if snode.number_of_devices:
+                number_of_devices = snode.number_of_devices
+            else:
+                logger.error("Unsupported instance type please specify --number-of-devices")
+                return False
+
+    snode.number_of_devices = number_of_devices
+
+    number_of_split = snode.num_partitions_per_dev if snode.num_partitions_per_dev else snode.num_partitions_per_dev + 1
+    number_of_alceml_devices = number_of_devices * number_of_split
+    small_pool_count, large_pool_count = utils.calculate_pool_count(
+        number_of_alceml_devices, snode.max_lvol, snode.max_snap, snode.cpu, len(snode.nvme_pollers_cores) or snode.cpu)
+
+    # Calculate minimum huge page memory
+    minimum_hp_memory = utils.calculate_minimum_hp_memory(small_pool_count, large_pool_count, snode.max_lvol, snode.max_snap, snode.cpu)
+
+    # Calculate minimum sys memory
+    minimum_sys_memory = utils.calculate_minimum_sys_memory(snode.max_prov)
+
+    # check for memory
+    if "memory_details" in node_info and node_info['memory_details']:
+        memory_details = node_info['memory_details']
+        logger.info("Node Memory info")
+        logger.info(f"Total: {utils.humanbytes(memory_details['total'])}")
+        logger.info(f"Free: {utils.humanbytes(memory_details['free'])}")
+    else:
+        logger.error(f"Cannot get memory info from the ec2 instance.. Exiting")
+
+    satisfied, spdk_mem = utils.calculate_spdk_memory(minimum_hp_memory,
+                                                      minimum_sys_memory,
+                                                      int(memory_details['free']),
+                                                      int(memory_details['huge_total']))
+    if not satisfied:
+        logger.error(f"Not enough memory for the provided max_lvo: {snode.max_lvol}, max_snap: {snode.max_snap}, max_prov: {utils.humanbytes(snode.max_prov)}.. Exiting")
+
+
     spdk_debug = snode.spdk_debug
     if set_spdk_debug:
         spdk_debug = spdk_debug
@@ -897,17 +1112,14 @@ def restart_storage_node(
 
     cluster_docker = utils.get_docker_client(snode.cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-    results, err = snode_api.spdk_process_start(cpu, mem, img, spdk_debug, cluster_ip)
+    results, err = snode_api.spdk_process_start(snode.spdk_cpu_mask, spdk_mem, img, spdk_debug, cluster_ip)
 
     if not results:
         logger.error(f"Failed to start spdk: {err}")
         return False
     time.sleep(3)
 
-    if small_pool_count:
-        snode.iobuf_small_pool_count = small_pool_count
-    if large_pool_count:
-        snode.iobuf_large_pool_count = large_pool_count
+
     if small_bufsize:
         snode.iobuf_small_bufsize = small_bufsize
     if large_bufsize:
@@ -931,13 +1143,41 @@ def restart_storage_node(
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- start spdk framework
+    # 2- set socket implementation options
+    ret = rpc_client.sock_impl_set_options()
+    if not ret:
+        logger.error("Failed socket implement set options")
+        return False
+
+    # 3- set nvme config
+    if snode.pollers_mask:
+        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
+        if not ret:
+            logger.error("Failed to set pollers mask")
+            return False
+
+    # 4- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 3- set nvme bdev options
+    # 5- set app_thread cpu mask
+    if snode.app_thread_mask:
+        ret = rpc_client.thread_get_stats()
+        app_thread_process_id = 0
+        if ret.get("threads"):
+            for entry in ret["threads"]:
+                if entry['name'] == 'app_thread':
+                    app_thread_process_id = entry['id']
+                    break
+
+        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
+        if not ret:
+            logger.error("Failed to set app thread mask")
+            return False
+
+    # 6- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")
@@ -970,22 +1210,23 @@ def restart_storage_node(
         else:
             logger.info(f"Device not found: {db_dev.get_id()}")
             db_dev.status = NVMeDevice.STATUS_REMOVED
-            distr_controller.send_dev_status_event(db_dev.cluster_device_order, "offline")
+            distr_controller.send_dev_status_event(db_dev, db_dev.status)
 
-    for dev in nvme_devs:
-        if dev.serial_number not in known_devices_sn:
-            logger.info(f"New device found: {dev.get_id()}")
-            dev.status = 'new'
-            new_devices.append(dev)
-            snode.nvme_devices.append(dev)
+    # todo: handle new devices
+    # for dev in nvme_devs:
+    #     if dev.serial_number not in known_devices_sn:
+    #         logger.info(f"New device found: {dev.get_id()}")
+    #         dev.status = NVMeDevice.STATUS_NEW
+    #         new_devices.append(dev)
+    #         snode.nvme_devices.append(dev)
 
-    dev_order = get_next_cluster_device_order(db_controller)
-    for index, nvme in enumerate(new_devices):
-        nvme.cluster_device_order = dev_order
-        dev_order += 1
+    # dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
+    # for index, nvme in enumerate(new_devices):
+    #     nvme.cluster_device_order = dev_order
+    #     dev_order += 1
 
     # prepare devices
-    ret = _prepare_cluster_devices(snode, after_restart=True)
+    ret = _prepare_cluster_devices_on_restart(snode)
     if not ret:
         logger.error("Failed to prepare cluster devices")
         return False
@@ -996,7 +1237,7 @@ def restart_storage_node(
 
     # make other nodes connect to the new devices
     logger.info("Make other nodes connect to the node devices")
-    snodes = db_controller.get_storage_nodes()
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
     for node_index, node in enumerate(snodes):
         if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
@@ -1034,20 +1275,23 @@ def restart_storage_node(
     storage_events.snode_status_change(snode, snode.status, old_status)
 
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode.get_id(), NVMeDevice.STATUS_ONLINE)
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
 
     logger.info("Sending devices event updates")
+    logger.info("Starting migration tasks")
     for dev in snode.nvme_devices:
         if dev.status != NVMeDevice.STATUS_ONLINE:
-            logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}")
+            logger.info(f"Device is not online: {dev.get_id()}, status: {dev.status}")
             continue
-        distr_controller.send_dev_status_event(dev.cluster_device_order, NVMeDevice.STATUS_ONLINE)
 
-    logger.info("Sending cluster map to current node")
-    ret = distr_controller.send_cluster_map_to_node(snode)
-    if not ret:
-        return False, "Failed to send cluster map"
-    time.sleep(3)
+        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
+        tasks_controller.add_device_mig_task(dev.get_id())
+
+    # logger.info("Sending cluster map to current node")
+    # ret = distr_controller.send_cluster_map_to_node(snode)
+    # if not ret:
+    #     return False, "Failed to send cluster map"
+    # time.sleep(3)
 
     for lvol_id in snode.lvols:
         lvol = lvol_controller.recreate_lvol(lvol_id, snode)
@@ -1062,9 +1306,12 @@ def restart_storage_node(
     return "Success"
 
 
-def list_storage_nodes(kv_store, is_json):
-    db_controller = DBController(kv_store)
-    nodes = db_controller.get_storage_nodes()
+def list_storage_nodes(is_json, cluster_id=None):
+    db_controller = DBController()
+    if cluster_id:
+        nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+    else:
+        nodes = db_controller.get_storage_nodes()
     data = []
     output = ""
 
@@ -1111,26 +1358,43 @@ def list_storage_devices(kv_store, node_id, sort, is_json):
         logger.error("This storage node is not part of the cluster")
         return False
 
-    data = []
+    storage_devices = []
+    jm_devices = []
+    remote_devices = []
     for device in snode.nvme_devices:
         logger.debug(device)
         logger.debug("*" * 20)
-        data.append({
+        storage_devices.append({
             "UUID": device.uuid,
             "Name": device.device_name,
-            "Hostname": snode.hostname,
             "Size": utils.humanbytes(device.size),
-            # "Sequential Number": device.sequential_number,
-            # "Partitions Count": device.partitions_count,
-            # "Model ID": device.model_id,
             "Serial Number": device.serial_number,
             "PCIe": device.pcie_address,
             "Status": device.status,
             "IO Err": device.io_error,
-            "Health": device.health_check,
+            "Health": device.health_check
+        })
 
+    if snode.jm_device:
+        jm_devices.append({
+            "UUID": snode.jm_device.uuid,
+            "Name": snode.jm_device.device_name,
+            "Size": utils.humanbytes(snode.jm_device.size),
+            "Status": snode.jm_device.status,
+            "IO Err": snode.jm_device.io_error,
+            "Health": snode.jm_device.health_check
         })
 
+    for device in snode.remote_devices:
+        logger.debug(device)
+        logger.debug("*" * 20)
+        remote_devices.append({
+            "UUID": device.uuid,
+            "Name": device.device_name,
+            "Size": utils.humanbytes(device.size),
+            "Serial Number": device.serial_number,
+            "Node ID": device.node_id,
+        })
     if sort and sort in ['node-seq', 'dev-seq', 'serial']:
         if sort == 'serial':
             sort_key = "Serial Number"
@@ -1139,13 +1403,20 @@ def list_storage_devices(kv_store, node_id, sort, is_json):
         elif sort == 'node-seq':
             # TODO: check this key
             sort_key = "Sequential Number"
-        sorted_data = sorted(data, key=lambda d: d[sort_key])
-        data = sorted_data
+        storage_devices = sorted(storage_devices, key=lambda d: d[sort_key])
 
+    data = {
+        "Storage Devices": storage_devices,
+        "JM Devices": jm_devices,
+        "Remote Devices": remote_devices,
+    }
     if is_json:
         return json.dumps(data, indent=2)
     else:
-        return utils.print_table(data)
+        out = ""
+        for d in data:
+            out += f"{d}\n{utils.print_table(data[d])}\n\n"
+        return out
 
 
 def shutdown_storage_node(node_id, force=False):
@@ -1186,7 +1457,7 @@ def shutdown_storage_node(node_id, force=False):
     for dev in snode.nvme_devices:
         if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
             device_controller.device_set_unavailable(dev.get_id())
-    distr_controller.send_node_status_event(snode.get_id(), "in_shutdown")
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_IN_SHUTDOWN)
 
     # shutdown node
     # make other nodes disconnect from this node
@@ -1206,7 +1477,7 @@ def shutdown_storage_node(node_id, force=False):
     snode_api = SNodeClient(snode.api_endpoint)
     results, err = snode_api.spdk_process_kill()
 
-    distr_controller.send_node_status_event(snode.get_id(), StorageNode.STATUS_OFFLINE)
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_OFFLINE)
 
     logger.info("Setting node status to offline")
     snode = db_controller.get_storage_node_by_id(node_id)
@@ -1233,22 +1504,24 @@ def suspend_storage_node(node_id, force=False):
         return False
 
     cluster = db_controller.get_cluster_by_id(snode.cluster_id)
-    snodes = db_controller.get_storage_nodes()
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
     online_nodes = 0
     for node in snodes:
         if node.status == node.STATUS_ONLINE:
             online_nodes += 1
-    if cluster.ha_type == "ha" and online_nodes <= 3 and cluster.status == cluster.STATUS_ACTIVE:
-        logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
-        if force is False:
-            return False
 
-    if cluster.ha_type == "ha" and cluster.status == cluster.STATUS_DEGRADED and force is False:
-        logger.warning(f"Cluster status is degraded, use --force but this will suspend the cluster")
-        return False
+    if cluster.ha_type == "ha":
+        if online_nodes <= 3 and cluster.status == cluster.STATUS_ACTIVE:
+            logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
+            if force is False:
+                return False
+
+        if cluster.status == cluster.STATUS_DEGRADED and force is False:
+            logger.warning(f"Cluster status is degraded, use --force but this will suspend the cluster")
+            return False
 
     logger.info("Suspending node")
-    distr_controller.send_node_status_event(snode.get_id(), "suspended")
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_SUSPENDED)
     for dev in snode.nvme_devices:
         if dev.status == NVMeDevice.STATUS_ONLINE:
             device_controller.device_set_unavailable(dev.get_id())
@@ -1292,7 +1565,7 @@ def resume_storage_node(node_id):
     logger.info("Resuming node")
 
     logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode.get_id(), "online")
+    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
 
     for dev in snode.nvme_devices:
         if dev.status == NVMeDevice.STATUS_UNAVAILABLE:
@@ -1668,7 +1941,6 @@ def deploy_cleaner():
     return True
 
 
-
 def get_host_secret(node_id):
     db_controller = DBController()
     node = db_controller.get_storage_node_by_id(node_id)
@@ -1831,7 +2103,7 @@ def set_node_status(node_id, status):
         snode.updated_at = str(datetime.datetime.now())
         snode.write_to_db(db_controller.kv_store)
         storage_events.snode_status_change(snode, snode.status, old_status, caused_by="monitor")
-        distr_controller.send_node_status_event(snode.get_id(), status)
+        distr_controller.send_node_status_event(snode, status)
 
     if snode.status == StorageNode.STATUS_ONLINE:
         logger.info("Connecting to remote devices")