sbcli-pre 1.2.3__zip → 1.2.5__zip

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/storage_node_ops.py

@@ -14,11 +14,11 @@ import docker
 from simplyblock_core import constants, scripts, distr_controller
 from simplyblock_core import utils
 from simplyblock_core.controllers import lvol_controller, storage_events, snapshot_controller, device_events, \
-    device_controller, tasks_controller
+    device_controller
 from simplyblock_core.kv_store import DBController
 from simplyblock_core import shell_utils
 from simplyblock_core.models.iface import IFace
-from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
+from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.pci_utils import get_nvme_devices, bind_spdk_driver
 from simplyblock_core.rpc_client import RPCClient
@@ -81,50 +81,55 @@ def _get_if_ip_address(ifname):
 
 
 def addNvmeDevices(cluster, rpc_client, devs, snode):
+    sequential_number = 0
     devices = []
     ret = rpc_client.bdev_nvme_controller_list()
-    ctr_map = {}
-    try:
-        if ret:
-            ctr_map = {i["ctrlrs"][0]['trid']['traddr']: i["name"] for i in ret}
-    except:
-        pass
+    if ret:
+        ctr_map = {i["ctrlrs"][0]['trid']['traddr']: i["name"] for i in ret}
+    else:
+        ctr_map = {}
 
-    next_physical_label = get_next_physical_device_order()
     for index, pcie in enumerate(devs):
 
         if pcie in ctr_map:
-            nvme_controller = ctr_map[pcie]
+            nvme_bdev = ctr_map[pcie] + "n1"
         else:
-            nvme_controller = "nvme_%s" % index
-            ret, err = rpc_client.bdev_nvme_controller_attach(nvme_controller, pcie)
+            name = "nvme_%s" % index
+            ret, err = rpc_client.bdev_nvme_controller_attach(name, pcie)
             time.sleep(2)
+            nvme_bdev = f"{name}n1"
 
-        nvme_bdev = f"{nvme_controller}n1"
-        rpc_client.bdev_examine(nvme_bdev)
-        time.sleep(5)
         ret = rpc_client.get_bdevs(nvme_bdev)
-        nvme_dict = ret[0]
-        nvme_driver_data = nvme_dict['driver_specific']['nvme'][0]
-        model_number = nvme_driver_data['ctrlr_data']['model_number']
-        total_size = nvme_dict['block_size'] * nvme_dict['num_blocks']
+        if ret:
+            nvme_dict = ret[0]
+            nvme_driver_data = nvme_dict['driver_specific']['nvme'][0]
+            model_number = nvme_driver_data['ctrlr_data']['model_number']
 
-        devices.append(
-            NVMeDevice({
-                'uuid': str(uuid.uuid4()),
-                'device_name': nvme_dict['name'],
-                'size': total_size,
-                'physical_label': next_physical_label,
-                'pcie_address': nvme_driver_data['pci_address'],
-                'model_id': model_number,
-                'serial_number': nvme_driver_data['ctrlr_data']['serial_number'],
-                'nvme_bdev': nvme_bdev,
-                'nvme_controller': nvme_controller,
-                'node_id': snode.get_id(),
-                'cluster_id': snode.cluster_id,
-                'status': NVMeDevice.STATUS_ONLINE
-        }))
-        next_physical_label += 1
+            size = nvme_dict['block_size'] * nvme_dict['num_blocks']
+            device_partitions_count = int(size / (cluster.blk_size * cluster.page_size_in_blocks))
+            devices.append(
+                NVMeDevice({
+                    'uuid': str(uuid.uuid4()),
+                    'device_name': nvme_dict['name'],
+                    'sequential_number': sequential_number,
+                    'partitions_count': device_partitions_count,
+                    'capacity': size,
+                    'size': size,
+                    'pcie_address': nvme_driver_data['pci_address'],
+                    'model_id': model_number,
+                    'serial_number': nvme_driver_data['ctrlr_data']['serial_number'],
+                    'nvme_bdev': nvme_bdev,
+                    'alloc_bdev': nvme_bdev,
+                    'node_id': snode.get_id(),
+                    'cluster_id': snode.cluster_id,
+
+                    # 'nvmf_nqn': subsystem_nqn,
+                    # 'nvmf_ip': IP,
+                    # 'nvmf_port': 4420,
+
+                    'status': 'online'
+                }))
+            sequential_number += device_partitions_count
     return devices
 
 
@@ -192,10 +197,10 @@ def _run_nvme_smart_log_add(dev_name):
     return data
 
 
-def get_next_cluster_device_order(db_controller, cluster_id):
+def get_next_cluster_device_order(db_controller):
     max_order = 0
     found = False
-    for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id):
+    for node in db_controller.get_storage_nodes():
         for dev in node.nvme_devices:
             found = True
             max_order = max(max_order, dev.cluster_device_order)
@@ -204,320 +209,92 @@ def get_next_cluster_device_order(db_controller, cluster_id):
     return 0
 
 
-def get_next_physical_device_order():
+def _prepare_cluster_devices(snode, after_restart=False):
     db_controller = DBController()
-    max_order = 0
-    found = False
-    for node in db_controller.get_storage_nodes():
-        for dev in node.nvme_devices:
-            found = True
-            max_order = max(max_order, dev.physical_label)
-    if found:
-        return max_order + 1
-    return 0
-
-
-def _search_for_partitions(rpc_client, nvme_device):
-    partitioned_devices = []
-    for bdev in rpc_client.get_bdevs():
-        name = bdev['name']
-        if name.startswith(f"{nvme_device.nvme_bdev}p"):
-            new_dev = NVMeDevice(nvme_device.to_dict())
-            new_dev.uuid = str(uuid.uuid4())
-            new_dev.device_name = name
-            new_dev.nvme_bdev = name
-            new_dev.size = bdev['block_size'] * bdev['num_blocks']
-            partitioned_devices.append(new_dev)
-    return partitioned_devices
-
 
-def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
-    raid_bdev = f"raid_jm_{snode.get_id()}"
-    ret = rpc_client.bdev_raid_create(raid_bdev, jm_nvme_bdevs)
-    if not ret:
-        logger.error(f"Failed to create raid_jm_{snode.get_id()}")
-        return False
-    alceml_name = f"alceml_jm_{snode.get_id()}"
-    pba_init_mode = 3
-    if after_restart:
-        pba_init_mode = 2
-    ret = rpc_client.bdev_alceml_create(alceml_name, raid_bdev, str(uuid.uuid4()), pba_init_mode=pba_init_mode)
-    if not ret:
-        logger.error(f"Failed to create alceml bdev: {alceml_name}")
-        return False
-
-    jm_bdev = f"jm_{snode.get_id()}"
-    ret = rpc_client.bdev_jm_create(jm_bdev, alceml_name)
-    if not ret:
-        logger.error(f"Failed to create {jm_bdev}")
-        return False
-    ret = rpc_client.get_bdevs(raid_bdev)
-
-    return JMDevice({
-        'uuid': str(uuid.uuid4()),
-        'device_name': jm_bdev,
-        'size': ret[0]["block_size"] * ret[0]["num_blocks"],
-        'status': JMDevice.STATUS_ONLINE,
-        'jm_nvme_bdev_list': jm_nvme_bdevs,
-        'raid_bdev': raid_bdev,
-        'alceml_bdev': alceml_name,
-        'jm_bdev': jm_bdev
-    })
-
-
-def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
-
-    alceml_id = nvme.get_id()
-    alceml_name = device_controller.get_alceml_name(alceml_id)
-    logger.info(f"adding {alceml_name}")
-
-    pba_init_mode = 3
-    if after_restart:
-        pba_init_mode = 2
-    ret = rpc_client.bdev_alceml_create(alceml_name, nvme.nvme_bdev, alceml_id, pba_init_mode=pba_init_mode)
-    if not ret:
-        logger.error(f"Failed to create alceml bdev: {alceml_name}")
-        return False
-
-    jm_bdev = f"jm_{snode.get_id()}"
-    ret = rpc_client.bdev_jm_create(jm_bdev, alceml_name)
-    if not ret:
-        logger.error(f"Failed to create {jm_bdev}")
-        return False
-
-    return JMDevice({
-        'uuid': alceml_id,
-        'device_name': jm_bdev,
-        'size': nvme.size,
-        'status': JMDevice.STATUS_ONLINE,
-        'alceml_bdev': alceml_name,
-        'nvme_bdev': nvme.nvme_bdev,
-        'jm_bdev': jm_bdev
-    })
-
-
-def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
-    test_name = f"{nvme.nvme_bdev}_test"
-    ret = rpc_client.bdev_passtest_create(test_name, nvme.nvme_bdev)
-    if not ret:
-        logger.error(f"Failed to create passtest bdev {test_name}")
-        return False
-    alceml_id = nvme.get_id()
-    alceml_name = device_controller.get_alceml_name(alceml_id)
-    logger.info(f"adding {alceml_name}")
-    pba_init_mode = 3
-    if after_restart:
-        pba_init_mode = 2
-    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode,
-                                            dev_cpu_mask=snode.dev_cpu_mask)
-    if not ret:
-        logger.error(f"Failed to create alceml bdev: {alceml_name}")
-        return False
-
-    # add pass through
-    pt_name = f"{alceml_name}_PT"
-    ret = rpc_client.bdev_PT_NoExcl_create(pt_name, alceml_name)
-    if not ret:
-        logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
-        return False
-
-    subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
-    logger.info("creating subsystem %s", subsystem_nqn)
-    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
-    IP = None
-    for iface in snode.data_nics:
-        if iface.ip4_address:
-            tr_type = iface.get_transport_type()
-            ret = rpc_client.transport_list()
-            found = False
-            if ret:
-                for ty in ret:
-                    if ty['trtype'] == tr_type:
-                        found = True
-            if found is False:
-                ret = rpc_client.transport_create(tr_type)
-            logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
-            ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, "4420")
-            IP = iface.ip4_address
-            break
-    logger.info(f"add {pt_name} to subsystem")
-    ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
-    if not ret:
-        logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
-        return False
-
-    nvme.testing_bdev = test_name
-    nvme.alceml_bdev = alceml_name
-    nvme.pt_bdev = pt_name
-    nvme.nvmf_nqn = subsystem_nqn
-    nvme.nvmf_ip = IP
-    nvme.nvmf_port = 4420
-    nvme.io_error = False
-    nvme.status = NVMeDevice.STATUS_ONLINE
-    return nvme
-
-
-def _create_device_partitions(rpc_client, nvme, snode):
-    nbd_device = rpc_client.nbd_start_disk(nvme.nvme_bdev)
-    time.sleep(3)
-    if not nbd_device:
-        logger.error(f"Failed to start nbd dev")
-        return False
-    snode_api = SNodeClient(snode.api_endpoint)
-    result, error = snode_api.make_gpt_partitions(
-        nbd_device, snode.jm_percent, snode.num_partitions_per_dev)
-    if error:
-        logger.error(f"Failed to make partitions")
-        logger.error(error)
-        return False
-    time.sleep(3)
-    rpc_client.nbd_stop_disk(nbd_device)
-    time.sleep(1)
-    rpc_client.bdev_nvme_detach_controller(nvme.nvme_controller)
-    time.sleep(1)
-    rpc_client.bdev_nvme_controller_attach(nvme.nvme_controller, nvme.pcie_address)
-    time.sleep(1)
-    rpc_client.bdev_examine(nvme.nvme_bdev)
-    time.sleep(1)
-    return True
-
-
-def _prepare_cluster_devices_partitions(snode, devices):
-    db_controller = DBController()
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)
 
-    new_devices = []
-    jm_devices = []
-    dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
-    for index, nvme in enumerate(devices):
-        if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY]:
-            logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}")
-            continue
-
-        # look for partitions
-        partitioned_devices = _search_for_partitions(rpc_client, nvme)
-        logger.debug("partitioned_devices")
-        logger.debug(partitioned_devices)
-        if len(partitioned_devices) == (1 + snode.num_partitions_per_dev):
-            logger.info("Partitioned devices found")
-        else:
-            logger.info(f"Creating partitions for {nvme.nvme_bdev}")
-            _create_device_partitions(rpc_client, nvme, snode)
-            partitioned_devices = _search_for_partitions(rpc_client, nvme)
-            if len(partitioned_devices) == (1 + snode.num_partitions_per_dev):
-                logger.info("Device partitions created")
-            else:
-                logger.error("Failed to create partitions")
-                return False
-
-        jm_devices.append(partitioned_devices.pop(0))
-
-        for dev in partitioned_devices:
-            new_device = _create_storage_device_stack(rpc_client, dev, snode, after_restart=False)
-            if not new_device:
-                logger.error("failed to create dev stack")
-                return False
-            new_device.cluster_device_order = dev_order
-            dev_order += 1
-            new_devices.append(new_device)
-            device_events.device_create(new_device)
-
-    snode.nvme_devices = new_devices
-
-    if jm_devices:
-        jm_nvme_bdevs = [dev.nvme_bdev for dev in jm_devices]
-        jm_device = _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=False)
-        if not jm_device:
-            logger.error(f"Failed to create JM device")
-            return False
-        snode.jm_device = jm_device
-
-    return True
-
-
-def _prepare_cluster_devices_jm_on_dev(snode, devices):
-    db_controller = DBController()
-
-    jm_device = devices[0]
-    # Set device cluster order
-    dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
-    for index, nvme in enumerate(devices):
-        nvme.cluster_device_order = dev_order
-        dev_order += 1
-        if nvme.size < jm_device.size:
-            jm_device = nvme
-        device_events.device_create(nvme)
-    jm_device.status = NVMeDevice.STATUS_JM
-
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-
-    new_devices = []
-    for index, nvme in enumerate(devices):
+    for index, nvme in enumerate(snode.nvme_devices):
         if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE,
                                NVMeDevice.STATUS_JM, NVMeDevice.STATUS_READONLY]:
             logger.debug(f"Device is not online or unavailable: {nvme.get_id()}, status: {nvme.status}")
             continue
 
-        if nvme.status == NVMeDevice.STATUS_JM:
-            jm_device = _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart=False)
-            if not jm_device:
-                logger.error(f"Failed to create JM device")
-                return False
-            snode.jm_device = jm_device
-        else:
-            new_device = _create_storage_device_stack(rpc_client, nvme, snode, after_restart=False)
-            if not new_device:
-                logger.error("failed to create dev stack")
-                return False
-            new_device.cluster_device_order = dev_order
-            dev_order += 1
-            new_devices.append(new_device)
-            device_events.device_create(new_device)
-
-    snode.nvme_devices = new_devices
-    return True
-
-
-def _prepare_cluster_devices_on_restart(snode):
-    db_controller = DBController()
-
-    rpc_client = RPCClient(
-        snode.mgmt_ip, snode.rpc_port,
-        snode.rpc_username, snode.rpc_password)
-
-    for index, nvme in enumerate(snode.nvme_devices):
-        if nvme.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY]:
-            logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}")
-            continue
-
-        dev = _create_storage_device_stack(rpc_client, nvme, snode, after_restart=True)
-        if not dev:
-            logger.error(f"Failed to create dev stack {nvme.get_id()}")
+        test_name = f"{nvme.nvme_bdev}_test"
+        # create testing bdev
+        ret = rpc_client.bdev_passtest_create(test_name, nvme.nvme_bdev)
+        if not ret:
+            logger.error(f"Failed to create bdev: {test_name}")
             return False
-        device_events.device_restarted(dev)
-
-    # prepare JM device
-    jm_device = snode.jm_device
-    if jm_device.jm_nvme_bdev_list:
-        ret = _create_jm_stack_on_raid(rpc_client, jm_device.jm_nvme_bdev_list, snode, after_restart=False)
+        alceml_id = nvme.get_id()
+        alceml_name = device_controller.get_alceml_name(alceml_id)
+        logger.info(f"adding {alceml_name}")
+        pba_init_mode = 3
+        if after_restart:
+            pba_init_mode = 2
+        ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode)
         if not ret:
-            logger.error(f"Failed to create JM device")
+            logger.error(f"Failed to create alceml bdev: {alceml_name}")
             return False
-    else:
 
-        ret = rpc_client.bdev_alceml_create(jm_device.alceml_bdev, jm_device.nvme_bdev, jm_device.get_id(), pba_init_mode=2)
+        # create jm
+        if nvme.jm_bdev:
+            ret = rpc_client.bdev_jm_create(nvme.jm_bdev, alceml_name)
+            if not ret:
+                logger.error(f"Failed to create JM bdev: {nvme.jm_bdev}")
+                return False
+            nvme.testing_bdev = test_name
+            nvme.alceml_bdev = alceml_name
+            nvme.io_error = True
+            nvme.status = NVMeDevice.STATUS_JM
+            continue
+
+        # add pass through
+        pt_name = f"{alceml_name}_PT"
+        ret = rpc_client.bdev_PT_NoExcl_create(pt_name, alceml_name)
         if not ret:
-            logger.error(f"Failed to create alceml bdev: {jm_device.alceml_bdev}")
+            logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
             return False
 
-        jm_bdev = f"jm_{snode.get_id()}"
-        ret = rpc_client.bdev_jm_create(jm_bdev, jm_device.alceml_bdev)
+        subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
+        logger.info("creating subsystem %s", subsystem_nqn)
+        ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
+        IP = None
+        for iface in snode.data_nics:
+            if iface.ip4_address:
+                tr_type = iface.get_transport_type()
+                ret = rpc_client.transport_list()
+                found = False
+                if ret:
+                    for ty in ret:
+                        if ty['trtype'] == tr_type:
+                            found = True
+                if found is False:
+                    ret = rpc_client.transport_create(tr_type)
+                logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
+                ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, "4420")
+                IP = iface.ip4_address
+                break
+        logger.info(f"add {pt_name} to subsystem")
+        ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
         if not ret:
-            logger.error(f"Failed to create {jm_bdev}")
+            logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
             return False
 
+        nvme.testing_bdev = test_name
+        nvme.alceml_bdev = alceml_name
+        nvme.pt_bdev = pt_name
+        nvme.nvmf_nqn = subsystem_nqn
+        nvme.nvmf_ip = IP
+        nvme.nvmf_port = 4420
+        nvme.io_error = False
+        old_status = nvme.status
+        nvme.status = NVMeDevice.STATUS_ONLINE
+        device_events.device_status_change(nvme, nvme.status, old_status)
+        snode.write_to_db(db_controller.kv_store)
+
     return True
 
 
@@ -530,7 +307,7 @@ def _connect_to_remote_devs(this_node):
 
     remote_devices = []
     # connect to remote devs
-    snodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id)
+    snodes = db_controller.get_storage_nodes()
     for node_index, node in enumerate(snodes):
         if node.get_id() == this_node.get_id() or node.status == node.STATUS_OFFLINE:
             continue
@@ -549,10 +326,9 @@ def _connect_to_remote_devs(this_node):
     return remote_devices
 
 
-def add_node(cluster_id, node_ip, iface_name, data_nics_list,
+def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
              spdk_mem, spdk_image=None, spdk_debug=False,
-             small_pool_count=0, large_pool_count=0, small_bufsize=0, large_bufsize=0,
-             num_partitions_per_dev=0, jm_percent=0):
+             small_pool_count=0, large_pool_count=0, small_bufsize=0, large_bufsize=0, jm_device_pcie=None):
     db_controller = DBController()
     kv_store = db_controller.kv_store
 
@@ -623,27 +399,6 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
             spdk_mem = huge_free
             logger.info(f"Using the free hugepages for spdk memory: {utils.humanbytes(huge_free)}")
 
-    # Tune cpu maks parameters
-    cpu_count = node_info["cpu_count"]
-    pollers_mask = ""
-    app_thread_mask = ""
-    dev_cpu_mask = ""
-    if cpu_count < 8:
-        mask = (1 << (cpu_count - 1)) - 1
-        mask <<= 1
-        spdk_cpu_mask = f'0x{mask:X}'
-        os_cores = [0]
-    else:
-        os_cores, nvme_pollers_cores, app_thread_core, dev_cpu_cores = \
-            utils.calculate_core_allocation(cpu_count)
-        spdk_cores = nvme_pollers_cores + app_thread_core + dev_cpu_cores
-
-        pollers_mask = utils.generate_mask(nvme_pollers_cores)
-        app_thread_mask = utils.generate_mask(app_thread_core)
-        spdk_cpu_mask = utils.generate_mask(spdk_cores)
-        dev_cpu_mask = utils.generate_mask(dev_cpu_cores)
-
-
     logger.info("Joining docker swarm...")
     cluster_docker = utils.get_docker_client(cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
@@ -721,19 +476,12 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     snode.spdk_image = spdk_image or ""
     snode.spdk_debug = spdk_debug or 0
     snode.write_to_db(kv_store)
-    snode.app_thread_mask = app_thread_mask or ""
-    snode.pollers_mask = pollers_mask or ""
-    snode.dev_cpu_mask = dev_cpu_mask or ""
-    snode.os_cores = os_cores or []
 
     snode.iobuf_small_pool_count = small_pool_count or 0
     snode.iobuf_large_pool_count = large_pool_count or 0
     snode.iobuf_small_bufsize = small_bufsize or 0
     snode.iobuf_large_bufsize = large_bufsize or 0
 
-    snode.num_partitions_per_dev = num_partitions_per_dev
-    snode.jm_percent = jm_percent
-
     snode.write_to_db(kv_store)
 
     # creating RPCClient instance
@@ -751,41 +499,13 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- set socket implementation options
-    ret = rpc_client.sock_impl_set_options()
-    if not ret:
-        logger.error("Failed socket implement set options")
-        return False
-
-    # 3- set nvme config
-    if snode.pollers_mask:
-        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
-        if not ret:
-            logger.error("Failed to set pollers mask")
-            return False
-
-    # 4- start spdk framework
+    # 2- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 5- set app_thread cpu mask
-    if snode.app_thread_mask:
-        ret = rpc_client.thread_get_stats()
-        app_thread_process_id = 0
-        if ret.get("threads"):
-            for entry in ret["threads"]:
-                if entry['name'] == 'app_thread':
-                    app_thread_process_id = entry['id']
-                    break
-
-        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
-        if not ret:
-            logger.error("Failed to set app thread mask")
-            return False
-
-    # 6- set nvme bdev options
+    # 3- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")
@@ -793,18 +513,36 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
 
     # get new node info after starting spdk
     node_info, _ = snode_api.info()
-
-    # discover devices
+    # adding devices
     nvme_devs = addNvmeDevices(cluster, rpc_client, node_info['spdk_pcie_list'], snode)
     if not nvme_devs:
         logger.error("No NVMe devices was found!")
         return False
 
+    snode.nvme_devices = nvme_devs
+
+    jm_device = snode.nvme_devices[0]
+    # Set device cluster order
+    dev_order = get_next_cluster_device_order(db_controller)
+    for index, nvme in enumerate(snode.nvme_devices):
+        nvme.cluster_device_order = dev_order
+        dev_order += 1
+        if jm_device_pcie:
+            if nvme.pcie_address == jm_device_pcie:
+                jm_device = nvme
+        elif nvme.size < jm_device.size:
+            jm_device = nvme
+        device_events.device_create(nvme)
+
+    # create jm
+    logger.info(f"Using device for JM: {jm_device.get_id()}")
+    jm_device.jm_bdev = f"jm_{snode.get_id()}"
+
+    # save object
+    snode.write_to_db(db_controller.kv_store)
+
     # prepare devices
-    if snode.num_partitions_per_dev == 0 or snode.jm_percent == 0:
-        ret = _prepare_cluster_devices_jm_on_dev(snode, nvme_devs)
-    else:
-        ret = _prepare_cluster_devices_partitions(snode, nvme_devs)
+    ret = _prepare_cluster_devices(snode)
     if not ret:
         logger.error("Failed to prepare cluster devices")
         return False
@@ -819,7 +557,7 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
 
     # make other nodes connect to the new devices
     logger.info("Make other nodes connect to the new devices")
-    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+    snodes = db_controller.get_storage_nodes()
     for node_index, node in enumerate(snodes):
         if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
@@ -861,16 +599,150 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     time.sleep(3)
 
     logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
+    distr_controller.send_node_status_event(snode.get_id(), "online")
 
     for dev in snode.nvme_devices:
-        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
+        distr_controller.send_dev_status_event(dev.cluster_device_order, "online")
 
     storage_events.snode_add(snode)
     logger.info("Done")
     return "Success"
 
 
+# Deprecated
+def add_storage_node(cluster_id, iface_name, data_nics):
+    db_controller = DBController()
+    kv_store = db_controller.kv_store
+
+    cluster = db_controller.get_cluster_by_id(cluster_id)
+    if not cluster:
+        logger.error("Cluster not found: %s", cluster_id)
+        return False
+
+    logger.info("Add Storage node")
+
+    hostname = utils.get_hostname()
+    snode = db_controller.get_storage_node_by_hostname(hostname)
+    if snode:
+        logger.error("Node already exists, try remove it first.")
+        exit(1)
+    else:
+        snode = StorageNode()
+        snode.uuid = str(uuid.uuid4())
+
+    mgmt_ip = _get_if_ip_address(iface_name)
+    system_id = utils.get_system_id()
+
+    BASE_NQN = cluster.nqn.split(":")[0]
+    subsystem_nqn = f"{BASE_NQN}:{hostname}"
+
+    if data_nics:
+        data_nics = _get_data_nics(data_nics)
+    else:
+        data_nics = _get_data_nics([iface_name])
+
+    rpc_user, rpc_pass = utils.generate_rpc_user_and_pass()
+
+    # creating storage node object
+    snode.status = StorageNode.STATUS_IN_CREATION
+    snode.baseboard_sn = utils.get_baseboard_sn()
+    snode.system_uuid = system_id
+    snode.hostname = hostname
+    snode.host_nqn = subsystem_nqn
+    snode.subsystem = subsystem_nqn
+    snode.data_nics = data_nics
+    snode.mgmt_ip = mgmt_ip
+    snode.rpc_port = constants.RPC_HTTP_PROXY_PORT
+    snode.rpc_username = rpc_user
+    snode.rpc_password = rpc_pass
+    snode.cluster_id = cluster_id
+    snode.write_to_db(kv_store)
+
+    # creating RPCClient instance
+    rpc_client = RPCClient(
+        snode.mgmt_ip,
+        snode.rpc_port,
+        snode.rpc_username,
+        snode.rpc_password)
+
+    logger.info("Getting nvme devices")
+    devs = get_nvme_devices()
+    logger.debug(devs)
+    pcies = [d[0] for d in devs]
+    nvme_devs = addNvmeDevices(cluster, rpc_client, pcies, snode)
+    if not nvme_devs:
+        logger.error("No NVMe devices was found!")
+
+    logger.debug(nvme_devs)
+    snode.nvme_devices = nvme_devs
+
+    # Set device cluster order
+    dev_order = get_next_cluster_device_order(db_controller)
+    for index, nvme in enumerate(snode.nvme_devices):
+        nvme.cluster_device_order = dev_order
+        dev_order += 1
+    snode.write_to_db(db_controller.kv_store)
+
+    # prepare devices
+    _prepare_cluster_devices(snode)
+
+    logger.info("Connecting to remote devices")
+    remote_devices = _connect_to_remote_devs(snode)
+    snode.remote_devices = remote_devices
+
+    logger.info("Setting node status to Active")
+    snode.status = StorageNode.STATUS_ONLINE
+    snode.write_to_db(kv_store)
+
+    # make other nodes connect to the new devices
+    logger.info("Make other nodes connect to the new devices")
+    snodes = db_controller.get_storage_nodes()
+    for node_index, node in enumerate(snodes):
+        if node.get_id() == snode.get_id():
+            continue
+        logger.info(f"Connecting to node: {node.get_id()}")
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        count = 0
+        for dev in snode.nvme_devices:
+            name = f"remote_{dev.alceml_bdev}"
+            ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
+            if not ret:
+                logger.error(f"Failed to connect to device: {name}")
+                continue
+
+            dev.remote_bdev = f"{name}n1"
+            idx = -1
+            for i, d in enumerate(node.remote_devices):
+                if d.get_id() == dev.get_id():
+                    idx = i
+                    break
+            if idx >= 0:
+                node.remote_devices[idx] = dev
+            else:
+                node.remote_devices.append(dev)
+            count += 1
+        node.write_to_db(kv_store)
+        logger.info(f"connected to devices count: {count}")
+
+    logger.info("Sending cluster map")
+    ret = distr_controller.send_cluster_map_to_node(snode)
+    if not ret:
+        return False, "Failed to send cluster map"
+    ret = distr_controller.send_cluster_map_add_node(snode)
+    if not ret:
+        return False, "Failed to send cluster map add node"
+    time.sleep(3)
+
+    logger.info("Sending cluster event updates")
+    distr_controller.send_node_status_event(snode.get_id(), "online")
+
+    for dev in snode.nvme_devices:
+        distr_controller.send_dev_status_event(dev.cluster_device_order, "online")
+
+    logger.info("Done")
+    return "Success"
+
+
 def delete_storage_node(node_id):
     db_controller = DBController()
     snode = db_controller.get_storage_node_by_id(node_id)
@@ -884,7 +756,7 @@ def delete_storage_node(node_id):
 
     snode.remove(db_controller.kv_store)
 
-    for lvol in db_controller.get_lvols(snode.cluster_id):
+    for lvol in db_controller.get_lvols():
         logger.info(f"Sending cluster map to LVol: {lvol.get_id()}")
         lvol_controller.send_cluster_map(lvol.get_id())
 
@@ -892,7 +764,7 @@ def delete_storage_node(node_id):
     logger.info("done")
 
 
-def remove_storage_node(node_id, force_remove=False, force_migrate=False):
+def remove_storage_node(node_id, force_remove=False):
     db_controller = DBController()
     snode = db_controller.get_storage_node_by_id(node_id)
     if not snode:
@@ -939,7 +811,7 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
                 distr_controller.disconnect_device(dev)
             old_status = dev.status
             dev.status = NVMeDevice.STATUS_FAILED
-            distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_FAILED)
+            distr_controller.send_dev_status_event(dev.cluster_device_order, NVMeDevice.STATUS_FAILED)
             device_events.device_status_change(dev, NVMeDevice.STATUS_FAILED, old_status)
 
     logger.info("Removing storage node")
@@ -953,29 +825,24 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
         pass
 
     try:
-        snode_api = SNodeClient(snode.api_endpoint, timeout=20)
+        snode_api = SNodeClient(snode.api_endpoint)
         snode_api.spdk_process_kill()
         snode_api.leave_swarm()
-        pci_address = []
-        for dev in snode.nvme_devices:
-            if dev.pcie_address not in pci_address:
-                ret = snode_api.delete_dev_gpt_partitions(dev.pcie_address)
-                logger.debug(ret)
-                pci_address.append(dev.pcie_address)
     except Exception as e:
-        logger.exception(e)
+        logger.warning(f"Failed to remove SPDK process: {e}")
 
     old_status = snode.status
     snode.status = StorageNode.STATUS_REMOVED
     snode.write_to_db(db_controller.kv_store)
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode, snode.status)
+    distr_controller.send_node_status_event(snode.get_id(), snode.status)
     storage_events.snode_status_change(snode, StorageNode.STATUS_REMOVED, old_status)
     logger.info("done")
 
 
 def restart_storage_node(
         node_id,
+        spdk_cpu_mask=None,
         spdk_mem=None,
         spdk_image=None,
         set_spdk_debug=None,
@@ -1001,7 +868,7 @@ def restart_storage_node(
     snode.status = StorageNode.STATUS_RESTARTING
     snode.write_to_db(kv_store)
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode, snode.status)
+    distr_controller.send_node_status_event(snode.get_id(), snode.status)
     storage_events.snode_status_change(snode, snode.status, old_status)
 
     logger.info(f"Restarting Storage node: {snode.mgmt_ip}")
@@ -1011,6 +878,10 @@ def restart_storage_node(
     logger.info(f"Node info: {node_info}")
 
     logger.info("Restarting SPDK")
+    cpu = snode.spdk_cpu_mask
+    if spdk_cpu_mask:
+        cpu = spdk_cpu_mask
+        snode.spdk_cpu_mask = cpu
     mem = snode.spdk_mem
     if spdk_mem:
         mem = spdk_mem
@@ -1026,7 +897,7 @@ def restart_storage_node(
 
     cluster_docker = utils.get_docker_client(snode.cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-    results, err = snode_api.spdk_process_start(snode.spdk_cpu_mask, mem, img, spdk_debug, cluster_ip)
+    results, err = snode_api.spdk_process_start(cpu, mem, img, spdk_debug, cluster_ip)
 
     if not results:
         logger.error(f"Failed to start spdk: {err}")
@@ -1060,41 +931,13 @@ def restart_storage_node(
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- set socket implementation options
-    ret = rpc_client.sock_impl_set_options()
-    if not ret:
-        logger.error("Failed socket implement set options")
-        return False
-
-    # 3- set nvme config
-    if snode.pollers_mask:
-        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
-        if not ret:
-            logger.error("Failed to set pollers mask")
-            return False
-
-    # 4- start spdk framework
+    # 2- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 5- set app_thread cpu mask
-    if snode.app_thread_mask:
-        ret = rpc_client.thread_get_stats()
-        app_thread_process_id = 0
-        if ret.get("threads"):
-            for entry in ret["threads"]:
-                if entry['name'] == 'app_thread':
-                    app_thread_process_id = entry['id']
-                    break
-
-        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
-        if not ret:
-            logger.error("Failed to set app thread mask")
-            return False
-
-    # 6- set nvme bdev options
+    # 3- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")
@@ -1127,23 +970,22 @@ def restart_storage_node(
         else:
             logger.info(f"Device not found: {db_dev.get_id()}")
             db_dev.status = NVMeDevice.STATUS_REMOVED
-            distr_controller.send_dev_status_event(db_dev, db_dev.status)
+            distr_controller.send_dev_status_event(db_dev.cluster_device_order, "offline")
 
-    # todo: handle new devices
-    # for dev in nvme_devs:
-    #     if dev.serial_number not in known_devices_sn:
-    #         logger.info(f"New device found: {dev.get_id()}")
-    #         dev.status = NVMeDevice.STATUS_NEW
-    #         new_devices.append(dev)
-    #         snode.nvme_devices.append(dev)
+    for dev in nvme_devs:
+        if dev.serial_number not in known_devices_sn:
+            logger.info(f"New device found: {dev.get_id()}")
+            dev.status = 'new'
+            new_devices.append(dev)
+            snode.nvme_devices.append(dev)
 
-    # dev_order = get_next_cluster_device_order(db_controller, snode.cluster_id)
-    # for index, nvme in enumerate(new_devices):
-    #     nvme.cluster_device_order = dev_order
-    #     dev_order += 1
+    dev_order = get_next_cluster_device_order(db_controller)
+    for index, nvme in enumerate(new_devices):
+        nvme.cluster_device_order = dev_order
+        dev_order += 1
 
     # prepare devices
-    ret = _prepare_cluster_devices_on_restart(snode)
+    ret = _prepare_cluster_devices(snode, after_restart=True)
     if not ret:
         logger.error("Failed to prepare cluster devices")
         return False
@@ -1154,7 +996,7 @@ def restart_storage_node(
 
     # make other nodes connect to the new devices
     logger.info("Make other nodes connect to the node devices")
-    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    snodes = db_controller.get_storage_nodes()
     for node_index, node in enumerate(snodes):
         if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
@@ -1192,23 +1034,20 @@ def restart_storage_node(
     storage_events.snode_status_change(snode, snode.status, old_status)
 
     logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
+    distr_controller.send_node_status_event(snode.get_id(), NVMeDevice.STATUS_ONLINE)
 
     logger.info("Sending devices event updates")
-    logger.info("Starting migration tasks")
     for dev in snode.nvme_devices:
         if dev.status != NVMeDevice.STATUS_ONLINE:
-            logger.info(f"Device is not online: {dev.get_id()}, status: {dev.status}")
+            logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}")
             continue
+        distr_controller.send_dev_status_event(dev.cluster_device_order, NVMeDevice.STATUS_ONLINE)
 
-        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
-        tasks_controller.add_device_mig_task(dev.get_id())
-
-    # logger.info("Sending cluster map to current node")
-    # ret = distr_controller.send_cluster_map_to_node(snode)
-    # if not ret:
-    #     return False, "Failed to send cluster map"
-    # time.sleep(3)
+    logger.info("Sending cluster map to current node")
+    ret = distr_controller.send_cluster_map_to_node(snode)
+    if not ret:
+        return False, "Failed to send cluster map"
+    time.sleep(3)
 
     for lvol_id in snode.lvols:
         lvol = lvol_controller.recreate_lvol(lvol_id, snode)
@@ -1223,12 +1062,9 @@ def restart_storage_node(
     return "Success"
 
 
-def list_storage_nodes(is_json, cluster_id=None):
-    db_controller = DBController()
-    if cluster_id:
-        nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
-    else:
-        nodes = db_controller.get_storage_nodes()
+def list_storage_nodes(kv_store, is_json):
+    db_controller = DBController(kv_store)
+    nodes = db_controller.get_storage_nodes()
     data = []
     output = ""
 
@@ -1275,43 +1111,26 @@ def list_storage_devices(kv_store, node_id, sort, is_json):
         logger.error("This storage node is not part of the cluster")
         return False
 
-    storage_devices = []
-    jm_devices = []
-    remote_devices = []
+    data = []
     for device in snode.nvme_devices:
         logger.debug(device)
         logger.debug("*" * 20)
-        storage_devices.append({
+        data.append({
             "UUID": device.uuid,
             "Name": device.device_name,
+            "Hostname": snode.hostname,
             "Size": utils.humanbytes(device.size),
+            # "Sequential Number": device.sequential_number,
+            # "Partitions Count": device.partitions_count,
+            # "Model ID": device.model_id,
             "Serial Number": device.serial_number,
             "PCIe": device.pcie_address,
             "Status": device.status,
             "IO Err": device.io_error,
-            "Health": device.health_check
-        })
+            "Health": device.health_check,
 
-    if snode.jm_device:
-        jm_devices.append({
-            "UUID": snode.jm_device.uuid,
-            "Name": snode.jm_device.device_name,
-            "Size": utils.humanbytes(snode.jm_device.size),
-            "Status": snode.jm_device.status,
-            "IO Err": snode.jm_device.io_error,
-            "Health": snode.jm_device.health_check
         })
 
-    for device in snode.remote_devices:
-        logger.debug(device)
-        logger.debug("*" * 20)
-        remote_devices.append({
-            "UUID": device.uuid,
-            "Name": device.device_name,
-            "Size": utils.humanbytes(device.size),
-            "Serial Number": device.serial_number,
-            "Node ID": device.node_id,
-        })
     if sort and sort in ['node-seq', 'dev-seq', 'serial']:
         if sort == 'serial':
             sort_key = "Serial Number"
@@ -1320,20 +1139,13 @@ def list_storage_devices(kv_store, node_id, sort, is_json):
         elif sort == 'node-seq':
             # TODO: check this key
             sort_key = "Sequential Number"
-        storage_devices = sorted(storage_devices, key=lambda d: d[sort_key])
+        sorted_data = sorted(data, key=lambda d: d[sort_key])
+        data = sorted_data
 
-    data = {
-        "Storage Devices": storage_devices,
-        "JM Devices": jm_devices,
-        "Remote Devices": remote_devices,
-    }
     if is_json:
         return json.dumps(data, indent=2)
     else:
-        out = ""
-        for d in data:
-            out += f"{d}\n{utils.print_table(data[d])}\n\n"
-        return out
+        return utils.print_table(data)
 
 
 def shutdown_storage_node(node_id, force=False):
@@ -1374,7 +1186,7 @@ def shutdown_storage_node(node_id, force=False):
     for dev in snode.nvme_devices:
         if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
             device_controller.device_set_unavailable(dev.get_id())
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_IN_SHUTDOWN)
+    distr_controller.send_node_status_event(snode.get_id(), "in_shutdown")
 
     # shutdown node
     # make other nodes disconnect from this node
@@ -1394,7 +1206,7 @@ def shutdown_storage_node(node_id, force=False):
     snode_api = SNodeClient(snode.api_endpoint)
     results, err = snode_api.spdk_process_kill()
 
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_OFFLINE)
+    distr_controller.send_node_status_event(snode.get_id(), StorageNode.STATUS_OFFLINE)
 
     logger.info("Setting node status to offline")
     snode = db_controller.get_storage_node_by_id(node_id)
@@ -1421,24 +1233,22 @@ def suspend_storage_node(node_id, force=False):
         return False
 
     cluster = db_controller.get_cluster_by_id(snode.cluster_id)
-    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    snodes = db_controller.get_storage_nodes()
     online_nodes = 0
     for node in snodes:
         if node.status == node.STATUS_ONLINE:
             online_nodes += 1
-
-    if cluster.ha_type == "ha":
-        if online_nodes <= 3 and cluster.status == cluster.STATUS_ACTIVE:
-            logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
-            if force is False:
-                return False
-
-        if cluster.status == cluster.STATUS_DEGRADED and force is False:
-            logger.warning(f"Cluster status is degraded, use --force but this will suspend the cluster")
+    if cluster.ha_type == "ha" and online_nodes <= 3 and cluster.status == cluster.STATUS_ACTIVE:
+        logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
+        if force is False:
             return False
 
+    if cluster.ha_type == "ha" and cluster.status == cluster.STATUS_DEGRADED and force is False:
+        logger.warning(f"Cluster status is degraded, use --force but this will suspend the cluster")
+        return False
+
     logger.info("Suspending node")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_SUSPENDED)
+    distr_controller.send_node_status_event(snode.get_id(), "suspended")
     for dev in snode.nvme_devices:
         if dev.status == NVMeDevice.STATUS_ONLINE:
             device_controller.device_set_unavailable(dev.get_id())
@@ -1482,7 +1292,7 @@ def resume_storage_node(node_id):
     logger.info("Resuming node")
 
     logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
+    distr_controller.send_node_status_event(snode.get_id(), "online")
 
     for dev in snode.nvme_devices:
         if dev.status == NVMeDevice.STATUS_UNAVAILABLE:
@@ -1858,6 +1668,7 @@ def deploy_cleaner():
     return True
 
 
+
 def get_host_secret(node_id):
     db_controller = DBController()
     node = db_controller.get_storage_node_by_id(node_id)
@@ -2020,7 +1831,7 @@ def set_node_status(node_id, status):
         snode.updated_at = str(datetime.datetime.now())
         snode.write_to_db(db_controller.kv_store)
         storage_events.snode_status_change(snode, snode.status, old_status, caused_by="monitor")
-        distr_controller.send_node_status_event(snode, status)
+        distr_controller.send_node_status_event(snode.get_id(), status)
 
     if snode.status == StorageNode.STATUS_ONLINE:
         logger.info("Connecting to remote devices")