sbcli-dev 3.8.36__zip → 3.8.37__zip

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.36
+Version: 3.8.37
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.36
+SIMPLY_BLOCK_VERSION=3.8.37
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.36
+Version: 3.8.37
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_cli/cli.py

@@ -56,6 +56,7 @@ class CLIWrapper:
         sub_command.add_argument("--iobuf_small_bufsize", help='bdev_set_options param', dest='small_bufsize',  type=int, default=0)
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
         sub_command.add_argument("--enable-test-device", help='Enable creation of test device', action='store_true')
+        sub_command.add_argument("--enable-ha-jm", help='Enable HA JM for ditrib creation', action='store_true')
 
 
         # delete storage node
@@ -717,6 +718,7 @@ class CLIWrapper:
                 max_prov = self.parse_size(args.max_prov)
                 number_of_devices = args.number_of_devices
                 enable_test_device = args.enable_test_device
+                enable_ha_jm = args.enable_ha_jm
                 number_of_distribs = args.number_of_distribs
                 if max_prov < 1 * 1024 * 1024 * 1024:
                     return f"Max provisioning memory:{args.max_prov} must be larger than 1G"
@@ -739,7 +741,8 @@ class CLIWrapper:
                     number_of_devices=number_of_devices,
                     enable_test_device=enable_test_device,
                     namespace=None,
-                    number_of_distribs=number_of_distribs
+                    number_of_distribs=number_of_distribs,
+                    enable_ha_jm=enable_ha_jm
                 )
 
                 return out

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/cluster_ops.py

@@ -369,7 +369,7 @@ def cluster_activate(cl_id):
             logger.info(f"Node {snode.get_id()} already has lvstore {snode.lvstore}... skipping")
             continue
         ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
-                                              cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size)
+                                              cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size, snodes)
         if not ret:
             logger.error("Failed to activate cluster")
             set_cluster_status(cl_id, Cluster.STATUS_UNREADY)
@@ -688,14 +688,14 @@ def update_cluster(cl_id):
     except Exception as e:
         print(e)
 
-    for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
-        node_docker = docker.DockerClient(base_url=f"tcp://{node.mgmt_ip}:2375", version="auto")
-        logger.info(f"Pulling image {constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}")
-        node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
-        if node.status == StorageNode.STATUS_ONLINE:
-            storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
-            time.sleep(3)
-        storage_node_ops.restart_storage_node(node.get_id())
+    # for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
+    #     node_docker = docker.DockerClient(base_url=f"tcp://{node.mgmt_ip}:2375", version="auto")
+    #     logger.info(f"Pulling image {constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}")
+    #     node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
+    #     if node.status == StorageNode.STATUS_ONLINE:
+    #         storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
+    #         time.sleep(3)
+    #     storage_node_ops.restart_storage_node(node.get_id())
 
     logger.info("Done")
     return True

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/constants.py

@@ -11,7 +11,7 @@ LOG_WEB_DEBUG = True
 
 INSTALL_DIR = os.path.dirname(os.path.realpath(__file__))
 
-NODE_MONITOR_INTERVAL_SEC = 3
+NODE_MONITOR_INTERVAL_SEC = 10
 DEVICE_MONITOR_INTERVAL_SEC = 5
 STAT_COLLECTOR_INTERVAL_SEC = 60*5  # 5 minutes
 LVOL_STAT_COLLECTOR_INTERVAL_SEC = 5
@@ -52,13 +52,15 @@ GRAYLOG_CHECK_INTERVAL_SEC = 60
 
 FDB_CHECK_INTERVAL_SEC = 60
 
-SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:main"
+SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:dev"
 SIMPLY_BLOCK_CLI_NAME = "sbcli-dev"
 TASK_EXEC_INTERVAL_SEC = 30
 TASK_EXEC_RETRY_COUNT = 8
 
 SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:latest"
+SIMPLY_BLOCK_SPDK_CORE_IMAGE_ARM64 = "simplyblock/spdk-core:latest-arm64"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:348-temp-migration-2-latest"
+SIMPLY_BLOCK_SPDK_ULTRA_IMAGE_JM = "simplyblock/spdk:feature-journal-ha-latest"
 
 GELF_PORT = 12201
 

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/controllers/lvol_controller.py

@@ -169,6 +169,21 @@ def get_jm_names(snode):
     return [snode.jm_device.jm_bdev] if snode.jm_device else []
 
 
+def get_ha_jm_names(current_node, snode_list):
+    jm_list = []
+    if current_node.jm_device:
+        jm_list.append(current_node.jm_device.jm_bdev)
+    else:
+        jm_list.append("JM_LOCAL")
+
+    for node in snode_list:
+        if node.get_id() == current_node.get_id():
+            continue
+        name = f"remote_{node.jm_device.jm_bdev}n1"
+        jm_list.append(name)
+    return jm_list[:3]
+
+
 def _get_next_3_nodes(cluster_id, lvol_size=0):
     snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     online_nodes = []
@@ -564,7 +579,7 @@ def _create_bdev_stack(lvol, snode, ha_comm_addrs, ha_inode_self):
         ret = None
 
         if type == "bdev_distr":
-            params['jm_names'] = get_jm_names(snode)
+            # params['jm_names'] = get_jm_names(snode)
             params['ha_comm_addrs'] = ha_comm_addrs
             params['ha_inode_self'] = ha_inode_self
             params['distrib_cpu_mask'] = snode.distrib_cpu_mask

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/controllers/tasks_controller.py

@@ -54,20 +54,22 @@ def _add_task(function_name, cluster_id, node_id, device_id,
 
 
 def add_device_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 
 def add_device_to_auto_restart(device):
-    return _add_task(JobSchedule.FN_DEV_RESTART, device.cluster_id, device.node_id, device.get_id())
+    return True
+    # return _add_task(JobSchedule.FN_DEV_RESTART, device.cluster_id, device.node_id, device.get_id())
 
 
 def add_node_to_auto_restart(node):
-    return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")
+    return True
+    # return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")
 
 
 def list_tasks(cluster_id):
@@ -135,20 +137,20 @@ def get_active_node_mig_task(cluster_id, node_id):
 
 
 def add_device_failed_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_FAILED_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_FAILED_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 
 def add_new_device_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_NEW_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_NEW_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/models/nvme_device.py

@@ -113,6 +113,7 @@ class JMDevice(BaseModel):
         "nvmf_nqn": {"type": str, 'default': ""},
         "nvmf_ip": {"type": str, 'default': ""},
         "nvmf_port": {"type": int, 'default': 4420},
+        "remote_bdev": {"type": str, 'default': ""},
 
         "health_check": {"type": bool, "default": True},
         "io_error": {"type": bool, 'default': False},

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/models/storage_node.py

@@ -108,6 +108,8 @@ class StorageNode(BaseModel):
         "num_partitions_per_dev": {"type": int, "default": 1},
         "jm_percent": {"type": int, "default": 3},
         "jm_device": {"type": JMDevice, "default": None},
+        "remote_jm_devices": {"type": List[JMDevice], 'default': []},
+        "enable_ha_jm": {"type": bool, "default": False},
 
         "namespace": {"type": str, "default": ""},
 

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/rpc_client.py

@@ -363,7 +363,7 @@ class RPCClient:
 
     def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names,
                             chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152,
-                            distrib_cpu_mask=""):
+                            distrib_cpu_mask="", ha_is_non_leader=True, jm_vuid=1):
         """"
             // Optional (not specified = no HA)
             // Comma-separated communication addresses, for each node, e.g. "192.168.10.1:45001,192.168.10.1:32768".
@@ -379,12 +379,14 @@ class RPCClient:
             "name": name,
             "jm_names": ",".join(jm_names),
             "vuid": vuid,
+            "jm_vuid": jm_vuid,
             "ndcs": ndcs,
             "npcs": npcs,
             "num_blocks": num_blocks,
             "block_size": block_size,
             "chunk_size": chunk_size,
-            "pba_page_size": pba_page_size
+            "pba_page_size": pba_page_size,
+            "ha_is_non_leader": ha_is_non_leader,
         }
         if ha_comm_addrs:
             params['ha_comm_addrs'] = ha_comm_addrs
@@ -470,6 +472,20 @@ class RPCClient:
         }
         return self._request("bdev_nvme_attach_controller", params)
 
+    def bdev_nvme_attach_controller_tcp_jm(self, name, nqn, ip, port):
+        params = {
+            "name": name,
+            "trtype": "tcp",
+            "traddr": ip,
+            "adrfam": "ipv4",
+            "trsvcid": str(port),
+            "subnqn": nqn,
+            "fast_io_fail_timeout_sec": 1,
+            "ctrlr_loss_timeout_sec": 1,
+            "reconnect_delay_sec": 1,
+        }
+        return self._request("bdev_nvme_attach_controller", params)
+
     def bdev_nvme_attach_controller_tcp_caching(self, name, nqn, ip, port):
         params = {
             "name": name,

{sbcli_dev-3.8.36 → sbcli_dev-3.8.37}/simplyblock_core/storage_node_ops.py

@@ -295,9 +295,9 @@ def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart):
         logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
         return False
 
-    subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
+    subsystem_nqn = snode.subsystem + ":dev:" + jm_bdev
     logger.info("creating subsystem %s", subsystem_nqn)
-    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
+    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
     IP = None
     for iface in snode.data_nics:
         if iface.ip4_address:
@@ -387,9 +387,9 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode, after_restart):
         logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
         return False
 
-    subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
+    subsystem_nqn = snode.subsystem + ":dev:" + jm_bdev
     logger.info("creating subsystem %s", subsystem_nqn)
-    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
+    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
     IP = None
     for iface in snode.data_nics:
         if iface.ip4_address:
@@ -648,6 +648,7 @@ def _prepare_cluster_devices_on_restart(snode):
         if not dev:
             logger.error(f"Failed to create dev stack {nvme.get_id()}")
             return False
+        nvme.status = NVMeDevice.STATUS_ONLINE
         device_events.device_restarted(dev)
 
     # prepare JM device
@@ -700,7 +701,6 @@ def _prepare_cluster_devices_on_restart(snode):
             logger.error(f"Failed to create {jm_bdev}")
             return False
 
-        alceml_id = jm_device.uuid
         # add pass through
         pt_name = f"{jm_bdev}_PT"
         ret = rpc_client.bdev_PT_NoExcl_create(pt_name, jm_bdev)
@@ -708,9 +708,9 @@ def _prepare_cluster_devices_on_restart(snode):
             logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
             return False
 
-        subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
+        subsystem_nqn = snode.subsystem + ":dev:" + jm_bdev
         logger.info("creating subsystem %s", subsystem_nqn)
-        ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id)
+        ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', jm_bdev)
         IP = None
         for iface in snode.data_nics:
             if iface.ip4_address:
@@ -733,7 +733,6 @@ def _prepare_cluster_devices_on_restart(snode):
             logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
             return False
 
-
     return True
 
 
@@ -750,26 +749,67 @@ def _connect_to_remote_devs(this_node):
     for node_index, node in enumerate(snodes):
         if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
+        logger.info(f"Connecting to node {node.get_id()}")
         for index, dev in enumerate(node.nvme_devices):
             if dev.status != NVMeDevice.STATUS_ONLINE:
                 logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}")
                 continue
+
             name = f"remote_{dev.alceml_bdev}"
-            logger.info(f"Connecting to {name}")
-            ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
-            if not ret:
-                logger.error(f"Failed to connect to device: {dev.get_id()}")
-                continue
-            dev.remote_bdev = f"{name}n1"
+            bdev_name = f"{name}n1"
+            ret = rpc_client.get_bdevs(bdev_name)
+            if ret:
+                logger.info(f"bdev found {bdev_name}")
+            else:
+                ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
+                if not ret:
+                    logger.error(f"Failed to connect to device: {dev.get_id()}")
+                    continue
+            dev.remote_bdev = bdev_name
             remote_devices.append(dev)
     return remote_devices
 
 
+def _connect_to_remote_jm_devs(this_node):
+    db_controller = DBController()
+
+    rpc_client = RPCClient(
+        this_node.mgmt_ip, this_node.rpc_port,
+        this_node.rpc_username, this_node.rpc_password)
+
+    remote_devices = []
+    # connect to remote devs
+    snodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id)
+    for node_index, node in enumerate(snodes):
+        if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
+            continue
+
+        logger.info(f"Connecting to node {node.get_id()}")
+        if node.jm_device:
+            name = f"remote_{node.jm_device.jm_bdev}"
+            bdev_name = f"{name}n1"
+            ret = rpc_client.get_bdevs(bdev_name)
+            if ret:
+                logger.info(f"bdev found {bdev_name}")
+            else:
+                logger.info(f"Connecting {this_node.get_id()} to {name}")
+                ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                    name, node.jm_device.nvmf_nqn, node.jm_device.nvmf_ip, node.jm_device.nvmf_port)
+                if not ret:
+                    logger.error(f"failed to connect to remote JM {node.jm_device.jm_bdev}")
+                    continue
+
+            node.jm_device.remote_bdev = bdev_name
+            remote_devices.append(node.jm_device)
+
+    return remote_devices
+
+
 def add_node(cluster_id, node_ip, iface_name, data_nics_list,
              max_lvol, max_snap, max_prov, spdk_image=None, spdk_debug=False,
              small_bufsize=0, large_bufsize=0, spdk_cpu_mask=None,
              num_partitions_per_dev=0, jm_percent=0, number_of_devices=0, enable_test_device=False,
-             namespace=None, number_of_distribs=2):
+             namespace=None, number_of_distribs=2, enable_ha_jm=False):
 
     db_controller = DBController()
     kv_store = db_controller.kv_store
@@ -913,7 +953,9 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         logger.error(f"Failed to Join docker swarm: {err}")
         return False
 
-    rpc_user, rpc_pass = utils.generate_rpc_user_and_pass()
+    #rpc_user, rpc_pass = utils.generate_rpc_user_and_pass()
+    rpc_user = "rpc_username"
+    rpc_pass = "rpc_password"
     mgmt_ip = node_info['network_interface'][iface_name]['ip']
 
     logger.info("Deploying SPDK")
@@ -971,6 +1013,9 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     snode.host_secret = utils.generate_string(20)
     snode.ctrl_secret = utils.generate_string(20)
     snode.number_of_distribs = number_of_distribs
+    snode.enable_ha_jm = enable_ha_jm
+    if enable_ha_jm:
+        spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE_JM
 
     if 'cpu_count' in node_info:
         snode.cpu = node_info['cpu_count']
@@ -1091,6 +1136,10 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     remote_devices = _connect_to_remote_devs(snode)
     snode.remote_devices = remote_devices
 
+    if snode.enable_ha_jm:
+        logger.info("Connecting to remote JMs")
+        snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
+
     logger.info("Setting node status to Active")
     snode.status = StorageNode.STATUS_ONLINE
     snode.write_to_db(kv_store)
@@ -1103,7 +1152,6 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
             continue
         logger.info(f"Connecting to node: {node.get_id()}")
         rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
-        count = 0
         for dev in snode.nvme_devices:
             if dev.status != NVMeDevice.STATUS_ONLINE:
                 logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}")
@@ -1124,9 +1172,11 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
                 node.remote_devices[idx] = dev
             else:
                 node.remote_devices.append(dev)
-            count += 1
+
+        if node.enable_ha_jm:
+            node.remote_jm_devices = _connect_to_remote_jm_devs(node)
         node.write_to_db(kv_store)
-        logger.info(f"connected to devices count: {count}")
+        logger.info(f"connected to devices count: {len(node.remote_devices)}")
         time.sleep(3)
 
     if cluster.status == cluster.STATUS_UNREADY:
@@ -1149,10 +1199,11 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
         tasks_controller.add_new_device_mig_task(dev.get_id())
 
+    nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     # Create distribs
     max_size = cluster.cluster_max_size
     ret = create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
-                         cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size)
+                         cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size, nodes)
     if not ret:
         return False, "Failed to create distribs on node"
 
@@ -1577,68 +1628,46 @@ def restart_storage_node(
             logger.error("Failed to prepare cluster devices")
             # return False
 
+    snode.write_to_db(kv_store)
+
     logger.info("Connecting to remote devices")
     remote_devices = _connect_to_remote_devs(snode)
     snode = db_controller.get_storage_node_by_id(node_id)
     snode.remote_devices = remote_devices
     snode.write_to_db(db_controller.kv_store)
 
-    # make other nodes connect to the new devices
-    logger.info("Make other nodes connect to the node devices")
-    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
-    for node_index, node in enumerate(snodes):
-        if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
-            continue
-        logger.info(f"Connecting to node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
-        count = 0
-        for dev in snode.nvme_devices:
-            if dev.status != 'online':
-                continue
-            name = f"remote_{dev.alceml_bdev}"
-            ret = rpc_client.bdev_nvme_controller_list(name)
-            if ret:
-                logger.debug(f"controller found, removing")
-                rpc_client.bdev_nvme_detach_controller(name)
-                time.sleep(1)
-            ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
-            if not ret:
-                logger.warning(f"Failed to connect to device: {name}")
-                return False
-
-            dev.remote_bdev = f"{name}n1"
-            idx = -1
-            for i, d in enumerate(node.remote_devices):
-                if d.get_id() == dev.get_id():
-                    idx = i
-                    break
-            if idx >= 0:
-                node.remote_devices[idx] = dev
-            else:
-                node.remote_devices.append(dev)
-            count += 1
-        node.write_to_db(kv_store)
-        logger.info(f"connected to devices count: {count}")
-        time.sleep(3)
+    if snode.enable_ha_jm:
+        logger.info("Connecting to remote JMs")
+        snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
 
     logger.info("Setting node status to Online")
-    snode = db_controller.get_storage_node_by_id(node_id)
     old_status = snode.status
     snode.status = StorageNode.STATUS_ONLINE
     snode.write_to_db(kv_store)
     storage_events.snode_status_change(snode, snode.status, old_status)
 
-    logger.info("Sending node event update")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
 
-    logger.info("Sending devices event updates")
+    # make other nodes connect to the new devices
+    logger.info("Make other nodes connect to the node devices")
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    for node in snodes:
+        if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+            continue
+        node.remote_devices = _connect_to_remote_devs(node)
+        if node.enable_ha_jm:
+            node.remote_jm_devices = _connect_to_remote_jm_devs(node)
+        node.write_to_db(kv_store)
+
+        logger.info(f"Sending cluster map to node {node.get_id()}")
+        distr_controller.send_cluster_map_to_node(node)
+
     logger.info("Starting migration tasks")
     for dev in snode.nvme_devices:
         if dev.status != NVMeDevice.STATUS_ONLINE:
             logger.info(f"Device is not online: {dev.get_id()}, status: {dev.status}")
             continue
 
-        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
+        # distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
         tasks_controller.add_device_mig_task(dev.get_id())
 
     # Create distribs, raid0, and lvstore and expose lvols to the fabrics
@@ -1655,15 +1684,6 @@ def restart_storage_node(
         ret = temp_rpc_client.bdev_wait_for_examine()
         time.sleep(1)
 
-        #logger.info("Sending cluster map to current node")
-        #ret = distr_controller.send_cluster_map_to_node(snode)
-        #if not ret:
-        #    return False, "Failed to send cluster map"
-        #time.sleep(3)
-
-        # temp_rpc_client.bdev_examine(snode.raid)
-        # time.sleep(3)
-
         if snode.lvols:
             for lvol_id in snode.lvols:
                 lvol = lvol_controller.recreate_lvol(lvol_id, snode)
@@ -1765,6 +1785,18 @@ def list_storage_devices(kv_store, node_id, sort, is_json):
             "Serial Number": device.serial_number,
             "Node ID": device.node_id,
         })
+
+    for device in snode.remote_jm_devices:
+        logger.debug(device)
+        logger.debug("*" * 20)
+        remote_devices.append({
+            "UUID": device.uuid,
+            "Name": device.remote_bdev,
+            "Size": utils.humanbytes(device.size),
+            "Serial Number": "",
+            "Node ID": "",
+        })
+
     if sort and sort in ['node-seq', 'dev-seq', 'serial']:
         if sort == 'serial':
             sort_key = "Serial Number"
@@ -2497,7 +2529,10 @@ def set_node_status(node_id, status):
 
     if snode.status == StorageNode.STATUS_ONLINE:
         logger.info("Connecting to remote devices")
-        _connect_to_remote_devs(snode)
+        snode.remote_devices = _connect_to_remote_devs(snode)
+        if snode.enable_ha_jm:
+            snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
+        snode.write_to_db(db_controller.kv_store)
 
     return True
 
@@ -2511,7 +2546,7 @@ def recreate_lvstore(snode):
     return True
 
 
-def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blocks, max_size):
+def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blocks, max_size, nodes):
     lvstore_stack = []
     distrib_list = []
     size = max_size // snode.number_of_distribs
@@ -2519,6 +2554,15 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
     cluster_sz = ndcs * page_size_in_blocks
     strip_size_kb = int((ndcs + npcs) * 64)
     strip_size_kb = utils.nearest_upper_power_of_2(strip_size_kb)
+    if snode.enable_ha_jm:
+        if len(nodes) > 3:
+            nodes = nodes[:3]
+        jm_names = lvol_controller.get_ha_jm_names(snode, nodes)
+    else:
+        jm_names = lvol_controller.get_jm_names(snode)
+
+    jm_vuid = utils.get_random_vuid()
+
     for _ in range(snode.number_of_distribs):
         distrib_vuid = utils.get_random_vuid()
         distrib_name = f"distrib_{distrib_vuid}"
@@ -2530,6 +2574,8 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
                     "name": distrib_name,
                     "params": {
                         "name": distrib_name,
+                        "jm_names": jm_names,
+                        "jm_vuid": jm_vuid,
                         "vuid": distrib_vuid,
                         "ndcs": ndcs,
                         "npcs": npcs,
@@ -2548,11 +2594,12 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
             {
                 "type": "bdev_raid",
                 "name": raid_device,
-                "params":
-                    {"name": raid_device,
-                     "raid_level": "0",
-                     "base_bdevs": distrib_list,
-                     "strip_size_kb": strip_size_kb},
+                "params": {
+                    "name": raid_device,
+                    "raid_level": "0",
+                    "base_bdevs": distrib_list,
+                    "strip_size_kb": strip_size_kb
+                },
                 "distribs_list": distrib_list
             },
             {
@@ -2602,7 +2649,6 @@ def _create_bdev_stack(snode, lvstore_stack=None):
         params = bdev['params']
 
         if type == "bdev_distr":
-            params['jm_names'] = lvol_controller.get_jm_names(snode)
             if snode.distrib_cpu_cores:
                 distrib_cpu_mask = utils.decimal_to_hex_power_of_2(snode.distrib_cpu_cores[snode.distrib_cpu_index])
                 params['distrib_cpu_mask'] = distrib_cpu_mask