sbcli-dev 4.0.5__zip → 4.0.7__zip

{sbcli_dev-4.0.5 → sbcli_dev-4.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.5
+Version: 4.0.7
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.5 → sbcli_dev-4.0.7}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.5
+SIMPLY_BLOCK_VERSION=4.0.7
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.5 → sbcli_dev-4.0.7}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.5
+Version: 4.0.7
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.5 → sbcli_dev-4.0.7}/simplyblock_core/controllers/device_controller.py

@@ -620,64 +620,91 @@ def add_device(device_id):
             break
 
     logger.info(f"Adding device {device_id}")
+    if snode.num_partitions_per_dev == 0:
+        ret = _def_create_device_stack(device_obj, snode)
+        if not ret:
+            logger.error("Failed to create device stack")
+            return False
+        dev_order = storage_node_ops.get_next_cluster_device_order(db_controller, snode.cluster_id)
+        device_obj.cluster_device_order = dev_order
+        logger.info("Setting device online")
+        device_obj.status = NVMeDevice.STATUS_ONLINE
+        snode.write_to_db(db_controller.kv_store)
+        device_events.device_create(device_obj)
 
-    ret = _def_create_device_stack(device_obj, snode)
-    if not ret:
-        logger.error("Failed to create device stack")
-        return False
+        logger.info("Make other nodes connect to the node devices")
+        snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+        for node in snodes:
+            if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+                continue
+            node.remote_devices = storage_node_ops._connect_to_remote_devs(node)
+            node.write_to_db()
 
-    dev_order = storage_node_ops.get_next_cluster_device_order(db_controller, snode.cluster_id)
-    device_obj.cluster_device_order = dev_order
+        tasks_controller.add_new_device_mig_task(device_id)
+        return device_id
 
-    logger.info("Setting device online")
-    device_obj.status = NVMeDevice.STATUS_ONLINE
-    snode.write_to_db(db_controller.kv_store)
 
-    logger.info("Make other nodes connect to the new device")
-    snodes = db_controller.get_storage_nodes()
-    for node_index, node in enumerate(snodes):
-        if node.get_id() == snode.get_id():
-            continue
-        if node.status != snode.STATUS_ONLINE:
-            continue
+    # create partitions
+    partitions = snode.num_partitions_per_dev
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+    # look for partitions
+    partitioned_devices = storage_node_ops._search_for_partitions(rpc_client, device_obj)
+    logger.debug("partitioned_devices")
+    logger.debug(partitioned_devices)
+    if len(partitioned_devices) == partitions+1:
+        logger.info("Partitioned devices found")
+    else:
+        logger.info(f"Creating partitions for {device_obj.nvme_bdev}")
+        storage_node_ops._create_device_partitions(rpc_client, device_obj, snode, partitions, snode.jm_percent)
+        partitioned_devices = storage_node_ops._search_for_partitions(rpc_client, device_obj)
+        if len(partitioned_devices) == partitions+1:
+            logger.info("Device partitions created")
+        else:
+            logger.error("Failed to create partitions")
+            return False
 
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
-        name = f"remote_{device_obj.alceml_bdev}"
-        ret = rpc_client.bdev_nvme_attach_controller_tcp(name, device_obj.nvmf_nqn, device_obj.nvmf_ip,
-                                                         device_obj.nvmf_port)
-        if not ret:
-            logger.error(f"Failed to connect to device: {name}")
+    jm_part = partitioned_devices.pop(0)
+    new_devices = []
+    dev_order = storage_node_ops.get_next_cluster_device_order(db_controller, snode.cluster_id)
+    for dev in partitioned_devices:
+        new_device = storage_node_ops._create_storage_device_stack(rpc_client, dev, snode, after_restart=False)
+        if not new_device:
+            logger.error("failed to create dev stack")
             continue
 
-        device_obj.remote_bdev = f"{name}n1"
-        idx = -1
-        for i, d in enumerate(node.remote_devices):
-            if d.get_id() == device_obj.get_id():
-                idx = i
-                break
-        if idx >= 0:
-            node.remote_devices[idx] = device_obj
-        else:
-            node.remote_devices.append(device_obj)
-        node.write_to_db(db_controller.kv_store)
-        time.sleep(1)
+        new_device.cluster_device_order = dev_order
+        dev_order += 1
+        device_events.device_create(new_device)
+        new_devices.append(new_device)
 
-    distr_controller.send_dev_status_event(device_obj, device_obj.status)
+    if new_devices:
+        snode.nvme_devices.remove(device_obj)
+        snode.nvme_devices.extend(new_devices)
+        snode.write_to_db(db_controller.kv_store)
+    else:
+        logger.error("failed to create devices")
+        return False
 
-    device_events.device_create(device_obj)
+    logger.info("Make other nodes connect to the node devices")
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    for node in snodes:
+        if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+            continue
+        node.remote_devices = storage_node_ops._connect_to_remote_devs(node)
+        node.write_to_db()
 
-    tasks_controller.add_new_device_mig_task(device_id)
+    for dev in new_devices:
+        tasks_controller.add_new_device_mig_task(dev.get_id())
 
     # add to jm raid
-    if snode.jm_device and snode.jm_device.raid_bdev:
+    if snode.jm_device and snode.jm_device.raid_bdev and jm_part:
         # looking for jm partition
-        rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-        jm_dev_part = f"{dev.nvme_bdev[:-1]}1"
+        jm_dev_part = jm_part.nvme_bdev
         ret = rpc_client.get_bdevs(jm_dev_part)
         if ret:
             logger.info(f"JM part found: {jm_dev_part}")
-            if snode.jm_device.status == JMDevice.STATUS_UNAVAILABLE:
-                restart_jm_device(snode.jm_device.get_id(), force=True)
+            if snode.jm_device.status in [JMDevice.STATUS_UNAVAILABLE, JMDevice.STATUS_REMOVED]:
+                restart_jm_device(snode.jm_device.get_id(), force=True, format_alceml=True)
 
             if snode.jm_device.status == JMDevice.STATUS_ONLINE and \
                     jm_dev_part not in snode.jm_device.jm_nvme_bdev_list:
@@ -714,6 +741,10 @@ def set_jm_device_state(device_id, state):
         for node_index, node in enumerate(snodes):
             if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
+            for jm_dev in node.remote_jm_devices:
+                if jm_dev.get_id() == device_id and jm_device.status != JMDevice.STATUS_ONLINE:
+                    node.remote_jm_devices.remove(jm_dev)
+                    break
             logger.info(f"Connecting to node: {node.get_id()}")
             node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
             node.write_to_db(db_controller.kv_store)
@@ -770,7 +801,7 @@ def remove_jm_device(device_id, force=False):
     return True
 
 
-def restart_jm_device(device_id, force=False):
+def restart_jm_device(device_id, force=False, format_alceml=False):
     db_controller = DBController()
     jm_device = None
     snode = None
@@ -796,13 +827,14 @@ def restart_jm_device(device_id, force=False):
         for dev in snode.nvme_devices:
             if dev.status != NVMeDevice.STATUS_ONLINE:
                 continue
-            dev_part = f"{dev.nvme_bdev[:-1]}1"
+            dev_part = f"{dev.nvme_bdev[:-2]}p1"
             if dev_part in bdevs_names:
                 if dev_part not in jm_nvme_bdevs:
                     jm_nvme_bdevs.append(dev_part)
 
         if len(jm_nvme_bdevs) > 0:
-            new_jm = storage_node_ops._create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode, after_restart=True)
+            new_jm = storage_node_ops._create_jm_stack_on_raid(
+                rpc_client, jm_nvme_bdevs, snode, after_restart=not format_alceml)
             if not new_jm:
                 logger.error("failed to create jm stack")
                 return False

{sbcli_dev-4.0.5 → sbcli_dev-4.0.7}/simplyblock_core/storage_node_ops.py

@@ -514,15 +514,14 @@ def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
     return nvme
 
 
-def _create_device_partitions(rpc_client, nvme, snode):
+def _create_device_partitions(rpc_client, nvme, snode, num_partitions_per_dev, jm_percent):
     nbd_device = rpc_client.nbd_start_disk(nvme.nvme_bdev)
     time.sleep(3)
     if not nbd_device:
         logger.error(f"Failed to start nbd dev")
         return False
     snode_api = SNodeClient(snode.api_endpoint)
-    result, error = snode_api.make_gpt_partitions(
-        nbd_device, snode.jm_percent, snode.num_partitions_per_dev)
+    result, error = snode_api.make_gpt_partitions(nbd_device, jm_percent, num_partitions_per_dev)
     if error:
         logger.error(f"Failed to make partitions")
         logger.error(error)
@@ -563,7 +562,7 @@ def _prepare_cluster_devices_partitions(snode, devices):
             logger.info("Partitioned devices found")
         else:
             logger.info(f"Creating partitions for {nvme.nvme_bdev}")
-            _create_device_partitions(rpc_client, nvme, snode)
+            _create_device_partitions(rpc_client, nvme, snode, snode.num_partitions_per_dev, snode.jm_percent)
             partitioned_devices = _search_for_partitions(rpc_client, nvme)
             if len(partitioned_devices) == (1 + snode.num_partitions_per_dev):
                 logger.info("Device partitions created")
@@ -798,16 +797,24 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
     if jm_ids:
         for jm_id in jm_ids:
             jm_dev = db_controller.get_jm_device_by_id(jm_id)
-            remote_devices.append(jm_dev)
+            if jm_dev:
+                remote_devices.append(jm_dev)
     elif len(this_node.remote_jm_devices) > 0:
         remote_devices = this_node.remote_jm_devices
     else:
         for node in db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id):
             if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
-            if node.jm_device:
+            if node.jm_device and node.jm_device.status == JMDevice.STATUS_ONLINE:
                 remote_devices.append(node.jm_device)
 
+    if len(remote_devices) < 2:
+        online_jms = get_next_ha_jms(this_node)
+        for jm_id in online_jms:
+            jm_dev = db_controller.get_jm_device_by_id(jm_id)
+            if jm_dev:
+                remote_devices.append(jm_dev)
+
     for jm_dev in remote_devices:
         name = f"remote_{jm_dev.jm_bdev}"
         bdev_name = f"{name}n1"
@@ -1354,6 +1361,10 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
                 distr_controller.disconnect_device(dev)
             device_controller.device_set_failed(dev.get_id())
 
+    if snode.jm_device:
+        logger.info("Removing JM")
+        device_controller.remove_jm_device(snode.jm_device.get_id(), force=True)
+
     logger.info("Removing storage node")
 
     logger.debug("Leaving swarm...")
@@ -1687,6 +1698,7 @@ def restart_storage_node(
 
     if snode.enable_ha_jm:
         logger.info("Connecting to remote JMs")
+        snode.remote_jm_devices = []
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
 
     logger.info("Setting node status to Online")
@@ -1714,6 +1726,9 @@ def restart_storage_node(
             continue
         tasks_controller.add_device_mig_task(dev.get_id())
 
+    if cluster.status == cluster.STATUS_UNREADY:
+        logger.info("Done")
+        return "Success"
     # Create distribs, raid0, and lvstore and expose lvols to the fabrics
     if snode.lvstore_stack:
         temp_rpc_client = RPCClient(
@@ -2562,12 +2577,15 @@ def get_next_ha_jms(current_node):
     for node in db_controller.get_storage_nodes_by_cluster_id(current_node.cluster_id):
         if node.get_id() == current_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
-        if node.jm_device:
+        if node.jm_device and node.jm_device.status == JMDevice.STATUS_ONLINE:
             jm_count[node.jm_device.get_id()] = 1 + jm_count.get(node.jm_device.get_id(), 0)
         for rem_jm_device in node.remote_jm_devices:
             if rem_jm_device.get_id() != current_node.jm_device.get_id():
-                jm_count[rem_jm_device.get_id()] = 1 + jm_count.get(rem_jm_device.get_id(), 0)
-
+                try:
+                    if db_controller.get_jm_device_by_id(rem_jm_device.get_id()).status == JMDevice.STATUS_ONLINE:
+                        jm_count[rem_jm_device.get_id()] = 1 + jm_count.get(rem_jm_device.get_id(), 0)
+                except :
+                    pass
     jm_count = dict(sorted(jm_count.items(), key=lambda x: x[1]))
     return list(jm_count.keys())[:2]
 
@@ -2597,6 +2615,7 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
     if snode.enable_ha_jm:
         jm_vuid = utils.get_random_vuid()
         online_jms = get_next_ha_jms(snode)
+        logger.debug(f"online_jms: {str(online_jms)}")
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, online_jms)
         snode.write_to_db()