sbcli-dev 4.0.16__zip → 4.0.18__zip

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.16
+Version: 4.0.18
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.16
+SIMPLY_BLOCK_VERSION=4.0.18
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.16
+Version: 4.0.18
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/cluster_ops.py

@@ -342,9 +342,10 @@ def cluster_activate(cl_id, force=False):
     if not cluster:
         logger.error(f"Cluster not found {cl_id}")
         return False
-    if cluster.status != Cluster.STATUS_UNREADY:
-        logger.error(f"Failed to activate cluster, Cluster is not in an UNREADY state")
+    if cluster.status == Cluster.STATUS_ACTIVE:
+        logger.warning("Cluster is not in an ACTIVE state")
         if not force:
+            logger.warning(f"Failed to activate cluster, Cluster is in an ACTIVE state, use --force to reactivate")
             return False
     set_cluster_status(cl_id, Cluster.STATUS_IN_ACTIVATION)
     snodes = db_controller.get_storage_nodes_by_cluster_id(cl_id)
@@ -367,17 +368,17 @@ def cluster_activate(cl_id, force=False):
 
     for snode in snodes:
         if snode.lvstore:
-            logger.info(f"Node {snode.get_id()} already has lvstore {snode.lvstore}... skipping")
-            if not force:
-                continue
-        ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
+            logger.warning(f"Node {snode.get_id()} already has lvstore {snode.lvstore}")
+            ret = storage_node_ops.recreate_lvstore(snode)
+        else:
+            ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
                                               cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size, snodes)
         if not ret:
             logger.error("Failed to activate cluster")
             set_cluster_status(cl_id, Cluster.STATUS_UNREADY)
             return False
-
-    cluster.cluster_max_size = max_size
+    if not cluster.cluster_max_size:
+        cluster.cluster_max_size = max_size
     cluster.write_to_db(db_controller.kv_store)
     set_cluster_status(cl_id, Cluster.STATUS_ACTIVE)
     logger.info("Cluster activated successfully")

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/controllers/health_controller.py

@@ -446,7 +446,7 @@ def check_jm_device(device_id):
     try:
         rpc_client = RPCClient(
             snode.mgmt_ip, snode.rpc_port,
-            snode.rpc_username, snode.rpc_password)
+            snode.rpc_username, snode.rpc_password, timeout=3, retry=2)
 
         ret = rpc_client.get_bdevs(jm_device.jm_bdev)
         if ret:

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/controllers/tasks_controller.py

@@ -52,6 +52,11 @@ def _add_task(function_name, cluster_id, node_id, device_id,
         if task_id:
             logger.info(f"Task found, skip adding new task: {task_id}")
             return False
+    elif function_name == JobSchedule.FN_DEV_MIG:
+        task_id = get_device_mig_task(cluster_id, node_id, function_params['distr_name'])
+        if task_id:
+            logger.info(f"Task found, skip adding new task: {task_id}")
+            return False
 
     task_obj = JobSchedule()
     task_obj.uuid = str(uuid.uuid4())
@@ -72,6 +77,8 @@ def _add_task(function_name, cluster_id, node_id, device_id,
 def add_device_mig_task(device_id):
     device = db_controller.get_storage_devices(device_id)
     for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+        if not node.lvols:
+            continue
         for bdev in node.lvstore_stack:
             if bdev['type'] == "bdev_distr":
                 _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
@@ -190,6 +197,7 @@ def get_active_node_task(cluster_id, node_id):
                 return task.uuid
     return False
 
+
 def get_new_device_mig_task(cluster_id, node_id, distr_name):
     tasks = db_controller.get_job_tasks(cluster_id)
     for task in tasks:
@@ -198,3 +206,13 @@ def get_new_device_mig_task(cluster_id, node_id, distr_name):
                     and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
                 return task.uuid
     return False
+
+
+def get_device_mig_task(cluster_id, node_id, distr_name):
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_DEV_MIG and task.node_id == node_id:
+            if task.status != JobSchedule.STATUS_DONE and task.canceled is False \
+                    and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
+                return task.uuid
+    return False

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/rpc_client.py

@@ -377,6 +377,12 @@ class RPCClient:
             //  This node (device) number, in the group, defined by ha_comm_addrs.
           "ha_inode_self": 1
         """
+        try:
+            ret = self.get_bdevs(name)
+            if ret:
+                return ret
+        except:
+            pass
         params = {
             "name": name,
             "jm_names": ",".join(jm_names),
@@ -417,6 +423,12 @@ class RPCClient:
         return self._request("bdev_get_iostat", params)
 
     def bdev_raid_create(self, name, bdevs_list, raid_level="0", strip_size_kb=4):
+        try:
+            ret = self.get_bdevs(name)
+            if ret:
+                return ret
+        except:
+            pass
         params = {
             "name": name,
             "raid_level": raid_level,
@@ -470,11 +482,11 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 1,
+            # "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
-            "ctrlr_loss_timeout_sec": 1,
+            # "ctrlr_loss_timeout_sec": 1,
             "multipath":"disable",
-            "reconnect_delay_sec":1
+            # "reconnect_delay_sec":1
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -487,7 +499,7 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fast_io_fail_timeout_sec": 1,
-            "ctrlr_loss_timeout_sec": 3,
+            "ctrlr_loss_timeout_sec": 1,
             "reconnect_delay_sec": 1,
             "fabrics_connect_timeout_us": 100000,
             "multipath":"disable"
@@ -547,6 +559,7 @@ class RPCClient:
 
     def bdev_nvme_set_options(self):
         params = {
+            # "action_on_timeout": "abort",
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
             "ctrlr_loss_timeout_sec": 2,

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/services/health_check_service.py

@@ -82,7 +82,6 @@ while True:
 
             health_check_status = is_node_online
             if not node_rpc_check:
-                logger.info("Putting all devices to unavailable state because RPC check failed")
                 for dev in snode.nvme_devices:
                     if dev.io_error:
                         logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
@@ -107,15 +106,51 @@ while True:
                     snode.mgmt_ip, snode.rpc_port,
                     snode.rpc_username, snode.rpc_password,
                     timeout=10, retry=1)
+                connected_devices = []
                 for remote_device in snode.remote_devices:
                     if db_controller.get_storage_device_by_id(remote_device.get_id()).status == NVMeDevice.STATUS_ONLINE:
                         ret = rpc_client.get_bdevs(remote_device.remote_bdev)
                         if ret:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                            connected_devices.append(remote_device.get_id())
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_detach_controller(remote_device.remote_bdev)
+                            time.sleep(1)
+                            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip, remote_device.nvmf_port)
+                            if ret:
+                                logger.info(f"Successfully connected to device: {remote_device.get_id()}")
+                                connected_devices.append(remote_device.get_id())
+                                distr_controller.send_dev_status_event(remote_device, NVMeDevice.STATUS_ONLINE, snode)
+                            else:
+                                logger.error(f"Failed to connect to device: {remote_device.get_id()}")
+
                         node_remote_devices_check &= bool(ret)
 
+                for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
+                    if node.status != StorageNode.STATUS_ONLINE:
+                        continue
+                    for dev in node.nvme_devices:
+                        if dev.status == StorageNode.STATUS_ONLINE:
+                            if dev.get_id() not in connected_devices:
+                                logger.info(f"connecting to online device: {dev.get_id()}")
+                                name = f"remote_{dev.alceml_bdev}"
+                                bdev_name = f"{name}n1"
+                                ret = rpc_client.bdev_nvme_detach_controller(name)
+                                time.sleep(1)
+                                ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                    name, dev.nvmf_nqn, dev.nvmf_ip,
+                                    dev.nvmf_port)
+                                if ret:
+                                    logger.info(f"Successfully connected to device: {dev.get_id()}")
+                                    dev.remote_bdev = bdev_name
+                                    snode.remote_devices.append(dev)
+                                    snode.write_to_db()
+                                    distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE, snode)
+                                else:
+                                    logger.error(f"Failed to connect to device: {dev.get_id()}")
+
                 if snode.jm_device:
                     jm_device = snode.jm_device
                     logger.info(f"Node JM: {jm_device}")
@@ -124,6 +159,7 @@ while True:
                         logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
                     else:
                         logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
+                        # todo: try add
                     node_devices_check &= ret
 
                 if snode.enable_ha_jm:
@@ -134,6 +170,14 @@ while True:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip,
+                                remote_device.nvmf_port)
+                            if ret:
+                                logger.info(f"Successfully connected to device: {remote_device.get_id()}")
+                            else:
+                                logger.error(f"Failed to connect to device: {remote_device.get_id()}")
+
                         # node_remote_devices_check &= bool(ret)
 
                 lvstore_check = True

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/services/storage_node_monitor.py

@@ -154,7 +154,6 @@ while True:
                 logger.info(f"Check 2: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
             # 2- check node API
-
             node_api_check = health_controller._check_node_api(snode.mgmt_ip)
             logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
@@ -162,7 +161,12 @@ while True:
             spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
             logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
 
-            is_node_online = ping_check and node_api_check and spdk_process
+            # 4- check rpc
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            is_node_online = ping_check and node_api_check and spdk_process and node_rpc_check
             if is_node_online:
                 set_node_online(snode)
 

{sbcli_dev-4.0.16 → sbcli_dev-4.0.18}/simplyblock_core/storage_node_ops.py

@@ -751,7 +751,7 @@ def _connect_to_remote_devs(this_node):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     remote_devices = []
     # connect to remote devs
@@ -771,6 +771,8 @@ def _connect_to_remote_devs(this_node):
             if ret:
                 logger.info(f"bdev found {bdev_name}")
             else:
+                ret = rpc_client.bdev_nvme_detach_controller(name)
+                time.sleep(1)
                 ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
                 if not ret:
                     logger.error(f"Failed to connect to device: {dev.get_id()}")
@@ -786,7 +788,7 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     node_bdevs = rpc_client.get_bdevs()
     if node_bdevs:
@@ -1245,7 +1247,8 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         logger.info(f"connected to devices count: {len(node.remote_devices)}")
         time.sleep(3)
 
-    if cluster.status == cluster.STATUS_UNREADY:
+    if cluster.status != cluster.STATUS_ACTIVE:
+        logger.warning(f"The cluster status is not active ({cluster.status}), adding the node without distribs and lvstore")
         logger.info("Done")
         return "Success"
 
@@ -1737,7 +1740,8 @@ def restart_storage_node(
             continue
         tasks_controller.add_device_mig_task(dev.get_id())
 
-    if cluster.status == cluster.STATUS_UNREADY:
+    if cluster.status != cluster.STATUS_ACTIVE:
+        logger.warning(f"The cluster status is not active ({cluster.status}), adding the node without distribs and lvstore")
         logger.info("Done")
         return "Success"
     # Create distribs, raid0, and lvstore and expose lvols to the fabrics
@@ -1747,23 +1751,7 @@ def restart_storage_node(
                 snode.rpc_username, snode.rpc_password)
         ret = recreate_lvstore(snode)
         if not ret:
-            return False, "Failed to create distribs on node"
-        time.sleep(1)
-        ret = temp_rpc_client.bdev_examine(snode.raid)
-        time.sleep(1)
-        ret = temp_rpc_client.bdev_wait_for_examine()
-        time.sleep(1)
-
-        if snode.lvols:
-            for lvol_id in snode.lvols:
-                lvol = lvol_controller.recreate_lvol(lvol_id, snode)
-                if not lvol:
-                    logger.error(f"Failed to create LVol: {lvol_id}")
-                    return False
-                lvol.status = lvol.STATUS_ONLINE
-                lvol.io_error = False
-                lvol.health_check = True
-                lvol.write_to_db(db_controller.kv_store)
+            return False, "Failed to recreate lvstore on node"
 
     logger.info("Done")
     return "Success"
@@ -2043,6 +2031,10 @@ def resume_storage_node(node_id):
     if snode.jm_device and snode.jm_device.status == JMDevice.STATUS_UNAVAILABLE:
         device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
 
+    logger.info("Connecting to remote devices")
+    snode.remote_devices = _connect_to_remote_devs(snode)
+    snode.write_to_db(db_controller.kv_store)
+
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)
@@ -2583,11 +2575,34 @@ def set_node_status(node_id, status):
 
 
 def recreate_lvstore(snode):
+
     ret, err = _create_bdev_stack(snode)
     if err:
         logger.error(f"Failed to recreate lvstore on node {snode.get_id()}")
         logger.error(err)
         return False
+
+    temp_rpc_client = RPCClient(
+            snode.mgmt_ip, snode.rpc_port,
+            snode.rpc_username, snode.rpc_password)
+    time.sleep(1)
+    ret = temp_rpc_client.bdev_examine(snode.raid)
+    time.sleep(1)
+    ret = temp_rpc_client.bdev_wait_for_examine()
+    time.sleep(1)
+
+    if snode.lvols:
+        db_controller = DBController()
+        for lvol_id in snode.lvols:
+            lvol = lvol_controller.recreate_lvol(lvol_id, snode)
+            if not lvol:
+                logger.error(f"Failed to create LVol: {lvol_id}")
+                return False
+            lvol.status = lvol.STATUS_ONLINE
+            lvol.io_error = False
+            lvol.health_check = True
+            lvol.write_to_db(db_controller.kv_store)
+
     return True