sbcli-dev 3.8.46__zip → 3.8.48__zip

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.46
+Version: 3.8.48
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.46
+SIMPLY_BLOCK_VERSION=3.8.48
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.46
+Version: 3.8.48
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/constants.py

@@ -58,7 +58,6 @@ TASK_EXEC_INTERVAL_SEC = 30
 TASK_EXEC_RETRY_COUNT = 8
 
 SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:latest"
-SIMPLY_BLOCK_SPDK_CORE_IMAGE_ARM64 = "simplyblock/spdk-core:latest-arm64"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:main-latest"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE_JM = "simplyblock/spdk:feature-journal-ha-latest"
 

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/controllers/health_controller.py

@@ -66,11 +66,11 @@ def _check_node_docker_api(ip):
     # return False
 
 
-def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password):
+def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password, timeout=60, retry=3):
     try:
         rpc_client = RPCClient(
             rpc_ip, rpc_port, rpc_username, rpc_password,
-            timeout=60, retry=3)
+            timeout=timeout, retry=retry)
         ret = rpc_client.get_version()
         if ret:
             logger.debug(f"SPDK version: {ret['version']}")
@@ -319,7 +319,7 @@ def check_remote_device(device_id):
             if node.get_id() == snode.get_id():
                 continue
             logger.info(f"Connecting to node: {node.get_id()}")
-            rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+            rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
             name = f"remote_{device.alceml_bdev}n1"
             ret = rpc_client.get_bdevs(name)
             if ret:

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/distr_controller.py

@@ -27,7 +27,7 @@ def send_node_status_event(node, node_status):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
         ret = rpc_client.distr_status_events_update(events)
 
 
@@ -47,7 +47,7 @@ def send_dev_status_event(device, dev_status):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -60,7 +60,7 @@ def disconnect_device(device):
         if node.status != node.STATUS_ONLINE:
             continue
         new_remote_devices = []
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
         for rem_dev in node.remote_devices:
             if rem_dev.get_id() == device.get_id():
                 ctrl_name = rem_dev.remote_bdev[:-2]
@@ -186,7 +186,7 @@ def parse_distr_cluster_map(map_string):
 def send_cluster_map_to_node(node):
     db_controller = DBController()
     snodes = db_controller.get_storage_nodes_by_cluster_id(node.cluster_id)
-    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
     cluster_map_data = get_distr_cluster_map(snodes, node)
     cluster_map_data['UUID_node_target'] = node.get_id()
     ret = rpc_client.distr_send_cluster_map(cluster_map_data)
@@ -204,7 +204,7 @@ def send_cluster_map_add_node(snode):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
 
         cluster_map_data = get_distr_cluster_map([snode], node)
         cl_map = {

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/services/storage_node_monitor.py

@@ -107,20 +107,7 @@ def set_node_offline(node):
     if node.status != StorageNode.STATUS_UNREACHABLE:
         for dev in node.nvme_devices:
             device_controller.device_set_unavailable(dev.get_id())
-
-        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
-        # add node to auto restart
-        tasks_controller.add_node_to_auto_restart(node)
-
-
-def set_jm_device_status(node):
-    if node.status != StorageNode.STATUS_UNREACHABLE:
-        for dev in node.nvme_devices:
-            device_controller.device_set_unavailable(dev.get_id())
-
         storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
-        # add node to auto restart
-        tasks_controller.add_node_to_auto_restart(node)
 
 
 logger.info("Starting node monitor")
@@ -152,11 +139,11 @@ while True:
             spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
             logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
 
-            node_rpc_check = True
+            # node_rpc_check = True
             # 3- check node RPC
-            # node_rpc_check = health_controller._check_node_rpc(
-            #     snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-            # logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=1)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
 
             # 4- docker API
             node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
@@ -167,6 +154,10 @@ while True:
                 set_node_online(snode)
             else:
                 set_node_offline(snode)
+                if ping_check and node_api_check and not spdk_process:
+                    # add node to auto restart
+                    if cluster.status != Cluster.STATUS_UNREADY:
+                        tasks_controller.add_node_to_auto_restart(snode)
 
             # check JM device
             if snode.jm_device:

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/services/tasks_runner_failed_migration.py

@@ -20,7 +20,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -40,6 +44,8 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password,
+                           timeout=5, retry=2)
     if "migration" not in task.function_params:
         device = db_controller.get_storage_devices(task.device_id)
         distr_name = task.function_params["distr_name"]
@@ -50,6 +56,7 @@ def task_runner(task):
             task.write_to_db(db_controller.kv_store)
             return True
 
+
         rsp = rpc_client.distr_migration_failure_start(distr_name, device.cluster_device_order)
         if not rsp:
             logger.error(f"Failed to start device migration task, storage_ID: {device.cluster_device_order}")

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/services/tasks_runner_migration.py

@@ -17,7 +17,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -37,11 +41,15 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password,
+                           timeout=5, retry=2)
     if "migration" not in task.function_params:
         all_devs_online = True
         for node in db_controller.get_storage_nodes_by_cluster_id(task.cluster_id):
             for dev in node.nvme_devices:
-                if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_FAILED_AND_MIGRATED]:
+                if dev.status not in [NVMeDevice.STATUS_ONLINE,
+                                      NVMeDevice.STATUS_FAILED,
+                                      NVMeDevice.STATUS_FAILED_AND_MIGRATED]:
                     all_devs_online = False
                     break
 

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/services/tasks_runner_new_dev_migration.py

@@ -20,7 +20,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -40,6 +44,7 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
     if "migration" not in task.function_params:
         all_devs_online = True
         for node in db_controller.get_storage_nodes_by_cluster_id(task.cluster_id):

{sbcli_dev-3.8.46 → sbcli_dev-3.8.48}/simplyblock_core/storage_node_ops.py

@@ -1853,7 +1853,7 @@ def shutdown_storage_node(node_id, force=False):
     storage_events.snode_status_change(snode, snode.status, old_status)
 
     rpc_client = RPCClient(
-        snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+        snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=10, retry=1)
 
     logger.debug("Removing LVols")
     _remove_bdev_stack(snode.lvstore_stack, rpc_client, remove_distr_only=True)