sbcli-dev 3.8.47__zip → 3.8.49__zip

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.47
+Version: 3.8.49
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.47
+SIMPLY_BLOCK_VERSION=3.8.49
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.47
+Version: 3.8.49
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_cli/cli.py

@@ -96,6 +96,8 @@ class CLIWrapper:
         sub_command.add_argument("--iobuf_small_bufsize", help='bdev_set_options param', dest='small_bufsize',  type=int, default=0)
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
 
+        sub_command.add_argument("--force", help='Force restart', required=False, action='store_true')
+
         # sub_command.add_argument("-t", '--test', help='Run smart test on the NVMe devices', action='store_true')
 
         # Shutdown storage node
@@ -773,7 +775,7 @@ class CLIWrapper:
                 ret = storage_ops.restart_storage_node(
                     node_id, max_lvol, max_snap, max_prov,
                     spdk_image, spdk_debug,
-                    small_bufsize, large_bufsize, number_of_devices, node_ip=args.node_ip)
+                    small_bufsize, large_bufsize, number_of_devices, node_ip=args.node_ip, force=args.force)
 
             elif sub_command == "list-devices":
                 ret = self.storage_node_list_devices(args)

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/distr_controller.py

@@ -27,7 +27,7 @@ def send_node_status_event(node, node_status):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
         ret = rpc_client.distr_status_events_update(events)
 
 
@@ -47,7 +47,7 @@ def send_dev_status_event(device, dev_status):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -108,8 +108,11 @@ def get_distr_cluster_map(snodes, target_node):
             dev_w_map.append({
                 "weight": dev_w,
                 "id": dev.cluster_device_order})
+        node_status = snode.status
+        if node_status == StorageNode.STATUS_SCHEDULABLE:
+            node_status = StorageNode.STATUS_UNREACHABLE
         map_cluster[snode.get_id()] = {
-            "status": snode.status,
+            "status": node_status,
             "devices": dev_map}
         map_prob.append({
             "weight": node_w,
@@ -186,7 +189,7 @@ def parse_distr_cluster_map(map_string):
 def send_cluster_map_to_node(node):
     db_controller = DBController()
     snodes = db_controller.get_storage_nodes_by_cluster_id(node.cluster_id)
-    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
     cluster_map_data = get_distr_cluster_map(snodes, node)
     cluster_map_data['UUID_node_target'] = node.get_id()
     ret = rpc_client.distr_send_cluster_map(cluster_map_data)
@@ -204,7 +207,7 @@ def send_cluster_map_add_node(snode):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
 
         cluster_map_data = get_distr_cluster_map([snode], node)
         cl_map = {

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/models/storage_node.py

@@ -19,6 +19,7 @@ class StorageNode(BaseModel):
 
     STATUS_IN_CREATION = 'in_creation'
     STATUS_UNREACHABLE = 'unreachable'
+    STATUS_SCHEDULABLE = 'schedulable'
 
     STATUS_CODE_MAP = {
         STATUS_ONLINE: 0,

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/services/storage_node_monitor.py

@@ -107,20 +107,7 @@ def set_node_offline(node):
     if node.status != StorageNode.STATUS_UNREACHABLE:
         for dev in node.nvme_devices:
             device_controller.device_set_unavailable(dev.get_id())
-
         storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
-        # add node to auto restart
-        tasks_controller.add_node_to_auto_restart(node)
-
-
-def set_jm_device_status(node):
-    if node.status != StorageNode.STATUS_UNREACHABLE:
-        for dev in node.nvme_devices:
-            device_controller.device_set_unavailable(dev.get_id())
-
-        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
-        # add node to auto restart
-        tasks_controller.add_node_to_auto_restart(node)
 
 
 logger.info("Starting node monitor")
@@ -167,6 +154,15 @@ while True:
                 set_node_online(snode)
             else:
                 set_node_offline(snode)
+                if ping_check and node_api_check and not spdk_process:
+                    # add node to auto restart
+                    if cluster.status != Cluster.STATUS_UNREADY:
+                        tasks_controller.add_node_to_auto_restart(snode)
+
+                if not ping_check and not node_api_check and not spdk_process:
+                    # restart on new node
+                    storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_SCHEDULABLE)
+                continue
 
             # check JM device
             if snode.jm_device:

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/services/tasks_runner_failed_migration.py

@@ -20,7 +20,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -40,6 +44,8 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password,
+                           timeout=5, retry=2)
     if "migration" not in task.function_params:
         device = db_controller.get_storage_devices(task.device_id)
         distr_name = task.function_params["distr_name"]
@@ -50,6 +56,7 @@ def task_runner(task):
             task.write_to_db(db_controller.kv_store)
             return True
 
+
         rsp = rpc_client.distr_migration_failure_start(distr_name, device.cluster_device_order)
         if not rsp:
             logger.error(f"Failed to start device migration task, storage_ID: {device.cluster_device_order}")

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/services/tasks_runner_migration.py

@@ -17,7 +17,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -37,11 +41,15 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password,
+                           timeout=5, retry=2)
     if "migration" not in task.function_params:
         all_devs_online = True
         for node in db_controller.get_storage_nodes_by_cluster_id(task.cluster_id):
             for dev in node.nvme_devices:
-                if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_FAILED_AND_MIGRATED]:
+                if dev.status not in [NVMeDevice.STATUS_ONLINE,
+                                      NVMeDevice.STATUS_FAILED,
+                                      NVMeDevice.STATUS_FAILED_AND_MIGRATED]:
                     all_devs_online = False
                     break
 

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/services/tasks_runner_new_dev_migration.py

@@ -20,7 +20,11 @@ from simplyblock_core.rpc_client import RPCClient
 def task_runner(task):
 
     snode = db_controller.get_storage_node_by_id(task.node_id)
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
+    if not snode:
+        task.status = JobSchedule.STATUS_DONE
+        task.function_result = f"Node not found: {task.node_id}"
+        task.write_to_db(db_controller.kv_store)
+        return True
 
     if task.canceled:
         task.function_result = "canceled"
@@ -40,6 +44,7 @@ def task_runner(task):
         task.write_to_db(db_controller.kv_store)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
     if "migration" not in task.function_params:
         all_devs_online = True
         for node in db_controller.get_storage_nodes_by_cluster_id(task.cluster_id):

{sbcli_dev-3.8.47 → sbcli_dev-3.8.49}/simplyblock_core/storage_node_ops.py

@@ -1372,7 +1372,8 @@ def restart_storage_node(
 
     if snode.status == StorageNode.STATUS_RESTARTING:
         logger.error(f"Node is in restart: {node_id}")
-        return False
+        if force is False:
+            return False
 
     task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
     if task_id: