sbcli-dev 4.0.17__zip → 4.0.18__zip

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.17
+Version: 4.0.18
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.17
+SIMPLY_BLOCK_VERSION=4.0.18
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.17
+Version: 4.0.18
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/controllers/health_controller.py

@@ -446,7 +446,7 @@ def check_jm_device(device_id):
     try:
         rpc_client = RPCClient(
             snode.mgmt_ip, snode.rpc_port,
-            snode.rpc_username, snode.rpc_password)
+            snode.rpc_username, snode.rpc_password, timeout=3, retry=2)
 
         ret = rpc_client.get_bdevs(jm_device.jm_bdev)
         if ret:

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/controllers/tasks_controller.py

@@ -52,6 +52,11 @@ def _add_task(function_name, cluster_id, node_id, device_id,
         if task_id:
             logger.info(f"Task found, skip adding new task: {task_id}")
             return False
+    elif function_name == JobSchedule.FN_DEV_MIG:
+        task_id = get_device_mig_task(cluster_id, node_id, function_params['distr_name'])
+        if task_id:
+            logger.info(f"Task found, skip adding new task: {task_id}")
+            return False
 
     task_obj = JobSchedule()
     task_obj.uuid = str(uuid.uuid4())
@@ -72,6 +77,8 @@ def _add_task(function_name, cluster_id, node_id, device_id,
 def add_device_mig_task(device_id):
     device = db_controller.get_storage_devices(device_id)
     for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+        if not node.lvols:
+            continue
         for bdev in node.lvstore_stack:
             if bdev['type'] == "bdev_distr":
                 _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
@@ -190,6 +197,7 @@ def get_active_node_task(cluster_id, node_id):
                 return task.uuid
     return False
 
+
 def get_new_device_mig_task(cluster_id, node_id, distr_name):
     tasks = db_controller.get_job_tasks(cluster_id)
     for task in tasks:
@@ -198,3 +206,13 @@ def get_new_device_mig_task(cluster_id, node_id, distr_name):
                     and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
                 return task.uuid
     return False
+
+
+def get_device_mig_task(cluster_id, node_id, distr_name):
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_DEV_MIG and task.node_id == node_id:
+            if task.status != JobSchedule.STATUS_DONE and task.canceled is False \
+                    and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
+                return task.uuid
+    return False

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/rpc_client.py

@@ -482,11 +482,11 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 1,
+            # "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
-            "ctrlr_loss_timeout_sec": 1,
+            # "ctrlr_loss_timeout_sec": 1,
             "multipath":"disable",
-            "reconnect_delay_sec":1
+            # "reconnect_delay_sec":1
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -499,7 +499,7 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fast_io_fail_timeout_sec": 1,
-            "ctrlr_loss_timeout_sec": 3,
+            "ctrlr_loss_timeout_sec": 1,
             "reconnect_delay_sec": 1,
             "fabrics_connect_timeout_us": 100000,
             "multipath":"disable"
@@ -559,6 +559,7 @@ class RPCClient:
 
     def bdev_nvme_set_options(self):
         params = {
+            # "action_on_timeout": "abort",
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
             "ctrlr_loss_timeout_sec": 2,

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/services/health_check_service.py

@@ -115,6 +115,8 @@ while True:
                             connected_devices.append(remote_device.get_id())
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_detach_controller(remote_device.remote_bdev)
+                            time.sleep(1)
                             ret = rpc_client.bdev_nvme_attach_controller_tcp(
                                 remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip, remote_device.nvmf_port)
                             if ret:
@@ -135,6 +137,8 @@ while True:
                                 logger.info(f"connecting to online device: {dev.get_id()}")
                                 name = f"remote_{dev.alceml_bdev}"
                                 bdev_name = f"{name}n1"
+                                ret = rpc_client.bdev_nvme_detach_controller(name)
+                                time.sleep(1)
                                 ret = rpc_client.bdev_nvme_attach_controller_tcp(
                                     name, dev.nvmf_nqn, dev.nvmf_ip,
                                     dev.nvmf_port)

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/services/storage_node_monitor.py

@@ -154,7 +154,6 @@ while True:
                 logger.info(f"Check 2: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
             # 2- check node API
-
             node_api_check = health_controller._check_node_api(snode.mgmt_ip)
             logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
@@ -162,7 +161,12 @@ while True:
             spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
             logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
 
-            is_node_online = ping_check and node_api_check and spdk_process
+            # 4- check rpc
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            is_node_online = ping_check and node_api_check and spdk_process and node_rpc_check
             if is_node_online:
                 set_node_online(snode)
 

{sbcli_dev-4.0.17 → sbcli_dev-4.0.18}/simplyblock_core/storage_node_ops.py

@@ -751,7 +751,7 @@ def _connect_to_remote_devs(this_node):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     remote_devices = []
     # connect to remote devs
@@ -771,6 +771,8 @@ def _connect_to_remote_devs(this_node):
             if ret:
                 logger.info(f"bdev found {bdev_name}")
             else:
+                ret = rpc_client.bdev_nvme_detach_controller(name)
+                time.sleep(1)
                 ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
                 if not ret:
                     logger.error(f"Failed to connect to device: {dev.get_id()}")
@@ -786,7 +788,7 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     node_bdevs = rpc_client.get_bdevs()
     if node_bdevs:
@@ -2029,6 +2031,10 @@ def resume_storage_node(node_id):
     if snode.jm_device and snode.jm_device.status == JMDevice.STATUS_UNAVAILABLE:
         device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
 
+    logger.info("Connecting to remote devices")
+    snode.remote_devices = _connect_to_remote_devs(snode)
+    snode.write_to_db(db_controller.kv_store)
+
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)