sbcli-dev 4.0.17__zip → 4.0.19__zip

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.17
+Version: 4.0.19
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.17
+SIMPLY_BLOCK_VERSION=4.0.19
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.17
+Version: 4.0.19
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/controllers/health_controller.py

@@ -446,7 +446,7 @@ def check_jm_device(device_id):
     try:
         rpc_client = RPCClient(
             snode.mgmt_ip, snode.rpc_port,
-            snode.rpc_username, snode.rpc_password)
+            snode.rpc_username, snode.rpc_password, timeout=3, retry=2)
 
         ret = rpc_client.get_bdevs(jm_device.jm_bdev)
         if ret:

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/controllers/tasks_controller.py

@@ -52,6 +52,11 @@ def _add_task(function_name, cluster_id, node_id, device_id,
         if task_id:
             logger.info(f"Task found, skip adding new task: {task_id}")
             return False
+    elif function_name == JobSchedule.FN_DEV_MIG:
+        task_id = get_device_mig_task(cluster_id, node_id, function_params['distr_name'])
+        if task_id:
+            logger.info(f"Task found, skip adding new task: {task_id}")
+            return False
 
     task_obj = JobSchedule()
     task_obj.uuid = str(uuid.uuid4())
@@ -72,6 +77,8 @@ def _add_task(function_name, cluster_id, node_id, device_id,
 def add_device_mig_task(device_id):
     device = db_controller.get_storage_devices(device_id)
     for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+        if not node.lvols:
+            continue
         for bdev in node.lvstore_stack:
             if bdev['type'] == "bdev_distr":
                 _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
@@ -190,6 +197,7 @@ def get_active_node_task(cluster_id, node_id):
                 return task.uuid
     return False
 
+
 def get_new_device_mig_task(cluster_id, node_id, distr_name):
     tasks = db_controller.get_job_tasks(cluster_id)
     for task in tasks:
@@ -198,3 +206,13 @@ def get_new_device_mig_task(cluster_id, node_id, distr_name):
                     and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
                 return task.uuid
     return False
+
+
+def get_device_mig_task(cluster_id, node_id, distr_name):
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_DEV_MIG and task.node_id == node_id:
+            if task.status != JobSchedule.STATUS_DONE and task.canceled is False \
+                    and "distr_name" in task.function_params and task.function_params["distr_name"] == distr_name:
+                return task.uuid
+    return False

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/rpc_client.py

@@ -482,27 +482,11 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 1,
+            # "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
-            "ctrlr_loss_timeout_sec": 1,
+            # "ctrlr_loss_timeout_sec": 1,
             "multipath":"disable",
-            "reconnect_delay_sec":1
-        }
-        return self._request("bdev_nvme_attach_controller", params)
-
-    def bdev_nvme_attach_controller_tcp_jm(self, name, nqn, ip, port):
-        params = {
-            "name": name,
-            "trtype": "tcp",
-            "traddr": ip,
-            "adrfam": "ipv4",
-            "trsvcid": str(port),
-            "subnqn": nqn,
-            "fast_io_fail_timeout_sec": 1,
-            "ctrlr_loss_timeout_sec": 3,
-            "reconnect_delay_sec": 1,
-            "fabrics_connect_timeout_us": 100000,
-            "multipath":"disable"
+            # "reconnect_delay_sec":1
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -559,6 +543,7 @@ class RPCClient:
 
     def bdev_nvme_set_options(self):
         params = {
+            # "action_on_timeout": "abort",
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
             "ctrlr_loss_timeout_sec": 2,

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/services/health_check_service.py

@@ -115,6 +115,8 @@ while True:
                             connected_devices.append(remote_device.get_id())
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_detach_controller(remote_device.remote_bdev)
+                            time.sleep(1)
                             ret = rpc_client.bdev_nvme_attach_controller_tcp(
                                 remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip, remote_device.nvmf_port)
                             if ret:
@@ -127,7 +129,7 @@ while True:
                         node_remote_devices_check &= bool(ret)
 
                 for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
-                    if node.status != StorageNode.STATUS_ONLINE:
+                    if node.status != StorageNode.STATUS_ONLINE or node.get_id() == snode.get_id():
                         continue
                     for dev in node.nvme_devices:
                         if dev.status == StorageNode.STATUS_ONLINE:
@@ -135,6 +137,8 @@ while True:
                                 logger.info(f"connecting to online device: {dev.get_id()}")
                                 name = f"remote_{dev.alceml_bdev}"
                                 bdev_name = f"{name}n1"
+                                ret = rpc_client.bdev_nvme_detach_controller(name)
+                                time.sleep(1)
                                 ret = rpc_client.bdev_nvme_attach_controller_tcp(
                                     name, dev.nvmf_nqn, dev.nvmf_ip,
                                     dev.nvmf_port)
@@ -200,6 +204,7 @@ while True:
                                     logger.info(f"Checking Distr map ... {is_passed}")
                                 else:
                                     logger.error("Failed to parse distr cluster map")
+
                                 lvstore_check &= is_passed
                         else:
                             logger.info(f"Checking distr bdev : {distr} ... not found")

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/services/main_distr_event_collector.py

@@ -48,8 +48,9 @@ def process_device_event(event):
         device_id = device.get_id()
         node = db_controller.get_storage_node_by_id(node_id)
         if device.node_id != node_id:
-            logger.info(f"Setting storage id: {storage_id} unavailable")
-            distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_UNAVAILABLE, node)
+            if event.message != 'SPDK_BDEV_EVENT_REMOVE':
+                logger.info(f"Setting storage id: {storage_id} unavailable")
+                distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_UNAVAILABLE, node)
             # if device.status == NVMeDevice.STATUS_ONLINE:
             #     device_controller.device_set_io_error(device_id, True)
             #     device_controller.device_set_unavailable(device_id)

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/services/storage_node_monitor.py

@@ -154,7 +154,6 @@ while True:
                 logger.info(f"Check 2: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
             # 2- check node API
-
             node_api_check = health_controller._check_node_api(snode.mgmt_ip)
             logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
@@ -162,7 +161,12 @@ while True:
             spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip)
             logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {spdk_process}")
 
-            is_node_online = ping_check and node_api_check and spdk_process
+            # 4- check rpc
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            is_node_online = ping_check and node_api_check and spdk_process and node_rpc_check
             if is_node_online:
                 set_node_online(snode)
 

{sbcli_dev-4.0.17 → sbcli_dev-4.0.19}/simplyblock_core/storage_node_ops.py

@@ -751,7 +751,7 @@ def _connect_to_remote_devs(this_node):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     remote_devices = []
     # connect to remote devs
@@ -771,6 +771,8 @@ def _connect_to_remote_devs(this_node):
             if ret:
                 logger.info(f"bdev found {bdev_name}")
             else:
+                ret = rpc_client.bdev_nvme_detach_controller(name)
+                time.sleep(1)
                 ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
                 if not ret:
                     logger.error(f"Failed to connect to device: {dev.get_id()}")
@@ -786,7 +788,7 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
     node_bdevs = rpc_client.get_bdevs()
     if node_bdevs:
@@ -2029,6 +2031,11 @@ def resume_storage_node(node_id):
     if snode.jm_device and snode.jm_device.status == JMDevice.STATUS_UNAVAILABLE:
         device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
 
+    logger.info("Connecting to remote devices")
+    snode = db_controller.get_storage_node_by_id(node_id)
+    snode.remote_devices = _connect_to_remote_devs(snode)
+    snode.write_to_db(db_controller.kv_store)
+
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)