sbcli-hmdi 0.0.18__tar.gz → 0.0.20__tar.gz

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sbcli-hmdi
-Version: 0.0.18
+Version: 0.0.20
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/sbcli_hmdi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sbcli-hmdi
-Version: 0.0.18
+Version: 0.0.20
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/controllers/health_controller.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-from simplyblock_core import utils, distr_controller
+from simplyblock_core import utils, distr_controller, storage_node_ops
 from simplyblock_core.db_controller import DBController
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
@@ -151,6 +151,11 @@ def _check_node_lvstore(lvstore_stack, node, auto_fix=False):
                                 if result['Kind'] == "Device":
                                     if result['Reported Status']:
                                         dev = db_controller.get_storage_device_by_id(result['UUID'])
+                                        if dev.status == NVMeDevice.STATUS_ONLINE:
+                                            node = db_controller.get_storage_node_by_id(node.get_id())
+                                            remote_devices = storage_node_ops._connect_to_remote_devs(node)
+                                            node.remote_devices = remote_devices
+                                            node.write_to_db()
                                         distr_controller.send_dev_status_event(dev, dev.status, node)
                                 if result['Kind'] == "Node":
                                     n = db_controller.get_storage_node_by_id(result['UUID'])

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/controllers/tasks_controller.py

@@ -32,7 +32,7 @@ def _validate_new_task_node_restart(cluster_id, node_id):
     tasks = db_controller.get_job_tasks(cluster_id)
     for task in tasks:
         if task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
-            if task.status != JobSchedule.STATUS_DONE:
+            if task.status != JobSchedule.STATUS_DONE and task.canceled is False:
                 return task.get_id()
     return False
 

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/distr_controller.py

@@ -32,7 +32,7 @@ def send_node_status_event(node, node_status, target_node=None):
         if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=2)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=1)
         ret = rpc_client.distr_status_events_update(events)
 
 
@@ -67,7 +67,7 @@ def send_dev_status_event(device, status, target_node=None):
             "storage_ID": storage_ID,
             "status": dev_status}]}
         logging.debug(f"Sending event updates, device: {storage_ID}, status: {dev_status}, node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=2)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=1, retry=1)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-hmdi
-SIMPLY_BLOCK_VERSION=0.0.18
+SIMPLY_BLOCK_VERSION=0.0.20
 
 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:hmdi
 DOCKER_USER=hamdysimplyblock

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/services/health_check_service.py

@@ -122,26 +122,26 @@ while True:
                             connected_devices.append(remote_device.get_id())
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                            # if not org_dev.alceml_bdev:
-                            #     logger.error(f"device alceml bdev not found!, {org_dev.get_id()}")
-                            #     continue
-                            # name = f"remote_{org_dev.alceml_bdev}"
+                            if not org_dev.alceml_bdev:
+                                logger.error(f"device alceml bdev not found!, {org_dev.get_id()}")
+                                continue
+                            name = f"remote_{org_dev.alceml_bdev}"
                             # if rpc_client.bdev_nvme_controller_list(name):
                             #     logger.info(f"detaching {name} from {snode.get_id()}")
                             #     rpc_client.bdev_nvme_detach_controller(name)
                             #     time.sleep(1)
-                            #
-                            # logger.info(f"Connecting {name} to {snode.get_id()}")
-                            # ret = rpc_client.bdev_nvme_attach_controller_tcp(
-                            #     name, org_dev.nvmf_nqn, org_dev.nvmf_ip, org_dev.nvmf_port)
-                            # if ret:
-                            #     logger.info(f"Successfully connected to device: {org_dev.get_id()}")
-                            #     remote_device.status = NVMeDevice.STATUS_ONLINE
-                            #     connected_devices.append(org_dev.get_id())
-                            #     snode.write_to_db()
-                            #     distr_controller.send_dev_status_event(org_dev, NVMeDevice.STATUS_ONLINE, snode)
-                            # else:
-                            #     logger.error(f"Failed to connect to device: {org_dev.get_id()}")
+
+                            logger.info(f"Connecting {name} to {snode.get_id()}")
+                            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                name, org_dev.nvmf_nqn, org_dev.nvmf_ip, org_dev.nvmf_port)
+                            if ret:
+                                logger.info(f"Successfully connected to device: {org_dev.get_id()}")
+                                remote_device.status = NVMeDevice.STATUS_ONLINE
+                                connected_devices.append(org_dev.get_id())
+                                snode.write_to_db()
+                                distr_controller.send_dev_status_event(org_dev, NVMeDevice.STATUS_ONLINE, snode)
+                            else:
+                                logger.error(f"Failed to connect to device: {org_dev.get_id()}")
 
                         node_remote_devices_check &= bool(ret)
 

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/services/main_distr_event_collector.py

@@ -71,9 +71,18 @@ def process_device_event(event):
             event.status = 'skipped:node_offline'
             return
 
-        logger.info(f"Setting device to unavailable")
-        device_controller.device_set_unavailable(device_obj.get_id())
-        if event.message in ['error_write', 'error_unmap', 'error_read']:
+        if device_node_obj.get_id() != event_node_obj.get_id() and event.message == 'error_open':
+            for dev in event_node_obj.remote_devices:
+                if dev.get_id() == device_obj.get_id():
+                    dev.status = NVMeDevice.STATUS_UNAVAILABLE
+                    event_node_obj.write_to_db()
+                    break
+
+        else:
+
+            logger.info(f"Setting device to unavailable")
+            device_controller.device_set_unavailable(device_obj.get_id())
+            # if event.message in ['error_write', 'error_unmap', 'error_read']:
             device_controller.device_set_io_error(device_obj.get_id(), True)
 
         event.status = 'processed'

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/services/storage_node_monitor.py

@@ -224,7 +224,8 @@ while True:
             if spdk_process:
                 # 4- check rpc
                 node_rpc_check = health_controller._check_node_rpc(
-                    snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+                    snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password,
+                    timeout=1, retry=3)
             logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
 
             is_node_online = ping_check and node_api_check and spdk_process and node_rpc_check

{sbcli_hmdi-0.0.18 → sbcli_hmdi-0.0.20}/simplyblock_core/storage_node_ops.py

@@ -2897,20 +2897,20 @@ def recreate_lvstore(snode):
     snode = db_controller.get_storage_node_by_id(snode.get_id())
     snode.remote_devices = _connect_to_remote_devs(snode)
     if snode.enable_ha_jm:
-        online_devs = []
-        for remote_device in snode.remote_jm_devices:
-            if remote_device.status == StorageNode.STATUS_ONLINE:
-                online_devs.append(remote_device)
-
-        if len(online_devs) < 2:
-            devs = get_sorted_ha_jms(snode)
-            for did in devs:
-                dev = db_controller.get_jm_device_by_id(did)
-                online_devs.append(dev)
-                if len(online_devs) > snode.ha_jm_count - 1:
-                    break
-
-        snode.remote_jm_devices = online_devs
+        # online_devs = []
+        # for remote_device in snode.remote_jm_devices:
+        #     if remote_device.status == StorageNode.STATUS_ONLINE:
+        #         online_devs.append(remote_device)
+        #
+        # if len(online_devs) < 2:
+        #     devs = get_sorted_ha_jms(snode)
+        #     for did in devs:
+        #         dev = db_controller.get_jm_device_by_id(did)
+        #         online_devs.append(dev)
+        #         if len(online_devs) > snode.ha_jm_count - 1:
+        #             break
+        #
+        # snode.remote_jm_devices = online_devs
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
     snode.write_to_db()
 
@@ -2953,6 +2953,9 @@ def recreate_lvstore(snode):
             lvol_obj.health_check = True
         lvol_obj.write_to_db()
 
+    if snode.jm_vuid:
+        ret = rpc_client.jc_explicit_synchronization(snode.jm_vuid)
+        logger.info(f"JM Sync res: {ret}")
 
     sec_node_api = SNodeClient(sec_node.api_endpoint)
     if prim_node_suspend: