sbcli-dev 3.9.5__zip → 3.9.6__zip

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.5
+Version: 3.9.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.9.5
+SIMPLY_BLOCK_VERSION=3.9.6
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.5
+Version: 3.9.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/simplyblock_core/constants.py

@@ -7,7 +7,7 @@ KVD_DB_TIMEOUT_MS = 10000
 SPK_DIR = '/home/ec2-user/spdk'
 RPC_HTTP_PROXY_PORT = 8080
 LOG_LEVEL = logging.INFO
-LOG_WEB_DEBUG = True
+LOG_WEB_DEBUG = False
 
 INSTALL_DIR = os.path.dirname(os.path.realpath(__file__))
 

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/simplyblock_core/distr_controller.py

@@ -43,10 +43,10 @@ def send_dev_status_event(device, dev_status):
     logger.debug(node_status_event)
     snodes = db_controller.get_storage_nodes_by_cluster_id(device.cluster_id)
     for node in snodes:
-        if node.status != node.STATUS_ONLINE:
+        if node.status != StorageNode.STATUS_ONLINE:
             continue
         logging.debug(f"Sending event updates, device: {storage_ID}, status: {dev_status}, node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=2)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -59,7 +59,7 @@ def disconnect_device(device):
         if node.status != node.STATUS_ONLINE:
             continue
         new_remote_devices = []
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=2)
         for rem_dev in node.remote_devices:
             if rem_dev.get_id() == device.get_id():
                 ctrl_name = rem_dev.remote_bdev[:-2]
@@ -98,7 +98,8 @@ def get_distr_cluster_map(snodes, target_node):
                         break
             if not name:
                 name = f"remote_{dev.alceml_bdev}n1"
-                dev_status = NVMeDevice.STATUS_UNAVAILABLE
+                if dev_status == NVMeDevice.STATUS_ONLINE:
+                    dev_status = NVMeDevice.STATUS_UNAVAILABLE
             logger.debug(f"Device: {dev.get_id()}, status: {dev_status}, bdev_name: {name}")
             dev_map[dev.cluster_device_order] = {
                 "UUID": dev.get_id(),

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/simplyblock_core/services/health_check_service.py

@@ -4,6 +4,7 @@ from datetime import datetime
 
 
 from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 from simplyblock_core import constants, kv_store, utils, distr_controller
@@ -107,12 +108,13 @@ while True:
                     snode.rpc_username, snode.rpc_password,
                     timeout=10, retry=1)
                 for remote_device in snode.remote_devices:
-                    ret = rpc_client.get_bdevs(remote_device.remote_bdev)
-                    if ret:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
-                    else:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    node_remote_devices_check &= bool(ret)
+                    if db_controller.get_storage_device_by_id(remote_device.get_id()).status == NVMeDevice.STATUS_ONLINE:
+                        ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                        if ret:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                        else:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                        node_remote_devices_check &= bool(ret)
 
                 if snode.jm_device:
                     jm_device = snode.jm_device

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/simplyblock_core/services/main_distr_event_collector.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+import datetime
 import threading
 import time
 
@@ -9,6 +10,7 @@ from simplyblock_core.models.lvol_model import LVol
 
 
 from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 
 
@@ -31,6 +33,17 @@ def process_device_event(event):
                     if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
                         logger.info(f"The storage device is not online, skipping. status: {dev.status}")
                         event.status = 'skipped'
+                        node_status_event = {
+                            "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
+                            "event_type": "device_status",
+                            "storage_ID": storage_id,
+                            "status": dev.status}
+                        events = {"events": [node_status_event]}
+                        snode = db_controller.get_storage_node_by_id(node_id)
+                        rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+                        ret = rpc_client.distr_status_events_update(events)
+                        if not ret:
+                            logger.warning("Failed to send event update")
                         return
 
                     device = dev
@@ -56,9 +69,27 @@ def process_device_event(event):
                     new_remote_devices.append(rem_dev)
             node.remote_devices = new_remote_devices
             node.write_to_db(db_controller.kv_store)
-            device_controller.device_set_unavailable(device_id)
+            node_status_event = {
+                "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
+                "event_type": "device_status",
+                "storage_ID": storage_id,
+                "status": NVMeDevice.STATUS_UNAVAILABLE}
+            events = {"events": [node_status_event]}
+            ret = rpc_client.distr_status_events_update(events)
+            if not ret:
+                logger.warning("Failed to send event update")
+
+            dev_node = db_controller.get_storage_node_by_id(device.node_id)
+            if dev_node.status == StorageNode.STATUS_ONLINE:
+                device_controller.device_set_io_error(device_id, True)
+                device_controller.device_set_unavailable(device_id)
 
         else:
+            if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]:
+                logger.info(f"Node is not online, skipping. status: {node.status}")
+                event.status = 'skipped'
+                return
+
             if event.message == 'SPDK_BDEV_EVENT_REMOVE':
                 if device.node_id == node_id:
                     logger.info(f"Removing storage id: {storage_id} from node: {node_id}")

{sbcli_dev-3.9.5 → sbcli_dev-3.9.6}/simplyblock_core/storage_node_ops.py

@@ -752,7 +752,7 @@ def _connect_to_remote_devs(this_node):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
 
     remote_devices = []
     # connect to remote devs
@@ -786,7 +786,7 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
 
     node_bdevs = rpc_client.get_bdevs()
     node_bdev_names = [b['name'] for b in node_bdevs]
@@ -1640,7 +1640,7 @@ def restart_storage_node(
             logger.info(f"Device not found: {db_dev.get_id()}")
             db_dev.status = NVMeDevice.STATUS_REMOVED
             removed_devices.append(db_dev)
-            # distr_controller.send_dev_status_event(db_dev, db_dev.status)
+            distr_controller.send_dev_status_event(db_dev, db_dev.status)
 
     if snode.jm_device and "serial_number" in snode.jm_device.device_data_dict:
         known_devices_sn.append(snode.jm_device.device_data_dict['serial_number'])