sbcli-dev 3.8.34__zip → 3.8.36__zip

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.34
+Version: 3.8.36
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.34
+SIMPLY_BLOCK_VERSION=3.8.36
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.34
+Version: 3.8.36
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/simplyblock_core/controllers/device_controller.py

@@ -689,3 +689,24 @@ def add_device(device_id):
 
 def device_set_failed_and_migrated(device_id):
     return device_set_state(device_id, NVMeDevice.STATUS_FAILED_AND_MIGRATED)
+
+
+def set_jm_device_state(device_id, state):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if jm_device.status == state:
+        return True
+
+    jm_device.status = state
+    snode.write_to_db(db_controller.kv_store)
+    return True

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/simplyblock_core/controllers/health_controller.py

@@ -178,6 +178,13 @@ def check_node(node_id, with_devices=True):
             else:
                 logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
             # node_remote_devices_check &= bool(ret)
+        if snode.jm_device:
+            jm_device = snode.jm_device
+            ret = check_jm_device(jm_device.get_id())
+            if ret:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
+            else:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
         lvstore_check = True
         if snode.lvstore and snode.lvstore_stack:
             distribs_list = []
@@ -226,6 +233,11 @@ def check_device(device_id):
     db_controller = DBController()
     device = db_controller.get_storage_device_by_id(device_id)
     if not device:
+        # is jm device ?
+        for node in db_controller.get_storage_nodes():
+            if node.jm_device and node.jm_device.get_id() == device_id:
+                return check_jm_device(node.jm_device.get_id())
+
         logger.error("device not found")
         return False
 
@@ -394,3 +406,44 @@ def check_snap(snap_id):
 
     ret = rpc_client.get_bdevs(snap.snap_bdev)
     return ret
+
+
+def check_jm_device(device_id):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if snode.status in [StorageNode.STATUS_OFFLINE, StorageNode.STATUS_REMOVED]:
+        logger.info(f"Skipping ,node status is {snode.status}")
+        return True
+
+    if jm_device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
+        logger.info(f"Skipping ,device status is {jm_device.status}")
+        return True
+
+    passed = True
+    try:
+        rpc_client = RPCClient(
+            snode.mgmt_ip, snode.rpc_port,
+            snode.rpc_username, snode.rpc_password)
+
+        ret = rpc_client.get_bdevs(jm_device.jm_bdev)
+        if ret:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... ok")
+        else:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... not found")
+            passed = False
+
+    except Exception as e:
+        logger.error(f"Failed to connect to node's SPDK: {e}")
+        passed = False
+
+    return passed

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/simplyblock_core/services/distr_event_collector.py

@@ -73,28 +73,27 @@ def process_device_event(event):
 
 def process_lvol_event(event):
     if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
-        vuid = event.object_dict['vuid']
-        lvol = None
+        # vuid = event.object_dict['vuid']
+        node_id = event.node_id
+        lvols = []
         for lv in db_controller.get_lvols():  # pass
-            if lv.vuid == vuid:
-                lvol = lv
-                break
+            if lv.node_id == node_id:
+                lvols.append(lv)
 
-        if not lvol:
-            logger.error(f"LVol with vuid {vuid} not found")
-            event.status = 'lvol_not_found'
+        if not lvols:
+            logger.error(f"LVols on node {node_id} not found")
+            event.status = 'lvols_not_found'
         else:
-            lvol.io_error = True
-            if lvol.status == LVol.STATUS_ONLINE:
-                logger.info("Setting LVol to offline")
-                old_status = lvol.status
-                lvol.status = LVol.STATUS_OFFLINE
-                lvol.write_to_db(db_controller.kv_store)
-                lvol_events.lvol_status_change(lvol, lvol.status, old_status, caused_by="monitor")
-                lvol_events.lvol_io_error_change(lvol, True, False, caused_by="monitor")
-                event.status = 'processed'
-            else:
-                event.status = 'skipped'
+            for lvol in lvols:
+                if lvol.status == LVol.STATUS_ONLINE:
+                    logger.info("Setting LVol to offline")
+                    lvol.io_error = True
+                    old_status = lvol.status
+                    lvol.status = LVol.STATUS_OFFLINE
+                    lvol.write_to_db(db_controller.kv_store)
+                    lvol_events.lvol_status_change(lvol, lvol.status, old_status, caused_by="monitor")
+                    lvol_events.lvol_io_error_change(lvol, True, False, caused_by="monitor")
+            event.status = 'processed'
     else:
         logger.error(f"Unknown LVol event message: {event.message}")
         event.status = "event_unknown"

{sbcli_dev-3.8.34 → sbcli_dev-3.8.36}/simplyblock_core/services/storage_node_monitor.py

@@ -5,7 +5,7 @@ import time
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, utils
 from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
 from simplyblock_core.models.cluster import Cluster
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 
@@ -113,6 +113,16 @@ def set_node_offline(node):
         tasks_controller.add_node_to_auto_restart(node)
 
 
+def set_jm_device_status(node):
+    if node.status != StorageNode.STATUS_UNREACHABLE:
+        for dev in node.nvme_devices:
+            device_controller.device_set_unavailable(dev.get_id())
+
+        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
+        # add node to auto restart
+        tasks_controller.add_node_to_auto_restart(node)
+
+
 logger.info("Starting node monitor")
 while True:
     clusters = db_controller.get_clusters()
@@ -158,6 +168,19 @@ while True:
             else:
                 set_node_offline(snode)
 
+            # check JM device
+            if snode.jm_device:
+                if snode.jm_device.status in [JMDevice.STATUS_ONLINE, JMDevice.STATUS_UNAVAILABLE]:
+                    ret = health_controller.check_jm_device(snode.jm_device.get_id())
+                    if ret:
+                        logger.error(f"JM bdev is online: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_ONLINE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+                    else:
+                        logger.error(f"JM bdev is offline: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_UNAVAILABLE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+
         update_cluster_status(cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")