sbcli-dev 3.8.35__zip → 3.8.36__zip

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.35
+Version: 3.8.36
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.35
+SIMPLY_BLOCK_VERSION=3.8.36
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.35
+Version: 3.8.36
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/simplyblock_core/controllers/device_controller.py

@@ -689,3 +689,24 @@ def add_device(device_id):
 
 def device_set_failed_and_migrated(device_id):
     return device_set_state(device_id, NVMeDevice.STATUS_FAILED_AND_MIGRATED)
+
+
+def set_jm_device_state(device_id, state):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if jm_device.status == state:
+        return True
+
+    jm_device.status = state
+    snode.write_to_db(db_controller.kv_store)
+    return True

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/simplyblock_core/controllers/health_controller.py

@@ -178,6 +178,13 @@ def check_node(node_id, with_devices=True):
             else:
                 logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
             # node_remote_devices_check &= bool(ret)
+        if snode.jm_device:
+            jm_device = snode.jm_device
+            ret = check_jm_device(jm_device.get_id())
+            if ret:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
+            else:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
         lvstore_check = True
         if snode.lvstore and snode.lvstore_stack:
             distribs_list = []
@@ -226,6 +233,11 @@ def check_device(device_id):
     db_controller = DBController()
     device = db_controller.get_storage_device_by_id(device_id)
     if not device:
+        # is jm device ?
+        for node in db_controller.get_storage_nodes():
+            if node.jm_device and node.jm_device.get_id() == device_id:
+                return check_jm_device(node.jm_device.get_id())
+
         logger.error("device not found")
         return False
 
@@ -394,3 +406,44 @@ def check_snap(snap_id):
 
     ret = rpc_client.get_bdevs(snap.snap_bdev)
     return ret
+
+
+def check_jm_device(device_id):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if snode.status in [StorageNode.STATUS_OFFLINE, StorageNode.STATUS_REMOVED]:
+        logger.info(f"Skipping ,node status is {snode.status}")
+        return True
+
+    if jm_device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
+        logger.info(f"Skipping ,device status is {jm_device.status}")
+        return True
+
+    passed = True
+    try:
+        rpc_client = RPCClient(
+            snode.mgmt_ip, snode.rpc_port,
+            snode.rpc_username, snode.rpc_password)
+
+        ret = rpc_client.get_bdevs(jm_device.jm_bdev)
+        if ret:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... ok")
+        else:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... not found")
+            passed = False
+
+    except Exception as e:
+        logger.error(f"Failed to connect to node's SPDK: {e}")
+        passed = False
+
+    return passed

{sbcli_dev-3.8.35 → sbcli_dev-3.8.36}/simplyblock_core/services/storage_node_monitor.py

@@ -5,7 +5,7 @@ import time
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, utils
 from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
 from simplyblock_core.models.cluster import Cluster
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 
@@ -113,6 +113,16 @@ def set_node_offline(node):
         tasks_controller.add_node_to_auto_restart(node)
 
 
+def set_jm_device_status(node):
+    if node.status != StorageNode.STATUS_UNREACHABLE:
+        for dev in node.nvme_devices:
+            device_controller.device_set_unavailable(dev.get_id())
+
+        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
+        # add node to auto restart
+        tasks_controller.add_node_to_auto_restart(node)
+
+
 logger.info("Starting node monitor")
 while True:
     clusters = db_controller.get_clusters()
@@ -158,6 +168,19 @@ while True:
             else:
                 set_node_offline(snode)
 
+            # check JM device
+            if snode.jm_device:
+                if snode.jm_device.status in [JMDevice.STATUS_ONLINE, JMDevice.STATUS_UNAVAILABLE]:
+                    ret = health_controller.check_jm_device(snode.jm_device.get_id())
+                    if ret:
+                        logger.error(f"JM bdev is online: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_ONLINE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+                    else:
+                        logger.error(f"JM bdev is offline: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_UNAVAILABLE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+
         update_cluster_status(cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")