sbcli-dev 3.8.35__zip → 3.8.37__zip

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.35
+Version: 3.8.37
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.35
+SIMPLY_BLOCK_VERSION=3.8.37
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.35
+Version: 3.8.37
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_cli/cli.py

@@ -56,6 +56,7 @@ class CLIWrapper:
         sub_command.add_argument("--iobuf_small_bufsize", help='bdev_set_options param', dest='small_bufsize',  type=int, default=0)
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
         sub_command.add_argument("--enable-test-device", help='Enable creation of test device', action='store_true')
+        sub_command.add_argument("--enable-ha-jm", help='Enable HA JM for ditrib creation', action='store_true')
 
 
         # delete storage node
@@ -717,6 +718,7 @@ class CLIWrapper:
                 max_prov = self.parse_size(args.max_prov)
                 number_of_devices = args.number_of_devices
                 enable_test_device = args.enable_test_device
+                enable_ha_jm = args.enable_ha_jm
                 number_of_distribs = args.number_of_distribs
                 if max_prov < 1 * 1024 * 1024 * 1024:
                     return f"Max provisioning memory:{args.max_prov} must be larger than 1G"
@@ -739,7 +741,8 @@ class CLIWrapper:
                     number_of_devices=number_of_devices,
                     enable_test_device=enable_test_device,
                     namespace=None,
-                    number_of_distribs=number_of_distribs
+                    number_of_distribs=number_of_distribs,
+                    enable_ha_jm=enable_ha_jm
                 )
 
                 return out

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/cluster_ops.py

@@ -369,7 +369,7 @@ def cluster_activate(cl_id):
             logger.info(f"Node {snode.get_id()} already has lvstore {snode.lvstore}... skipping")
             continue
         ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
-                                              cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size)
+                                              cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size, snodes)
         if not ret:
             logger.error("Failed to activate cluster")
             set_cluster_status(cl_id, Cluster.STATUS_UNREADY)
@@ -688,14 +688,14 @@ def update_cluster(cl_id):
     except Exception as e:
         print(e)
 
-    for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
-        node_docker = docker.DockerClient(base_url=f"tcp://{node.mgmt_ip}:2375", version="auto")
-        logger.info(f"Pulling image {constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}")
-        node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
-        if node.status == StorageNode.STATUS_ONLINE:
-            storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
-            time.sleep(3)
-        storage_node_ops.restart_storage_node(node.get_id())
+    # for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
+    #     node_docker = docker.DockerClient(base_url=f"tcp://{node.mgmt_ip}:2375", version="auto")
+    #     logger.info(f"Pulling image {constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}")
+    #     node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
+    #     if node.status == StorageNode.STATUS_ONLINE:
+    #         storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
+    #         time.sleep(3)
+    #     storage_node_ops.restart_storage_node(node.get_id())
 
     logger.info("Done")
     return True

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/constants.py

@@ -11,7 +11,7 @@ LOG_WEB_DEBUG = True
 
 INSTALL_DIR = os.path.dirname(os.path.realpath(__file__))
 
-NODE_MONITOR_INTERVAL_SEC = 3
+NODE_MONITOR_INTERVAL_SEC = 10
 DEVICE_MONITOR_INTERVAL_SEC = 5
 STAT_COLLECTOR_INTERVAL_SEC = 60*5  # 5 minutes
 LVOL_STAT_COLLECTOR_INTERVAL_SEC = 5
@@ -52,13 +52,15 @@ GRAYLOG_CHECK_INTERVAL_SEC = 60
 
 FDB_CHECK_INTERVAL_SEC = 60
 
-SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:main"
+SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:dev"
 SIMPLY_BLOCK_CLI_NAME = "sbcli-dev"
 TASK_EXEC_INTERVAL_SEC = 30
 TASK_EXEC_RETRY_COUNT = 8
 
 SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:latest"
+SIMPLY_BLOCK_SPDK_CORE_IMAGE_ARM64 = "simplyblock/spdk-core:latest-arm64"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:348-temp-migration-2-latest"
+SIMPLY_BLOCK_SPDK_ULTRA_IMAGE_JM = "simplyblock/spdk:feature-journal-ha-latest"
 
 GELF_PORT = 12201
 

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/controllers/device_controller.py

@@ -689,3 +689,24 @@ def add_device(device_id):
 
 def device_set_failed_and_migrated(device_id):
     return device_set_state(device_id, NVMeDevice.STATUS_FAILED_AND_MIGRATED)
+
+
+def set_jm_device_state(device_id, state):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if jm_device.status == state:
+        return True
+
+    jm_device.status = state
+    snode.write_to_db(db_controller.kv_store)
+    return True

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/controllers/health_controller.py

@@ -178,6 +178,13 @@ def check_node(node_id, with_devices=True):
             else:
                 logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
             # node_remote_devices_check &= bool(ret)
+        if snode.jm_device:
+            jm_device = snode.jm_device
+            ret = check_jm_device(jm_device.get_id())
+            if ret:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
+            else:
+                logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
         lvstore_check = True
         if snode.lvstore and snode.lvstore_stack:
             distribs_list = []
@@ -226,6 +233,11 @@ def check_device(device_id):
     db_controller = DBController()
     device = db_controller.get_storage_device_by_id(device_id)
     if not device:
+        # is jm device ?
+        for node in db_controller.get_storage_nodes():
+            if node.jm_device and node.jm_device.get_id() == device_id:
+                return check_jm_device(node.jm_device.get_id())
+
         logger.error("device not found")
         return False
 
@@ -394,3 +406,44 @@ def check_snap(snap_id):
 
     ret = rpc_client.get_bdevs(snap.snap_bdev)
     return ret
+
+
+def check_jm_device(device_id):
+    db_controller = DBController()
+    jm_device = None
+    snode = None
+    for node in db_controller.get_storage_nodes():
+        if node.jm_device.get_id() == device_id:
+            jm_device = node.jm_device
+            snode = node
+            break
+    if not jm_device:
+        logger.error("device not found")
+        return False
+
+    if snode.status in [StorageNode.STATUS_OFFLINE, StorageNode.STATUS_REMOVED]:
+        logger.info(f"Skipping ,node status is {snode.status}")
+        return True
+
+    if jm_device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
+        logger.info(f"Skipping ,device status is {jm_device.status}")
+        return True
+
+    passed = True
+    try:
+        rpc_client = RPCClient(
+            snode.mgmt_ip, snode.rpc_port,
+            snode.rpc_username, snode.rpc_password)
+
+        ret = rpc_client.get_bdevs(jm_device.jm_bdev)
+        if ret:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... ok")
+        else:
+            logger.debug(f"Checking bdev: {jm_device.jm_bdev} ... not found")
+            passed = False
+
+    except Exception as e:
+        logger.error(f"Failed to connect to node's SPDK: {e}")
+        passed = False
+
+    return passed

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/controllers/lvol_controller.py

@@ -169,6 +169,21 @@ def get_jm_names(snode):
     return [snode.jm_device.jm_bdev] if snode.jm_device else []
 
 
+def get_ha_jm_names(current_node, snode_list):
+    jm_list = []
+    if current_node.jm_device:
+        jm_list.append(current_node.jm_device.jm_bdev)
+    else:
+        jm_list.append("JM_LOCAL")
+
+    for node in snode_list:
+        if node.get_id() == current_node.get_id():
+            continue
+        name = f"remote_{node.jm_device.jm_bdev}n1"
+        jm_list.append(name)
+    return jm_list[:3]
+
+
 def _get_next_3_nodes(cluster_id, lvol_size=0):
     snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     online_nodes = []
@@ -564,7 +579,7 @@ def _create_bdev_stack(lvol, snode, ha_comm_addrs, ha_inode_self):
         ret = None
 
         if type == "bdev_distr":
-            params['jm_names'] = get_jm_names(snode)
+            # params['jm_names'] = get_jm_names(snode)
             params['ha_comm_addrs'] = ha_comm_addrs
             params['ha_inode_self'] = ha_inode_self
             params['distrib_cpu_mask'] = snode.distrib_cpu_mask

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/controllers/tasks_controller.py

@@ -54,20 +54,22 @@ def _add_task(function_name, cluster_id, node_id, device_id,
 
 
 def add_device_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 
 def add_device_to_auto_restart(device):
-    return _add_task(JobSchedule.FN_DEV_RESTART, device.cluster_id, device.node_id, device.get_id())
+    return True
+    # return _add_task(JobSchedule.FN_DEV_RESTART, device.cluster_id, device.node_id, device.get_id())
 
 
 def add_node_to_auto_restart(node):
-    return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")
+    return True
+    # return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")
 
 
 def list_tasks(cluster_id):
@@ -135,20 +137,20 @@ def get_active_node_mig_task(cluster_id, node_id):
 
 
 def add_device_failed_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_FAILED_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_FAILED_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 
 def add_new_device_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
-        for lvol_id in node.lvols:
-            _add_task(JobSchedule.FN_NEW_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
-                      max_retry=0, function_params={'lvol_id': lvol_id})
+    # device = db_controller.get_storage_devices(device_id)
+    # for node in db_controller.get_storage_nodes_by_cluster_id(device.cluster_id):
+    #     for lvol_id in node.lvols:
+    #         _add_task(JobSchedule.FN_NEW_DEV_MIG, device.cluster_id, node.get_id(), device.get_id(),
+    #                   max_retry=0, function_params={'lvol_id': lvol_id})
     return True
 
 

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/models/nvme_device.py

@@ -113,6 +113,7 @@ class JMDevice(BaseModel):
         "nvmf_nqn": {"type": str, 'default': ""},
         "nvmf_ip": {"type": str, 'default': ""},
         "nvmf_port": {"type": int, 'default': 4420},
+        "remote_bdev": {"type": str, 'default': ""},
 
         "health_check": {"type": bool, "default": True},
         "io_error": {"type": bool, 'default': False},

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/models/storage_node.py

@@ -108,6 +108,8 @@ class StorageNode(BaseModel):
         "num_partitions_per_dev": {"type": int, "default": 1},
         "jm_percent": {"type": int, "default": 3},
         "jm_device": {"type": JMDevice, "default": None},
+        "remote_jm_devices": {"type": List[JMDevice], 'default': []},
+        "enable_ha_jm": {"type": bool, "default": False},
 
         "namespace": {"type": str, "default": ""},
 

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/rpc_client.py

@@ -363,7 +363,7 @@ class RPCClient:
 
     def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names,
                             chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152,
-                            distrib_cpu_mask=""):
+                            distrib_cpu_mask="", ha_is_non_leader=True, jm_vuid=1):
         """"
             // Optional (not specified = no HA)
             // Comma-separated communication addresses, for each node, e.g. "192.168.10.1:45001,192.168.10.1:32768".
@@ -379,12 +379,14 @@ class RPCClient:
             "name": name,
             "jm_names": ",".join(jm_names),
             "vuid": vuid,
+            "jm_vuid": jm_vuid,
             "ndcs": ndcs,
             "npcs": npcs,
             "num_blocks": num_blocks,
             "block_size": block_size,
             "chunk_size": chunk_size,
-            "pba_page_size": pba_page_size
+            "pba_page_size": pba_page_size,
+            "ha_is_non_leader": ha_is_non_leader,
         }
         if ha_comm_addrs:
             params['ha_comm_addrs'] = ha_comm_addrs
@@ -470,6 +472,20 @@ class RPCClient:
         }
         return self._request("bdev_nvme_attach_controller", params)
 
+    def bdev_nvme_attach_controller_tcp_jm(self, name, nqn, ip, port):
+        params = {
+            "name": name,
+            "trtype": "tcp",
+            "traddr": ip,
+            "adrfam": "ipv4",
+            "trsvcid": str(port),
+            "subnqn": nqn,
+            "fast_io_fail_timeout_sec": 1,
+            "ctrlr_loss_timeout_sec": 1,
+            "reconnect_delay_sec": 1,
+        }
+        return self._request("bdev_nvme_attach_controller", params)
+
     def bdev_nvme_attach_controller_tcp_caching(self, name, nqn, ip, port):
         params = {
             "name": name,

{sbcli_dev-3.8.35 → sbcli_dev-3.8.37}/simplyblock_core/services/storage_node_monitor.py

@@ -5,7 +5,7 @@ import time
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, utils
 from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
 from simplyblock_core.models.cluster import Cluster
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 
@@ -113,6 +113,16 @@ def set_node_offline(node):
         tasks_controller.add_node_to_auto_restart(node)
 
 
+def set_jm_device_status(node):
+    if node.status != StorageNode.STATUS_UNREACHABLE:
+        for dev in node.nvme_devices:
+            device_controller.device_set_unavailable(dev.get_id())
+
+        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
+        # add node to auto restart
+        tasks_controller.add_node_to_auto_restart(node)
+
+
 logger.info("Starting node monitor")
 while True:
     clusters = db_controller.get_clusters()
@@ -158,6 +168,19 @@ while True:
             else:
                 set_node_offline(snode)
 
+            # check JM device
+            if snode.jm_device:
+                if snode.jm_device.status in [JMDevice.STATUS_ONLINE, JMDevice.STATUS_UNAVAILABLE]:
+                    ret = health_controller.check_jm_device(snode.jm_device.get_id())
+                    if ret:
+                        logger.error(f"JM bdev is online: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_ONLINE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
+                    else:
+                        logger.error(f"JM bdev is offline: {snode.jm_device.get_id()}")
+                        if snode.jm_device.status != JMDevice.STATUS_UNAVAILABLE:
+                            device_controller.set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_UNAVAILABLE)
+
         update_cluster_status(cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")