sbcli-dev 4.0.15__zip → 4.0.17__zip

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.15
+Version: 4.0.17
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.15
+SIMPLY_BLOCK_VERSION=4.0.17
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.15
+Version: 4.0.17
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/simplyblock_core/cluster_ops.py

@@ -26,12 +26,6 @@ TOP_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
 def _create_update_user(cluster_id, grafana_url, grafana_secret, user_secret, update_secret=False):
-    if not grafana_url.startswith("https://"):
-        if grafana_url.startswith("http://"):
-            grafana_url = grafana_url.replace("http://", "https://", 1)
-        else:
-            grafana_url = "https://" + grafana_url
-
     session = requests.session()
     session.auth = ("admin", grafana_secret)
     headers = {
@@ -348,9 +342,10 @@ def cluster_activate(cl_id, force=False):
     if not cluster:
         logger.error(f"Cluster not found {cl_id}")
         return False
-    if cluster.status != Cluster.STATUS_UNREADY:
-        logger.error(f"Failed to activate cluster, Cluster is not in an UNREADY state")
+    if cluster.status == Cluster.STATUS_ACTIVE:
+        logger.warning("Cluster is not in an ACTIVE state")
         if not force:
+            logger.warning(f"Failed to activate cluster, Cluster is in an ACTIVE state, use --force to reactivate")
             return False
     set_cluster_status(cl_id, Cluster.STATUS_IN_ACTIVATION)
     snodes = db_controller.get_storage_nodes_by_cluster_id(cl_id)
@@ -373,17 +368,17 @@ def cluster_activate(cl_id, force=False):
 
     for snode in snodes:
         if snode.lvstore:
-            logger.info(f"Node {snode.get_id()} already has lvstore {snode.lvstore}... skipping")
-            if not force:
-                continue
-        ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
+            logger.warning(f"Node {snode.get_id()} already has lvstore {snode.lvstore}")
+            ret = storage_node_ops.recreate_lvstore(snode)
+        else:
+            ret = storage_node_ops.create_lvstore(snode, cluster.distr_ndcs, cluster.distr_npcs, cluster.distr_bs,
                                               cluster.distr_chunk_bs, cluster.page_size_in_blocks, max_size, snodes)
         if not ret:
             logger.error("Failed to activate cluster")
             set_cluster_status(cl_id, Cluster.STATUS_UNREADY)
             return False
-
-    cluster.cluster_max_size = max_size
+    if not cluster.cluster_max_size:
+        cluster.cluster_max_size = max_size
     cluster.write_to_db(db_controller.kv_store)
     set_cluster_status(cl_id, Cluster.STATUS_ACTIVE)
     logger.info("Cluster activated successfully")

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/simplyblock_core/rpc_client.py

@@ -377,6 +377,12 @@ class RPCClient:
             //  This node (device) number, in the group, defined by ha_comm_addrs.
           "ha_inode_self": 1
         """
+        try:
+            ret = self.get_bdevs(name)
+            if ret:
+                return ret
+        except:
+            pass
         params = {
             "name": name,
             "jm_names": ",".join(jm_names),
@@ -417,6 +423,12 @@ class RPCClient:
         return self._request("bdev_get_iostat", params)
 
     def bdev_raid_create(self, name, bdevs_list, raid_level="0", strip_size_kb=4):
+        try:
+            ret = self.get_bdevs(name)
+            if ret:
+                return ret
+        except:
+            pass
         params = {
             "name": name,
             "raid_level": raid_level,
@@ -472,7 +484,9 @@ class RPCClient:
             "fabrics_connect_timeout_us": 100000,
             "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
-            "ctrlr_loss_timeout_sec": 2,
+            "ctrlr_loss_timeout_sec": 1,
+            "multipath":"disable",
+            "reconnect_delay_sec":1
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -485,8 +499,10 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fast_io_fail_timeout_sec": 1,
-            "ctrlr_loss_timeout_sec": 1,
+            "ctrlr_loss_timeout_sec": 3,
             "reconnect_delay_sec": 1,
+            "fabrics_connect_timeout_us": 100000,
+            "multipath":"disable"
         }
         return self._request("bdev_nvme_attach_controller", params)
 

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/simplyblock_core/services/health_check_service.py

@@ -82,7 +82,6 @@ while True:
 
             health_check_status = is_node_online
             if not node_rpc_check:
-                logger.info("Putting all devices to unavailable state because RPC check failed")
                 for dev in snode.nvme_devices:
                     if dev.io_error:
                         logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
@@ -107,15 +106,47 @@ while True:
                     snode.mgmt_ip, snode.rpc_port,
                     snode.rpc_username, snode.rpc_password,
                     timeout=10, retry=1)
+                connected_devices = []
                 for remote_device in snode.remote_devices:
                     if db_controller.get_storage_device_by_id(remote_device.get_id()).status == NVMeDevice.STATUS_ONLINE:
                         ret = rpc_client.get_bdevs(remote_device.remote_bdev)
                         if ret:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                            connected_devices.append(remote_device.get_id())
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip, remote_device.nvmf_port)
+                            if ret:
+                                logger.info(f"Successfully connected to device: {remote_device.get_id()}")
+                                connected_devices.append(remote_device.get_id())
+                                distr_controller.send_dev_status_event(remote_device, NVMeDevice.STATUS_ONLINE, snode)
+                            else:
+                                logger.error(f"Failed to connect to device: {remote_device.get_id()}")
+
                         node_remote_devices_check &= bool(ret)
 
+                for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
+                    if node.status != StorageNode.STATUS_ONLINE:
+                        continue
+                    for dev in node.nvme_devices:
+                        if dev.status == StorageNode.STATUS_ONLINE:
+                            if dev.get_id() not in connected_devices:
+                                logger.info(f"connecting to online device: {dev.get_id()}")
+                                name = f"remote_{dev.alceml_bdev}"
+                                bdev_name = f"{name}n1"
+                                ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                    name, dev.nvmf_nqn, dev.nvmf_ip,
+                                    dev.nvmf_port)
+                                if ret:
+                                    logger.info(f"Successfully connected to device: {dev.get_id()}")
+                                    dev.remote_bdev = bdev_name
+                                    snode.remote_devices.append(dev)
+                                    snode.write_to_db()
+                                    distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE, snode)
+                                else:
+                                    logger.error(f"Failed to connect to device: {dev.get_id()}")
+
                 if snode.jm_device:
                     jm_device = snode.jm_device
                     logger.info(f"Node JM: {jm_device}")
@@ -124,6 +155,7 @@ while True:
                         logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
                     else:
                         logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... not found")
+                        # todo: try add
                     node_devices_check &= ret
 
                 if snode.enable_ha_jm:
@@ -134,6 +166,14 @@ while True:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                                remote_device.remote_bdev, remote_device.nvmf_nqn, remote_device.nvmf_ip,
+                                remote_device.nvmf_port)
+                            if ret:
+                                logger.info(f"Successfully connected to device: {remote_device.get_id()}")
+                            else:
+                                logger.error(f"Failed to connect to device: {remote_device.get_id()}")
+
                         # node_remote_devices_check &= bool(ret)
 
                 lvstore_check = True

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/simplyblock_core/services/main_distr_event_collector.py

@@ -50,9 +50,9 @@ def process_device_event(event):
         if device.node_id != node_id:
             logger.info(f"Setting storage id: {storage_id} unavailable")
             distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_UNAVAILABLE, node)
-            if device.status == NVMeDevice.STATUS_ONLINE:
-                device_controller.device_set_io_error(device_id, True)
-                device_controller.device_set_unavailable(device_id)
+            # if device.status == NVMeDevice.STATUS_ONLINE:
+            #     device_controller.device_set_io_error(device_id, True)
+            #     device_controller.device_set_unavailable(device_id)
 
         else:
             if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]:

{sbcli_dev-4.0.15 → sbcli_dev-4.0.17}/simplyblock_core/storage_node_ops.py

@@ -829,7 +829,10 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
             logger.debug(f"bdev found {bdev_name}")
             jm_dev.status = JMDevice.STATUS_ONLINE
         else:
-            logger.info(f"Connecting {this_node.get_id()} to {name}")
+
+            logger.info(f"Connecting {name} to {this_node.get_id()}")
+            ret = rpc_client.bdev_nvme_detach_controller(name)
+            time.sleep(1)
             ret = rpc_client.bdev_nvme_attach_controller_tcp(
                 name, jm_dev.nvmf_nqn, jm_dev.nvmf_ip, jm_dev.nvmf_port)
             if ret:
@@ -1242,7 +1245,8 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         logger.info(f"connected to devices count: {len(node.remote_devices)}")
         time.sleep(3)
 
-    if cluster.status == cluster.STATUS_UNREADY:
+    if cluster.status != cluster.STATUS_ACTIVE:
+        logger.warning(f"The cluster status is not active ({cluster.status}), adding the node without distribs and lvstore")
         logger.info("Done")
         return "Success"
 
@@ -1704,8 +1708,11 @@ def restart_storage_node(
 
     if snode.enable_ha_jm:
         logger.info("Connecting to remote JMs")
+        snode.remote_jm_devices = []
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
 
+    snode.write_to_db(db_controller.kv_store)
+
     logger.info("Setting node status to Online")
     set_node_status(node_id, StorageNode.STATUS_ONLINE)
 
@@ -1731,7 +1738,8 @@ def restart_storage_node(
             continue
         tasks_controller.add_device_mig_task(dev.get_id())
 
-    if cluster.status == cluster.STATUS_UNREADY:
+    if cluster.status != cluster.STATUS_ACTIVE:
+        logger.warning(f"The cluster status is not active ({cluster.status}), adding the node without distribs and lvstore")
         logger.info("Done")
         return "Success"
     # Create distribs, raid0, and lvstore and expose lvols to the fabrics
@@ -1741,23 +1749,7 @@ def restart_storage_node(
                 snode.rpc_username, snode.rpc_password)
         ret = recreate_lvstore(snode)
         if not ret:
-            return False, "Failed to create distribs on node"
-        time.sleep(1)
-        ret = temp_rpc_client.bdev_examine(snode.raid)
-        time.sleep(1)
-        ret = temp_rpc_client.bdev_wait_for_examine()
-        time.sleep(1)
-
-        if snode.lvols:
-            for lvol_id in snode.lvols:
-                lvol = lvol_controller.recreate_lvol(lvol_id, snode)
-                if not lvol:
-                    logger.error(f"Failed to create LVol: {lvol_id}")
-                    return False
-                lvol.status = lvol.STATUS_ONLINE
-                lvol.io_error = False
-                lvol.health_check = True
-                lvol.write_to_db(db_controller.kv_store)
+            return False, "Failed to recreate lvstore on node"
 
     logger.info("Done")
     return "Success"
@@ -2577,11 +2569,34 @@ def set_node_status(node_id, status):
 
 
 def recreate_lvstore(snode):
+
     ret, err = _create_bdev_stack(snode)
     if err:
         logger.error(f"Failed to recreate lvstore on node {snode.get_id()}")
         logger.error(err)
         return False
+
+    temp_rpc_client = RPCClient(
+            snode.mgmt_ip, snode.rpc_port,
+            snode.rpc_username, snode.rpc_password)
+    time.sleep(1)
+    ret = temp_rpc_client.bdev_examine(snode.raid)
+    time.sleep(1)
+    ret = temp_rpc_client.bdev_wait_for_examine()
+    time.sleep(1)
+
+    if snode.lvols:
+        db_controller = DBController()
+        for lvol_id in snode.lvols:
+            lvol = lvol_controller.recreate_lvol(lvol_id, snode)
+            if not lvol:
+                logger.error(f"Failed to create LVol: {lvol_id}")
+                return False
+            lvol.status = lvol.STATUS_ONLINE
+            lvol.io_error = False
+            lvol.health_check = True
+            lvol.write_to_db(db_controller.kv_store)
+
     return True