sbcli-dev 17.3.4__tar.gz → 17.3.6__tar.gz

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.3.4
+Version: 17.3.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.3.4
+Version: 17.3.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/sbcli_dev.egg-info/SOURCES.txt

@@ -50,6 +50,7 @@ simplyblock_core/models/caching_node.py
 simplyblock_core/models/cluster.py
 simplyblock_core/models/deployer.py
 simplyblock_core/models/events.py
+simplyblock_core/models/hublvol.py
 simplyblock_core/models/iface.py
 simplyblock_core/models/job_schedule.py
 simplyblock_core/models/lvol_model.py

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/controllers/lvol_controller.py

@@ -23,14 +23,6 @@ from simplyblock_core.snode_client import SNodeClient
 logger = lg.getLogger()
 
 
-def _generate_hex_string(length):
-    def _generate_string(length):
-        return ''.join(random.SystemRandom().choice(
-            string.ascii_letters + string.digits) for _ in range(length))
-
-    return _generate_string(length).encode('utf-8').hex()
-
-
 def _create_crypto_lvol(rpc_client, name, base_name, key1, key2):
     key_name = f'key_{name}'
     ret = rpc_client.lvol_crypto_key_create(key_name, key1, key2)
@@ -427,7 +419,7 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp,
     lvol.ha_type = ha_type
     lvol.bdev_stack = []
     lvol.uuid = uid or str(uuid.uuid4())
-    lvol.guid = _generate_hex_string(16)
+    lvol.guid = utils.generate_hex_string(16)
     lvol.vuid = vuid
     lvol.lvol_bdev = f"LVOL_{vuid}"
 

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/controllers/snapshot_controller.py

@@ -439,7 +439,7 @@ def clone(snapshot_id, clone_name, new_size=0):
     lvol.pool_uuid = pool.get_id()
     lvol.ha_type = snap.lvol.ha_type
     lvol.lvol_type = 'lvol'
-    lvol.guid = lvol_controller._generate_hex_string(16)
+    lvol.guid = utils.generate_hex_string(16)
     lvol.vuid = snap.lvol.vuid
     lvol.snapshot_name = snap.snap_bdev
     lvol.subsys_port = snap.lvol.subsys_port

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=17.3.4
+SIMPLY_BLOCK_VERSION=17.3.6
 
 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/models/base_model.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 import pprint
+import logging
 
 import json
 from inspect import ismethod
@@ -115,8 +116,10 @@ class BaseModel(object):
             for k, v in kv_store.get_range_startswith(prefix.strip().encode('utf-8'),  limit=limit, reverse=reverse):
                 objects.append(self.__class__().from_dict(json.loads(v)))
             return objects
-        except Exception as e:
-            print(f"Error reading from FDB: {e}")
+        except Exception:
+            from simplyblock_core import utils
+            logger = utils.get_logger(__name__)
+            logger.exception('Error reading from FDB')
             return []
 
     def get_last(self, kv_store):

sbcli_dev-17.3.6/simplyblock_core/models/hublvol.py

@@ -0,0 +1,12 @@
+# coding=utf-8
+
+from simplyblock_core.models.base_model import BaseModel
+
+
+class HubLVol(BaseModel):
+    """Identifying information of a HubLVol
+    """
+    uuid: str = ""
+    nqn: str = ""
+    name: str = ""
+    nvmf_port: int = 9099

sbcli_dev-17.3.6/simplyblock_core/models/storage_node.py

@@ -0,0 +1,211 @@
+# coding=utf-8
+
+from typing import List
+from uuid import uuid4
+
+from simplyblock_core import utils
+from simplyblock_core.models.base_model import BaseNodeObject
+from simplyblock_core.models.hublvol import HubLVol
+from simplyblock_core.models.iface import IFace
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
+from simplyblock_core.rpc_client import RPCClient, RPCException
+
+logger = utils.get_logger(__name__)
+
+
+class StorageNode(BaseNodeObject):
+
+    alceml_cpu_cores: List[int] = []
+    alceml_cpu_index: int = 0
+    alceml_worker_cpu_cores: List[int] = []
+    alceml_worker_cpu_index: int = 0
+    api_endpoint: str = ""
+    app_thread_mask: str = ""
+    baseboard_sn: str = ""
+    cloud_instance_id: str = ""
+    cloud_instance_public_ip: str = ""
+    cloud_instance_type: str = ""
+    cloud_name: str = ""
+    cluster_id: str = ""
+    cpu: int = 0
+    cpu_hz: int = 0
+    ctrl_secret: str = ""
+    data_nics: List[IFace] = []
+    distrib_cpu_cores: List[int] = []
+    distrib_cpu_index: int = 0
+    distrib_cpu_mask: str = ""
+    enable_ha_jm: bool = False
+    ha_jm_count: int = 3
+    enable_test_device: bool = False
+    health_check: bool = True
+    host_nqn: str = ""
+    host_secret: str = ""
+    hostname: str = ""
+    hugepages: int = 0
+    ib_devices: List[IFace] = []
+    id_device_by_nqn: bool = False
+    iobuf_large_bufsize: int = 0
+    iobuf_large_pool_count: int = 0
+    iobuf_small_bufsize: int = 0
+    iobuf_small_pool_count: int = 0
+    is_secondary_node: bool = False
+    jc_singleton_mask: str = ""
+    jm_cpu_mask: str = ""
+    jm_device: JMDevice = None
+    jm_percent: int = 3
+    jm_vuid: int = 0
+    lvols: int = 0
+    lvstore: str = ""
+    lvstore_stack: List[dict] = []
+    lvstore_stack_secondary_1: List[dict] = []
+    lvstore_stack_secondary_2: List[dict] = []
+    lvol_subsys_port: int = 9090
+    max_lvol: int = 0
+    max_prov: int = 0
+    max_snap: int = 0
+    memory: int = 0
+    mgmt_ip: str = ""
+    namespace: str = ""
+    node_lvs: str = "lvs"
+    num_partitions_per_dev: int = 1
+    number_of_devices: int = 0
+    number_of_distribs: int = 4
+    nvme_devices: List[NVMeDevice] = []
+    online_since: str = ""
+    partitions_count: int = 0  # Unused
+    poller_cpu_cores: List[int] = []
+    ssd_pcie: List = []
+    pollers_mask: str = ""
+    primary_ip: str = ""
+    raid: str = ""
+    remote_devices: List[NVMeDevice] = []
+    remote_jm_devices: List[JMDevice] = []
+    rpc_password: str = ""
+    rpc_port: int = -1
+    rpc_username: str = ""
+    secondary_node_id: str = ""
+    sequential_number: int = 0  # Unused
+    jm_ids: List[str] = []
+    spdk_cpu_mask: str = ""
+    spdk_debug: bool = False
+    spdk_image: str = ""
+    spdk_mem: int = 0
+    partition_size: int = 0
+    subsystem: str = ""
+    system_uuid: str = ""
+    lvstore_status: str = ""
+    nvmf_port: int = 4420
+    physical_label: int = 0
+    hublvol: HubLVol = None
+
+    def rpc_client(self, **kwargs):
+        """Return rpc client to this node
+        """
+        return RPCClient(
+            self.mgmt_ip, self.rpc_port,
+            self.rpc_username, self.rpc_password, **kwargs)
+
+    def expose_bdev(self, nqn, bdev_name, model_number, uuid, nguid, port):
+        rpc_client = self.rpc_client()
+
+        try:
+            if not rpc_client.subsystem_create(
+                    nqn=nqn,
+                    serial_number='sbcli-cn',
+                    model_number=model_number,
+            ):
+                raise RPCException(f'Failed to create subsystem for {nqn}')
+
+            for ip in (iface.ip4_address for iface in self.data_nics):
+                if not rpc_client.listeners_create(
+                        nqn=nqn,
+                        trtype='TCP',
+                        traddr=ip,
+                        trsvcid=port,
+                ):
+                    raise RPCException(f'Failed to create listener for {nqn}')
+
+            if not rpc_client.nvmf_subsystem_add_ns(
+                    nqn=nqn,
+                    dev_name=bdev_name,
+                    uuid=uuid,
+                    nguid=nguid,
+            ):
+                raise RPCException(f'Failed to add namespace to subsytem {nqn}')
+        except RPCException:
+            if self.hublvol and rpc_client.subsystem_list(self.hublvol.nqn):
+                rpc_client.subsystem_delete(self.hublvol.nqn)
+
+            raise
+
+    def create_hublvol(self, cluster_nqn):
+        """Create a hublvol for this node's lvstore
+        """
+        logger.info(f'Creating hublvol on {self.get_id()}')
+        rpc_client = self.rpc_client()
+
+        hublvol_uuid = None
+        try:
+            hublvol_uuid = rpc_client.bdev_lvol_create_hublvol(self.lvstore)
+            if not hublvol_uuid:
+                raise RPCException('Failed to create hublvol')
+            self.hublvol = HubLVol({
+                'uuid': hublvol_uuid,
+                'nqn': f'{cluster_nqn}:lvol:{hublvol_uuid}',
+                'name': f'{self.lvstore}/hublvol',
+            })
+
+            self.expose_bdev(
+                    nqn=self.hublvol.nqn,
+                    bdev_name=self.hublvol.name,
+                    model_number=str(uuid4()),
+                    uuid=self.hublvol.uuid,
+                    nguid=utils.generate_hex_string(16),
+                    port=self.hublvol.nvmf_port
+            )
+        except RPCException:
+            if hublvol_uuid is not None and rpc_client.get_bdevs(hublvol_uuid):
+                rpc_client.bdev_lvol_delete_hublvol(self.hublvol.nqn)
+
+            if self.hublvol and rpc_client.subsystem_list(self.hublvol.nqn):
+                rpc_client.subsystem_delete(self.hublvol.nqn)
+                self.hublvol = None
+
+            raise
+
+        self.write_to_db()
+        return self.hublvol
+
+    def connect_to_hublvol(self, primary_node):
+        """Connect to a primary node's hublvol
+        """
+        logger.info(f'Connecting node {self.get_id()} to hublvol on {primary_node.get_id()}')
+
+        if primary_node.hublvol is None:
+            raise ValueError(f"HubLVol of primary node {primary_node.get_id()} is not present")
+
+        rpc_client = self.rpc_client()
+
+        remote_bdev = None
+        for ip in (iface.ip4_address for iface in primary_node.data_nics):
+            remote_bdev = rpc_client.bdev_nvme_attach_controller_tcp(
+                    primary_node.hublvol.name, primary_node.hublvol.nqn,
+                    ip, primary_node.hublvol.nvmf_port)[0]
+            if remote_bdev is not None:
+                break
+            else:
+                logger.warning(f'Failed to connect to hublvol on {ip}')
+
+        if remote_bdev is None:
+            raise RPCException('Failed to connect to hublvol')
+
+        if not rpc_client.bdev_lvol_set_lvs_opts(
+                primary_node.lvstore,
+                groupid=primary_node.jm_vuid,
+                subsystem_port=self.lvol_subsys_port,
+                secondary=True,
+        ):
+            raise RPCException('Failed to set secondary lvstore options')
+
+        if not rpc_client.bdev_lvol_connect_hublvol(primary_node.lvstore, remote_bdev):
+            raise RPCException('Failed to connect secondary lvstore to primary')

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/rpc_client.py

@@ -1,4 +1,5 @@
 import json
+import inspect
 
 import requests
 
@@ -913,18 +914,12 @@ class RPCClient:
             params = {"jm_vuid": jm_vuid}
         return self._request("bdev_distrib_force_to_non_leader", params)
 
-    def bdev_lvol_set_leader(self, is_leader=False, uuid=None, lvs_name=None, bs_nonleadership=False):
-        params = {
-            "lvs_leadership": is_leader,
-        }
-        if uuid:
-            params["uuid"] = uuid
-        elif lvs_name:
-            params["lvs_name"] = lvs_name
-
-        params["bs_nonleadership"] = bs_nonleadership
-
-        return self._request("bdev_lvol_set_leader_all", params)
+    def bdev_lvol_set_leader(self, lvs, *, leader=False, bs_nonleadership=False):
+        return self._request("bdev_lvol_set_leader_all", {
+            "uuid" if utils.UUID_PATTERN.match(lvs) else "lvs_name": lvs,
+            "lvs_leadership": leader,
+            "bs_nonleadership": bs_nonleadership,
+        })
 
     def bdev_lvol_register(self, name, lvs_name, registered_uuid, blobid, priority_class=0):
         params = {
@@ -982,13 +977,19 @@ class RPCClient:
         }
         return self._request("nvmf_set_max_subsystems", params)
 
-    def bdev_lvol_set_lvs_ops(self, lvs_name, groupid, subsystem_port=9090):
-        params = {
+    def bdev_lvol_set_lvs_opts(self, lvs, *, groupid, subsystem_port=9090, primary=False, secondary=False):
+        """Set lvstore options
+
+        `lvs` must be either an ID or the lvstore name.
+        """
+
+        return self._request(inspect.currentframe().f_code.co_name, {
+            "uuid" if utils.UUID_PATTERN.match(lvs) else "lvs_name": lvs,
             "groupid": groupid,
-            "lvs_name": lvs_name,
             "subsystem_port": subsystem_port,
-        }
-        return self._request("bdev_lvol_set_lvs_op", params)
+            "primary": primary,
+            "secondary": secondary,
+        })
 
     def bdev_lvol_get_lvol_delete_status(self, name):
         """
@@ -1010,3 +1011,18 @@ class RPCClient:
         }
         return self._request("bdev_lvol_set_lvs_read_only", params)
 
+    def bdev_lvol_create_hublvol(self, lvs):
+        return self._request(inspect.currentframe().f_code.co_name, {
+            "uuid" if utils.UUID_PATTERN.match(lvs) else "lvs_name": lvs,
+        })
+
+    def bdev_lvol_delete_hublvol(self, lvs):
+        return self._request(inspect.currentframe().f_code.co_name, {
+            "uuid" if utils.UUID_PATTERN.match(lvs) else "lvs_name": lvs,
+        })
+
+    def bdev_lvol_connect_hublvol(self, lvs, bdev):
+        return self._request(inspect.currentframe().f_code.co_name, {
+            "uuid" if utils.UUID_PATTERN.match(lvs) else "lvs_name": lvs,
+            "remote_bdev": bdev,
+        })

{sbcli_dev-17.3.4 → sbcli_dev-17.3.6}/simplyblock_core/storage_node_ops.py

@@ -25,7 +25,7 @@ from simplyblock_core.models.lvol_model import LVol
 from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.models.cluster import Cluster
-from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core.rpc_client import RPCClient, RPCException
 from simplyblock_core.snode_client import SNodeClient
 
 logger = utils.get_logger(__name__)
@@ -2204,7 +2204,7 @@ def suspend_storage_node(node_id, force=False):
                             ret = rpc_client.nvmf_subsystem_listener_set_ana_state(
                                 lvol.nqn, iface.ip4_address, lvol.subsys_port, False, ana="inaccessible")
 
-                rpc_client.bdev_lvol_set_leader(False, lvs_name=node.lvstore)
+                rpc_client.bdev_lvol_set_leader(node.lvstore, leader=False)
                 rpc_client.bdev_distrib_force_to_non_leader(node.jm_vuid)
 
 
@@ -2219,7 +2219,7 @@ def suspend_storage_node(node_id, force=False):
                     ret = sec_node_client.nvmf_subsystem_listener_set_ana_state(
                         lvol.nqn, iface.ip4_address, lvol.subsys_port, False, ana="inaccessible")
         time.sleep(1)
-        # sec_node_client.bdev_lvol_set_leader(False, lvs_name=snode.lvstore)
+        # sec_node_client.bdev_lvol_set_leader(snode.lvstore, leader=False)
         # sec_node_client.bdev_distrib_force_to_non_leader(snode.jm_vuid)
 
     for lvol in db_controller.get_lvols_by_node_id(snode.get_id()):
@@ -2229,7 +2229,7 @@ def suspend_storage_node(node_id, force=False):
                     lvol.nqn, iface.ip4_address, lvol.subsys_port, False, ana="inaccessible")
     time.sleep(1)
 
-    rpc_client.bdev_lvol_set_leader(False, lvs_name=snode.lvstore)
+    rpc_client.bdev_lvol_set_leader(snode.lvstore, leader=False)
     rpc_client.bdev_distrib_force_to_non_leader(snode.jm_vuid)
     time.sleep(1)
 
@@ -2317,7 +2317,7 @@ def resume_storage_node(node_id):
                         ret = sec_node_client.nvmf_subsystem_listener_set_ana_state(
                             lvol.nqn, iface.ip4_address, lvol.subsys_port, False, ana="inaccessible")
             time.sleep(1)
-            sec_node_client.bdev_lvol_set_leader(False, lvs_name=snode.lvstore)
+            sec_node_client.bdev_lvol_set_leader(snode.lvstore, leader=False)
             sec_node_client.bdev_distrib_force_to_non_leader(snode.jm_vuid)
             time.sleep(1)
 
@@ -2803,71 +2803,76 @@ def set_node_status(node_id, status, reconnect_on_online=True):
     return True
 
 
-def recreate_lvstore_on_sec(snode):
+def recreate_lvstore_on_sec(secondary_node):
     db_controller = DBController()
-    rpc_client = RPCClient(
-        snode.mgmt_ip, snode.rpc_port,
-        snode.rpc_username, snode.rpc_password)
+    secondary_rpc_client = RPCClient(
+        secondary_node.mgmt_ip, secondary_node.rpc_port,
+        secondary_node.rpc_username, secondary_node.rpc_password)
 
-    nodes = db_controller.get_primary_storage_nodes_by_secondary_node_id(snode.get_id())
+    primary_nodes = db_controller.get_primary_storage_nodes_by_secondary_node_id(secondary_node.get_id())
 
-    for node in nodes:
-        remote_rpc_client = RPCClient(
-            node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
+    for primary_node in primary_nodes:
+        primary_rpc_client = RPCClient(
+            primary_node.mgmt_ip, primary_node.rpc_port, primary_node.rpc_username, primary_node.rpc_password)
 
-        node.lvstore_status = "in_creation"
-        node.write_to_db()
+        primary_node.lvstore_status = "in_creation"
+        primary_node.write_to_db()
 
         lvol_list = []
-        for lv in db_controller.get_lvols_by_node_id(node.get_id()):
+        for lv in db_controller.get_lvols_by_node_id(primary_node.get_id()):
             if lv.status not in [LVol.STATUS_IN_DELETION, LVol.STATUS_IN_CREATION]:
                 lvol_list.append(lv)
 
         ### 1- create distribs and raid
-        ret, err = _create_bdev_stack(snode, node.lvstore_stack, primary_node=node)
+        ret, err = _create_bdev_stack(secondary_node, primary_node.lvstore_stack, primary_node=primary_node)
         if err:
-            logger.error(f"Failed to recreate lvstore on node {snode.get_id()}")
+            logger.error(f"Failed to recreate lvstore on node {secondary_node.get_id()}")
             logger.error(err)
             return False
 
-        node_api = SNodeClient(node.api_endpoint)
+        primary_node_api = SNodeClient(primary_node.api_endpoint)
 
         ### 2- create lvols nvmf subsystems
         for lvol in lvol_list:
             logger.info("creating subsystem %s", lvol.nqn)
-            rpc_client.subsystem_create(lvol.nqn, 'sbcli-cn', lvol.uuid, 1000)
+            secondary_rpc_client.subsystem_create(lvol.nqn, 'sbcli-cn', lvol.uuid, 1000)
 
-        if node.status == StorageNode.STATUS_ONLINE:
+        if primary_node.status == StorageNode.STATUS_ONLINE:
 
             ### 3- block primary port
-            node_api.firewall_set_port(node.lvol_subsys_port, "tcp", "block")
-            tcp_ports_events.port_deny(node, node.lvol_subsys_port)
+            primary_node_api.firewall_set_port(primary_node.lvol_subsys_port, "tcp", "block")
+            tcp_ports_events.port_deny(primary_node, primary_node.lvol_subsys_port)
 
             ### 4- set leadership to false
-            remote_rpc_client.bdev_lvol_set_leader(False, lvs_name=node.lvstore)
-            remote_rpc_client.bdev_distrib_force_to_non_leader(node.jm_vuid)
+            primary_rpc_client.bdev_lvol_set_leader(primary_node.lvstore, leader=False)
+            primary_rpc_client.bdev_distrib_force_to_non_leader(primary_node.jm_vuid)
             # time.sleep(1)
 
 
         ### 5- examine
-        ret = rpc_client.bdev_examine(node.raid)
+        ret = secondary_rpc_client.bdev_examine(primary_node.raid)
 
         ### 6- wait for examine
-        ret = rpc_client.bdev_wait_for_examine()
-        ret = rpc_client.bdev_lvol_set_lvs_ops(node.lvstore, node.jm_vuid, node.lvol_subsys_port)
+        ret = secondary_rpc_client.bdev_wait_for_examine()
+        try:
+            secondary_node.connect_to_hublvol(primary_node)
+
+        except RPCException as e:
+            logger.error("Error connecting to hublvol: %s", e.message)
+            return False
 
         ### 8- allow port on primary
-        node_api.firewall_set_port(node.lvol_subsys_port, "tcp", "allow")
-        tcp_ports_events.port_allowed(node, node.lvol_subsys_port)
+        primary_node_api.firewall_set_port(primary_node.lvol_subsys_port, "tcp", "allow")
+        tcp_ports_events.port_allowed(primary_node, primary_node.lvol_subsys_port)
 
         ### 7- add lvols to subsystems
         executor = ThreadPoolExecutor(max_workers=100)
         for lvol in lvol_list:
-            a = executor.submit(add_lvol_thread, lvol, snode,  lvol_ana_state="non_optimized")
+            a = executor.submit(add_lvol_thread, lvol, secondary_node,  lvol_ana_state="non_optimized")
 
-        node = db_controller.get_storage_node_by_id(node.get_id())
-        node.lvstore_status = "ready"
-        node.write_to_db()
+        primary_node = db_controller.get_storage_node_by_id(primary_node.get_id())
+        primary_node.lvstore_status = "ready"
+        primary_node.write_to_db()
 
     return True
 
@@ -2936,7 +2941,7 @@ def recreate_lvstore(snode):
 
             # time.sleep(1)
             ### 4- set leadership to false
-            sec_rpc_client.bdev_lvol_set_leader(False, lvs_name=snode.lvstore, bs_nonleadership=True)
+            sec_rpc_client.bdev_lvol_set_leader(snode.lvstore, leader=False, bs_nonleadership=True)
             sec_rpc_client.bdev_distrib_force_to_non_leader(snode.jm_vuid)
             # time.sleep(1)
 
@@ -2946,10 +2951,33 @@ def recreate_lvstore(snode):
 
     ### 6- wait for examine
     ret = rpc_client.bdev_wait_for_examine()
-    ret = rpc_client.bdev_lvol_set_lvs_ops(snode.lvstore, snode.jm_vuid, snode.lvol_subsys_port)
-    ret = rpc_client.bdev_lvol_set_leader(True, lvs_name=snode.lvstore)
+    ret = rpc_client.bdev_lvol_set_lvs_opts(
+            snode.lvstore,
+            groupid=snode.jm_vuid,
+            subsystem_port=snode.lvol_subsys_port,
+            primary=True
+    )
+    ret = rpc_client.bdev_lvol_set_leader(snode.lvstore, leader=True)
+
+    if sec_node:
+        ### 7- create and connect hublvol
+        cluster_nqn = db_controller.get_cluster_by_id(snode.cluster_id).nqn
+        try:
+            snode.create_hublvol(cluster_nqn)
+            if sec_node.status == StorageNode.STATUS_ONLINE:
+                sec_node.connect_to_hublvol(snode)
+
+        except RPCException as e:
+            logger.error("Error establishing hublvol: %s", e.message)
+            return False
 
-    ### 7- add lvols to subsystems
+        finally:
+            ### 8- allow secondary port
+            if sec_node.status == StorageNode.STATUS_ONLINE:
+                sec_node_api.firewall_set_port(snode.lvol_subsys_port, "tcp", "allow", sec_node.rpc_port)
+                tcp_ports_events.port_allowed(sec_node, snode.lvol_subsys_port)
+
+    ### 9- add lvols to subsystems
     executor = ThreadPoolExecutor(max_workers=100)
     for lvol in lvol_list:
         a = executor.submit(add_lvol_thread, lvol, snode, lvol_ana_state)
@@ -2959,11 +2987,8 @@ def recreate_lvstore(snode):
         logger.info("Node status changed to suspended")
         return False
 
-    ### 8- allow secondary port
+    ### 10- finish
     if sec_node.status == StorageNode.STATUS_ONLINE:
-        time.sleep(7)
-        sec_node_api.firewall_set_port(snode.lvol_subsys_port, "tcp", "allow", sec_node.rpc_port)
-        tcp_ports_events.port_allowed(sec_node, snode.lvol_subsys_port)
         sec_node = db_controller.get_storage_node_by_id(snode.secondary_node_id)
         sec_node.lvstore_status = "ready"
         sec_node.write_to_db()
@@ -3019,6 +3044,8 @@ def get_sorted_ha_jms(current_node):
         if (node.get_id() == current_node.get_id() or node.status != StorageNode.STATUS_ONLINE  or
                 node.is_secondary_node):  # pass
             continue
+        if node.mgmt_ip == current_node.mgmt_ip:
+            continue
         if node.jm_device and node.jm_device.status == JMDevice.STATUS_ONLINE:
             jm_count[node.jm_device.get_id()] = 1 + jm_count.get(node.jm_device.get_id(), 0)
         for rem_jm_device in node.remote_jm_devices:
@@ -3185,26 +3212,42 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
         logger.error(err)
         return False
 
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password )
+    ret = rpc_client.bdev_lvol_set_lvs_opts(
+            snode.lvstore,
+            groupid=snode.jm_vuid,
+            subsystem_port=snode.lvol_subsys_port,
+            primary=True
+    )
+    ret = rpc_client.bdev_lvol_set_leader(snode.lvstore, leader=True)
+
     if snode.secondary_node_id:
+        sec_node = db_controller.get_storage_node_by_id(snode.secondary_node_id)
+
         # creating lvstore on secondary
-        sec_node_1 = db_controller.get_storage_node_by_id(snode.secondary_node_id)
-        sec_node_1.remote_jm_devices = _connect_to_remote_jm_devs(sec_node_1)
-        sec_node_1.write_to_db()
-        ret, err = _create_bdev_stack(sec_node_1, lvstore_stack, primary_node=snode)
+        sec_node.remote_jm_devices = _connect_to_remote_jm_devs(sec_node)
+        sec_node.write_to_db()
+        ret, err = _create_bdev_stack(sec_node, lvstore_stack, primary_node=snode)
         if err:
-            logger.error(f"Failed to create lvstore on node {sec_node_1.get_id()}")
+            logger.error(f"Failed to create lvstore on node {sec_node.get_id()}")
             logger.error(err)
             return False
 
-        temp_rpc_client = RPCClient(
-                sec_node_1.mgmt_ip, sec_node_1.rpc_port,
-                sec_node_1.rpc_username, sec_node_1.rpc_password)
+        sec_rpc_client = sec_node.rpc_client()
+        sec_rpc_client.bdev_examine(snode.raid)
+        sec_rpc_client.bdev_wait_for_examine()
+
+        cluster_nqn = db_controller.get_cluster_by_id(snode.cluster_id).nqn
+        try:
+            snode.create_hublvol(cluster_nqn)
+            if sec_node.status == StorageNode.STATUS_ONLINE:
+                sec_node.connect_to_hublvol(snode)
 
-        ret = temp_rpc_client.bdev_examine(snode.raid)
-        ret = temp_rpc_client.bdev_wait_for_examine()
-        ret = temp_rpc_client.bdev_lvol_set_lvs_ops(snode.lvstore, snode.jm_vuid, snode.lvol_subsys_port)
+        except RPCException as e:
+            logger.error("Error establishing hublvol: %s", e.message)
+            return False
 
-        sec_node_1.write_to_db()
+        sec_node.write_to_db()
 
     return True