sbcli-pre 1.2.6__zip → 1.2.7__zip

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.2.6
+Version: 1.2.7
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=1.2.6
+SIMPLY_BLOCK_VERSION=1.2.7
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
 

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.2.6
+Version: 1.2.7
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_cli/cli.py

@@ -43,12 +43,14 @@ class CLIWrapper:
         sub_command.add_argument("--jm-percent", help='Number in percent to use for JM from each device',
                                  type=int, default=3, dest='jm_percent')
         sub_command.add_argument("--data-nics", help='Data interface names', nargs='+', dest='data_nics')
-        sub_command.add_argument("--memory", help='SPDK huge memory allocation, default is 4G', dest='spdk_mem')
+        sub_command.add_argument("--max-lvol", help='Max lvol per storage node', dest='max_lvol', type=int)
+        sub_command.add_argument("--max-snap", help='Max snapshot per storage node', dest='max_snap', type=int)
+        sub_command.add_argument("--max-prov", help='Max provisioning size of all storage nodes', dest='max_prov')
+        sub_command.add_argument("--number-of-devices", help='Number of devices per storage node if it\'s not supported EC2 instance', dest='number_of_devices', type=int)
+
         sub_command.add_argument("--spdk-image", help='SPDK image uri', dest='spdk_image')
         sub_command.add_argument("--spdk-debug", help='Enable spdk debug logs', dest='spdk_debug', required=False, action='store_true')
 
-        sub_command.add_argument("--iobuf_small_pool_count", help='bdev_set_options param', dest='small_pool_count',  type=int, default=0)
-        sub_command.add_argument("--iobuf_large_pool_count", help='bdev_set_options param', dest='large_pool_count',  type=int, default=0)
         sub_command.add_argument("--iobuf_small_bufsize", help='bdev_set_options param', dest='small_bufsize',  type=int, default=0)
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
 
@@ -77,12 +79,14 @@ class CLIWrapper:
                                   'During restart, the node does not accept IO. In a high-availability setup, '
                                   'this will not impact operations')
         sub_command.add_argument("node_id", help='UUID of storage node')
-        sub_command.add_argument("--memory", help='SPDK huge memory allocation, default is 4G', dest='spdk_mem')
+        sub_command.add_argument("--max-lvol", help='Max lvol per storage node', dest='max_lvol', type=int, default=0)
+        sub_command.add_argument("--max-snap", help='Max snapshot per storage node', dest='max_snap', type=int, default=0)
+        sub_command.add_argument("--max-prov", help='Max provisioning size of all storage nodes', dest='max_prov', default="")
+        sub_command.add_argument("--number-of-devices", help='Number of devices per storage node if it\'s not supported EC2 instance', dest='number_of_devices', type=int)
+
         sub_command.add_argument("--spdk-image", help='SPDK image uri', dest='spdk_image')
         sub_command.add_argument("--spdk-debug", help='Enable spdk debug logs', dest='spdk_debug', required=False, action='store_true')
 
-        sub_command.add_argument("--iobuf_small_pool_count", help='bdev_set_options param', dest='small_pool_count',  type=int, default=0)
-        sub_command.add_argument("--iobuf_large_pool_count", help='bdev_set_options param', dest='large_pool_count',  type=int, default=0)
         sub_command.add_argument("--iobuf_small_bufsize", help='bdev_set_options param', dest='small_bufsize',  type=int, default=0)
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
 
@@ -636,6 +640,12 @@ class CLIWrapper:
                 ret = storage_ops.deploy_cleaner()
 
             elif sub_command == "add-node":
+                if not args.max_lvol:
+                    self.parser.error(f"Mandatory argument '--max-lvol' not provided for {sub_command}")
+                if not args.max_snap:
+                    self.parser.error(f"Mandatory argument '--max-snap' not provided for {sub_command}")
+                if not args.max_prov:
+                    self.parser.error(f"Mandatory argument '--max-prov' not provided for {sub_command}")
                 cluster_id = args.cluster_id
                 node_ip = args.node_ip
                 ifname = args.ifname
@@ -643,22 +653,21 @@ class CLIWrapper:
                 spdk_image = args.spdk_image
                 spdk_debug = args.spdk_debug
 
-                small_pool_count = args.small_pool_count
-                large_pool_count = args.large_pool_count
                 small_bufsize = args.small_bufsize
                 large_bufsize = args.large_bufsize
                 num_partitions_per_dev = args.partitions
                 jm_percent = args.jm_percent
 
-                spdk_mem = None
-                if args.spdk_mem:
-                    spdk_mem = self.parse_size(args.spdk_mem)
-                    if spdk_mem < 1 * 1024 * 1024:
-                        return f"SPDK memory:{args.spdk_mem} must be larger than 1G"
+                max_lvol = args.max_lvol
+                max_snap = args.max_snap
+                max_prov = self.parse_size(args.max_prov)
+                number_of_devices = args.number_of_devices
+                if max_prov < 1 * 1024 * 1024:
+                    return f"Max provisioning memory:{args.max_prov} must be larger than 1G"
 
                 out = storage_ops.add_node(
-                    cluster_id, node_ip, ifname, data_nics, spdk_mem, spdk_image, spdk_debug,
-                    small_pool_count, large_pool_count, small_bufsize, large_bufsize, num_partitions_per_dev, jm_percent)
+                    cluster_id, node_ip, ifname, data_nics, max_lvol, max_snap, max_prov, spdk_image, spdk_debug,
+                    small_bufsize, large_bufsize, num_partitions_per_dev, jm_percent, number_of_devices)
                 return out
 
             elif sub_command == "list":
@@ -676,22 +685,17 @@ class CLIWrapper:
                 spdk_image = args.spdk_image
                 spdk_debug = args.spdk_debug
 
-                spdk_mem = None
-                if args.spdk_mem:
-                    spdk_mem = self.parse_size(args.spdk_mem)
-                    if spdk_mem < 1 * 1024 * 1024:
-                        return f"SPDK memory:{args.spdk_mem} must be larger than 1G"
 
+                max_lvol = args.max_lvol
+                max_snap = args.max_snap
+                max_prov = self.parse_size(args.max_prov)
 
-                small_pool_count = args.small_pool_count
-                large_pool_count = args.large_pool_count
                 small_bufsize = args.small_bufsize
                 large_bufsize = args.large_bufsize
 
                 ret = storage_ops.restart_storage_node(
-                    node_id, spdk_mem,
+                    node_id, max_lvol, max_snap, max_prov,
                     spdk_image, spdk_debug,
-                    small_pool_count, large_pool_count,
                     small_bufsize, large_bufsize)
 
             elif sub_command == "list-devices":

sbcli_pre-1.2.7/simplyblock_core/constants.py

@@ -0,0 +1,91 @@
+import logging
+import os
+
+KVD_DB_VERSION = 730
+KVD_DB_FILE_PATH = '/etc/foundationdb/fdb.cluster'
+KVD_DB_TIMEOUT_MS = 10000
+SPK_DIR = '/home/ec2-user/spdk'
+RPC_HTTP_PROXY_PORT = 8080
+LOG_LEVEL = logging.INFO
+LOG_WEB_DEBUG = True
+
+INSTALL_DIR = os.path.dirname(os.path.realpath(__file__))
+
+NODE_MONITOR_INTERVAL_SEC = 3
+DEVICE_MONITOR_INTERVAL_SEC = 5
+STAT_COLLECTOR_INTERVAL_SEC = 60*5  # 5 minutes
+LVOL_STAT_COLLECTOR_INTERVAL_SEC = 2
+LVOL_MONITOR_INTERVAL_SEC = 60
+DEV_MONITOR_INTERVAL_SEC = 10
+DEV_STAT_COLLECTOR_INTERVAL_SEC = 2
+PROT_STAT_COLLECTOR_INTERVAL_SEC = 2
+DISTR_EVENT_COLLECTOR_INTERVAL_SEC = 2
+DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS = 10
+CAP_MONITOR_INTERVAL_SEC = 30
+SSD_VENDOR_WHITE_LIST = ["1d0f:cd01", "1d0f:cd00"]
+
+PMEM_DIR = '/tmp/pmem'
+
+NVME_PROGRAM_FAIL_COUNT = 50
+NVME_ERASE_FAIL_COUNT = 50
+NVME_CRC_ERROR_COUNT = 50
+DEVICE_OVERLOAD_STDEV_VALUE = 50
+DEVICE_OVERLOAD_CAPACITY_THRESHOLD = 50
+
+CLUSTER_NQN = "nqn.2023-02.io.simplyblock"
+
+weights = {
+    "lvol": 50,
+    "cpu": 10,
+    "r_io": 10,
+    "w_io": 10,
+    "r_b": 10,
+    "w_b": 10
+}
+
+# To use 75% of hugepages to calculate ssd size to use for the ocf bdev
+CACHING_NODE_MEMORY_FACTOR = 0.75
+
+HEALTH_CHECK_INTERVAL_SEC = 60
+
+GRAYLOG_CHECK_INTERVAL_SEC = 60
+
+FDB_CHECK_INTERVAL_SEC = 60
+
+SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:pre-release"
+SIMPLY_BLOCK_CLI_NAME = "sbcli"
+TASK_EXEC_INTERVAL_SEC = 30
+TASK_EXEC_RETRY_COUNT = 8
+
+SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:latest"
+SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:prerelease-latest"
+
+GELF_PORT = 12201
+
+MIN_HUGE_PAGE_MEMORY_FOR_LVOL = 209715200
+MIN_SYS_MEMORY_FOR_LVOL = 524288000
+EXTRA_SMALL_POOL_COUNT = 1024
+EXTRA_LARGE_POOL_COUNT = 128
+EXTRA_HUGE_PAGE_MEMORY = 2147483648
+EXTRA_SYS_MEMORY = 2147483648
+
+INSTANCE_STORAGE_DATA = {
+        'i4i.large': {'number_of_devices': 1, 'size_per_device_gb': 468},
+        'i4i.xlarge': {'number_of_devices': 1, 'size_per_device_gb': 937},
+        'i4i.2xlarge': {'number_of_devices': 1, 'size_per_device_gb': 1875},
+        'i4i.4xlarge': {'number_of_devices': 1, 'size_per_device_gb': 3750},
+        'i4i.8xlarge': {'number_of_devices': 2, 'size_per_device_gb': 3750},
+        'i4i.12xlarge': {'number_of_devices': 3, 'size_per_device_gb': 3750},
+        'i4i.16xlarge': {'number_of_devices': 4, 'size_per_device_gb': 3750},
+        'i4i.24xlarge': {'number_of_devices': 6, 'size_per_device_gb': 3750},
+        'i4i.32xlarge': {'number_of_devices': 8, 'size_per_device_gb': 3750},
+        'i4i.metal': {'number_of_devices': 8, 'size_per_device_gb': 3750},
+        'i3en.large': {'number_of_devices': 1, 'size_per_device_gb': 1250},
+        'i3en.xlarge': {'number_of_devices': 1, 'size_per_device_gb': 2500},
+        'i3en.2xlarge': {'number_of_devices': 2, 'size_per_device_gb': 2500},
+        'i3en.3xlarge': {'number_of_devices': 1, 'size_per_device_gb': 7500},
+        'i3en.6xlarge': {'number_of_devices': 2, 'size_per_device_gb': 7500},
+        'i3en.12xlarge': {'number_of_devices': 4, 'size_per_device_gb': 7500},
+        'i3en.24xlarge': {'number_of_devices': 8, 'size_per_device_gb': 7500},
+        'i3en.metal': {'number_of_devices': 8, 'size_per_device_gb': 7500},
+    }

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/controllers/lvol_controller.py

@@ -17,7 +17,7 @@ from simplyblock_core.models.pool import Pool
 from simplyblock_core.models.lvol_model import LVol
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
-
+from simplyblock_core.snode_client import SNodeClient
 
 logger = lg.getLogger()
 db_controller = DBController()
@@ -364,12 +364,23 @@ def add_lvol(name, size, host_id_or_name, pool_id_or_name, use_comp, use_crypto,
     return lvol_id, None
 
 
-def _get_next_3_nodes(cluster_id):
+def _get_next_3_nodes(cluster_id, lvol_size=0):
     snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     online_nodes = []
     node_stats = {}
     for node in snodes:
         if node.status == node.STATUS_ONLINE:
+            # Validate Eligible nodes for adding lvol
+            snode_api = SNodeClient(node.api_endpoint)
+            result, _ = snode_api.info()
+            memory_free = result["memory_details"]["free"]
+            huge_free = result["memory_details"]["huge_free"]
+            total_node_capacity = db_controller.get_snode_size(node.get_id())
+            error = utils.validate_add_lvol_or_snap_on_node(memory_free, huge_free, node.max_lvol, lvol_size,  total_node_capacity, len(node.lvols))
+            if error:
+                logger.warning(error)
+                continue
+
             online_nodes.append(node)
             node_stat_list = db_controller.get_node_stats(node, limit=1000)
             combined_record = utils.sum_records(node_stat_list)
@@ -695,7 +706,7 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp,
         lvol.lvol_type += ',compress'
         lvol.top_bdev = lvol.comp_bdev
 
-    nodes = _get_next_3_nodes(cl.get_id())
+    nodes = _get_next_3_nodes(cl.get_id(), lvol.size)
     if not nodes:
         return False, f"No nodes found with enough resources to create the LVol"
 
@@ -800,6 +811,18 @@ def add_lvol_on_node(lvol, snode, ha_comm_addrs=None, ha_inode_self=None):
     rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
     spdk_mem_info_before = rpc_client.ultra21_util_get_malloc_stats()
 
+    # Validate adding lvol on storage node
+    snode_api = SNodeClient(snode.api_endpoint)
+    result, _ = snode_api.info()
+    memory_free = result["memory_details"]["free"]
+    huge_free = result["memory_details"]["huge_free"]
+
+    total_node_capacity = db_controller.get_snode_size(snode.get_id())
+    error = utils.validate_add_lvol_or_snap_on_node(memory_free, huge_free, snode.max_lvol, lvol.size,  total_node_capacity, len(snode.lvols))
+    if error:
+        logger.error(error)
+        return False, f"Failed to add lvol on node {snode.get_id()}"
+
     ret, msg = _create_bdev_stack(lvol, snode, ha_comm_addrs, ha_inode_self)
     if not ret:
         return False, msg

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/controllers/snapshot_controller.py

@@ -13,6 +13,8 @@ from simplyblock_core.models.pool import Pool
 from simplyblock_core.models.snapshot import SnapShot
 from simplyblock_core.models.lvol_model import LVol
 from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core.snode_client import SNodeClient
+
 
 logger = lg.getLogger()
 
@@ -33,6 +35,21 @@ def add(lvol_id, snapshot_name):
     logger.info(f"Creating snapshot: {snapshot_name} from LVol: {lvol.id}")
     snode = db_controller.get_storage_node_by_id(lvol.node_id)
 
+##############################################################################
+    # Validate adding snap on storage node
+    snode_api = SNodeClient(snode.api_endpoint)
+    result, _ = snode_api.info()
+    memory_free = result["memory_details"]["free"]
+    huge_free = result["memory_details"]["huge_free"]
+    total_node_capacity = db_controller.get_snode_size(snode.get_id())
+
+    error = utils.validate_add_lvol_or_snap_on_node(memory_free, huge_free, snode.max_snap, lvol.size, total_node_capacity,
+                                                    len(db_controller.get_snapshots_by_node_id(snode.get_id())))
+
+    if error:
+        logger.error(f"Failed to add snap on node {snode.get_id()}")
+        logger.error(error)
+        return False
 
 ##############################################################################
     snap_count = 0
@@ -199,6 +216,16 @@ def clone(snapshot_id, clone_name, new_size=0):
                 logger.error(msg)
                 return False, msg
 
+    # Validate cloning snap on storage node
+    snode_api = SNodeClient(snode.api_endpoint)
+    result, _ = snode_api.info()
+    memory_free = result["memory_details"]["free"]
+    huge_free = result["memory_details"]["huge_free"]
+    total_node_capacity = db_controller.get_snode_size(snode.get_id())
+    error = utils.validate_add_lvol_or_snap_on_node(memory_free, huge_free, snode.max_lvol, snap.lvol.size,  total_node_capacity, len(snode.lvols))
+    if error:
+        logger.error(error)
+        return False, f"Failed to add lvol on node {snode.get_id()}"
     lvol = LVol()
     lvol.lvol_name = clone_name
     lvol.size = snap.lvol.size

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/kv_store.py

@@ -288,3 +288,18 @@ class DBController:
         for task in self.get_job_tasks(""):
             if task.uuid == task_id:
                 return task
+
+    def get_snapshots_by_node_id(self, node_id):
+        ret = []
+        snaps = SnapShot().read_from_db(self.kv_store)
+        for snap in snaps:
+            if snap.lvol.host_id == node_id:
+                ret.append(snap)
+        return ret
+
+    def get_snode_size(self, node_id):
+        snode = self.get_storage_node_by_id(node_id)
+        total_node_capacity = 0
+        for dev in snode.nvme_devices:
+            total_node_capacity += dev.size
+        return total_node_capacity

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/models/storage_node.py

@@ -61,7 +61,9 @@ class StorageNode(BaseModel):
         "remote_devices": {"type": List[NVMeDevice], 'default': []},
         "host_secret": {"type": str, "default": ""},
         "ctrl_secret": {"type": str, "default": ""},
-
+        "max_lvol": {"type": int, "default": 0},
+        "max_snap": {"type": int, "default": 0},
+        "max_prov": {"type": str, "default": ""},
         "cpu": {"type": int, "default": 0},
         "cpu_hz": {"type": int, "default": 0},
         "memory": {"type": int, "default": 0},
@@ -73,11 +75,13 @@ class StorageNode(BaseModel):
         "app_thread_mask": {"type": str, "default": ""},
         "pollers_mask": {"type": str, "default": ""},
         "os_cores": {"type": str, "default": []},
+        "nvme_pollers_cores": {"type": str, "default": ""},
         "dev_cpu_mask": {"type": str, "default": ""},
         "spdk_mem": {"type": int, "default": 0},
         "spdk_image": {"type": str, "default": ""},
         "spdk_debug": {"type": bool, "default": False},
 
+
         "ec2_metadata": {"type": dict, "default": {}},
         "ec2_instance_id": {"type": str, "default": ""},
         "ec2_public_ip": {"type": str, "default": ""},

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/services/tasks_runner_restart.py

@@ -209,7 +209,7 @@ while True:
                 if task.function_name in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART]:
                     while task.status != JobSchedule.STATUS_DONE:
                         # get new task object because it could be changed from cancel task
-                        task = db_controller.get_task_by_id(task.get_id())
+                        task = db_controller.get_task_by_id(task.uuid)
                         res = task_runner(task)
                         if res:
                             tasks_events.task_updated(task)

{sbcli_pre-1.2.6 → sbcli_pre-1.2.7}/simplyblock_core/storage_node_ops.py

@@ -550,9 +550,9 @@ def _connect_to_remote_devs(this_node):
 
 
 def add_node(cluster_id, node_ip, iface_name, data_nics_list,
-             spdk_mem, spdk_image=None, spdk_debug=False,
-             small_pool_count=0, large_pool_count=0, small_bufsize=0, large_bufsize=0,
-             num_partitions_per_dev=0, jm_percent=0):
+             max_lvol, max_snap, max_prov, spdk_image=None, spdk_debug=False,
+             small_bufsize=0, large_bufsize=0,
+             num_partitions_per_dev=0, jm_percent=0, number_of_devices=0):
     db_controller = DBController()
     kv_store = db_controller.kv_store
 
@@ -608,26 +608,12 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
                 logger.error(f"Node already exists, try remove it first: {ec2_metadata['instanceId']}")
                 return False
 
-    # check for memory
-    if "memory_details" in node_info and node_info['memory_details']:
-        memory_details = node_info['memory_details']
-        logger.info("Node Memory info")
-        logger.info(f"Total: {utils.humanbytes(memory_details['total'])}")
-        logger.info(f"Free: {utils.humanbytes(memory_details['free'])}")
-        logger.info(f"Hugepages Total: {utils.humanbytes(memory_details['huge_total'])}")
-        huge_free = memory_details['huge_free']
-        logger.info(f"Hugepages Free: {utils.humanbytes(huge_free)}")
-        if huge_free < 1 * 1024 * 1024:
-            logger.warning(f"Free hugepages are less than 1G: {utils.humanbytes(huge_free)}")
-        if not spdk_mem:
-            spdk_mem = huge_free
-            logger.info(f"Using the free hugepages for spdk memory: {utils.humanbytes(huge_free)}")
-
     # Tune cpu maks parameters
     cpu_count = node_info["cpu_count"]
     pollers_mask = ""
     app_thread_mask = ""
     dev_cpu_mask = ""
+    nvme_pollers_cores = []
     if cpu_count < 8:
         mask = (1 << (cpu_count - 1)) - 1
         mask <<= 1
@@ -643,6 +629,50 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         spdk_cpu_mask = utils.generate_mask(spdk_cores)
         dev_cpu_mask = utils.generate_mask(dev_cpu_cores)
 
+    # Calculate pool count
+    if ec2_metadata and ec2_metadata.get('instanceType'):
+        supported_type, storage_devices, device_size = utils.get_total_size_per_instance_type(ec2_metadata["instanceType"])
+        if not supported_type:
+            logger.warning(f"Unsupported ec2 instance-type {ec2_metadata['instanceType']} for deployment")
+            if not number_of_devices:
+                logger.error(f"Unsupported ec2 instance-type {ec2_metadata['instanceType']} "
+                             "for deployment, please specify --number-of-devices")
+                return False
+        number_of_devices = storage_devices
+    else:
+        logger.warning("Can not get ec2 instance type for this instance.")
+        if not number_of_devices:
+            logger.error("Unsupported instance type please specify --number-of-devices.")
+            return False
+
+    number_of_split = num_partitions_per_dev if num_partitions_per_dev else num_partitions_per_dev + 1
+    number_of_alceml_devices = number_of_devices * number_of_split
+    small_pool_count, large_pool_count = utils.calculate_pool_count(
+        number_of_alceml_devices, max_lvol, max_snap, cpu_count, len(nvme_pollers_cores) or cpu_count)
+
+    # Calculate minimum huge page memory
+    minimum_hp_memory = utils.calculate_minimum_hp_memory(small_pool_count, large_pool_count, max_lvol, max_snap, cpu_count)
+
+    # Calculate minimum sys memory
+    minimum_sys_memory = utils.calculate_minimum_sys_memory(max_prov)
+
+    # check for memory
+    if "memory_details" in node_info and node_info['memory_details']:
+        memory_details = node_info['memory_details']
+        logger.info("Node Memory info")
+        logger.info(f"Total: {utils.humanbytes(memory_details['total'])}")
+        logger.info(f"Free: {utils.humanbytes(memory_details['free'])}")
+    else:
+        logger.error(f"Cannot get memory info from the ec2 instance.. Exiting")
+        return False
+
+    satisfied, spdk_mem = utils.calculate_spdk_memory(minimum_hp_memory,
+                                                      minimum_sys_memory,
+                                                      int(memory_details['free']),
+                                                      int(memory_details['huge_total']))
+    if not satisfied:
+        logger.error(f"Not enough memory for the provided max_lvo: {max_lvol}, max_snap: {max_snap}, max_prov: {max_prov}.. Exiting")
+        return False
 
     logger.info("Joining docker swarm...")
     cluster_docker = utils.get_docker_client(cluster_id)
@@ -717,12 +747,17 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
         snode.hugepages = node_info['hugepages']
 
     snode.spdk_cpu_mask = spdk_cpu_mask or ""
-    snode.spdk_mem = spdk_mem or 0
+    snode.spdk_mem = spdk_mem
+    snode.max_lvol = max_lvol
+    snode.max_snap = max_snap
+    snode.max_prov = max_prov
+    snode.number_of_devices = number_of_devices
     snode.spdk_image = spdk_image or ""
     snode.spdk_debug = spdk_debug or 0
     snode.write_to_db(kv_store)
     snode.app_thread_mask = app_thread_mask or ""
     snode.pollers_mask = pollers_mask or ""
+    snode.nvme_pollers_cores = nvme_pollers_cores or []
     snode.dev_cpu_mask = dev_cpu_mask or ""
     snode.os_cores = os_cores or []
 
@@ -975,12 +1010,10 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
 
 
 def restart_storage_node(
-        node_id,
-        spdk_mem=None,
+        node_id, max_lvol=0, max_snap=0, max_prov="",
         spdk_image=None,
         set_spdk_debug=None,
-        small_pool_count=0, large_pool_count=0,
-        small_bufsize=0, large_bufsize=0):
+        small_bufsize=0, large_bufsize=0, number_of_devices=0):
 
     db_controller = DBController()
     kv_store = db_controller.kv_store
@@ -1011,14 +1044,67 @@ def restart_storage_node(
     logger.info(f"Node info: {node_info}")
 
     logger.info("Restarting SPDK")
-    mem = snode.spdk_mem
-    if spdk_mem:
-        mem = spdk_mem
-        snode.spdk_mem = mem
+
     img = snode.spdk_image
+    if max_lvol:
+        snode.max_lvol = max_lvol
+    if max_snap:
+        snode.max_snap = max_snap
+    if max_prov:
+        snode.max_prov = max_prov
     if spdk_image:
         img = spdk_image
         snode.spdk_image = img
+
+    # Calculate pool count
+    if snode.ec2_metadata and snode.ec2_metadata.get('instanceType'):
+        supported_type, storage_devices, device_size = utils.get_total_size_per_instance_type(snode.ec2_metadata["instanceType"])
+        if not supported_type:
+            logger.warning(f"Unsupported ec2 instance-type {snode.ec2_metadata['instanceType']} for deployment")
+            if not number_of_devices:
+                logger.error(f"Unsupported ec2 instance-type {snode.ec2_metadata['instanceType']} "
+                             "for deployment, please specify --number-of-devices")
+                return False
+        number_of_devices = storage_devices
+    else:
+        logger.warning("Can not get ec2 instance type for this instance..")
+        if not number_of_devices:
+            if snode.number_of_devices:
+                number_of_devices = snode.number_of_devices
+            else:
+                logger.error("Unsupported instance type please specify --number-of-devices")
+                return False
+
+    snode.number_of_devices = number_of_devices
+
+    number_of_split = snode.num_partitions_per_dev if snode.num_partitions_per_dev else snode.num_partitions_per_dev + 1
+    number_of_alceml_devices = number_of_devices * number_of_split
+    small_pool_count, large_pool_count = utils.calculate_pool_count(
+        number_of_alceml_devices, snode.max_lvol, snode.max_snap, snode.cpu, len(snode.nvme_pollers_cores) or snode.cpu)
+
+    # Calculate minimum huge page memory
+    minimum_hp_memory = utils.calculate_minimum_hp_memory(small_pool_count, large_pool_count, snode.max_lvol, snode.max_snap, snode.cpu)
+
+    # Calculate minimum sys memory
+    minimum_sys_memory = utils.calculate_minimum_sys_memory(snode.max_prov)
+
+    # check for memory
+    if "memory_details" in node_info and node_info['memory_details']:
+        memory_details = node_info['memory_details']
+        logger.info("Node Memory info")
+        logger.info(f"Total: {utils.humanbytes(memory_details['total'])}")
+        logger.info(f"Free: {utils.humanbytes(memory_details['free'])}")
+    else:
+        logger.error(f"Cannot get memory info from the ec2 instance.. Exiting")
+
+    satisfied, spdk_mem = utils.calculate_spdk_memory(minimum_hp_memory,
+                                                      minimum_sys_memory,
+                                                      int(memory_details['free']),
+                                                      int(memory_details['huge_total']))
+    if not satisfied:
+        logger.error(f"Not enough memory for the provided max_lvo: {snode.max_lvol}, max_snap: {snode.max_snap}, max_prov: {utils.humanbytes(snode.max_prov)}.. Exiting")
+
+
     spdk_debug = snode.spdk_debug
     if set_spdk_debug:
         spdk_debug = spdk_debug
@@ -1026,17 +1112,14 @@ def restart_storage_node(
 
     cluster_docker = utils.get_docker_client(snode.cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-    results, err = snode_api.spdk_process_start(snode.spdk_cpu_mask, mem, img, spdk_debug, cluster_ip)
+    results, err = snode_api.spdk_process_start(snode.spdk_cpu_mask, spdk_mem, img, spdk_debug, cluster_ip)
 
     if not results:
         logger.error(f"Failed to start spdk: {err}")
         return False
     time.sleep(3)
 
-    if small_pool_count:
-        snode.iobuf_small_pool_count = small_pool_count
-    if large_pool_count:
-        snode.iobuf_large_pool_count = large_pool_count
+
     if small_bufsize:
         snode.iobuf_small_bufsize = small_bufsize
     if large_bufsize: