sbcli-pre 1.1.0__zip → 1.1.2__zip

{sbcli_pre-1.1.0 → sbcli_pre-1.1.2}/simplyblock_core/storage_node_ops.py

@@ -277,7 +277,8 @@ def _create_storage_device_stack(rpc_client, nvme, snode, after_restart):
     pba_init_mode = 3
     if after_restart:
         pba_init_mode = 2
-    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode)
+    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=pba_init_mode,
+                                            dev_cpu_mask=snode.dev_cpu_mask)
     if not ret:
         logger.error(f"Failed to create alceml bdev: {alceml_name}")
         return False
@@ -479,7 +480,7 @@ def _connect_to_remote_devs(this_node):
     return remote_devices
 
 
-def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
+def add_node(cluster_id, node_ip, iface_name, data_nics_list,
              spdk_mem, spdk_image=None, spdk_debug=False,
              small_pool_count=0, large_pool_count=0, small_bufsize=0, large_bufsize=0,
              num_partitions_per_dev=0, jm_percent=0):
@@ -553,6 +554,27 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
             spdk_mem = huge_free
             logger.info(f"Using the free hugepages for spdk memory: {utils.humanbytes(huge_free)}")
 
+    # Tune cpu maks parameters
+    cpu_count = node_info["cpu_count"]
+    pollers_mask = ""
+    app_thread_mask = ""
+    dev_cpu_mask = ""
+    if cpu_count < 8:
+        mask = (1 << (cpu_count - 1)) - 1
+        mask <<= 1
+        spdk_cpu_mask = f'0x{mask:X}'
+        os_cores = [0]
+    else:
+        os_cores, nvme_pollers_cores, app_thread_core, dev_cpu_cores = \
+            utils.calculate_core_allocation(cpu_count)
+        spdk_cores = nvme_pollers_cores + app_thread_core + dev_cpu_cores
+
+        pollers_mask = utils.generate_mask(nvme_pollers_cores)
+        app_thread_mask = utils.generate_mask(app_thread_core)
+        spdk_cpu_mask = utils.generate_mask(spdk_cores)
+        dev_cpu_mask = utils.generate_mask(dev_cpu_cores)
+
+
     logger.info("Joining docker swarm...")
     cluster_docker = utils.get_docker_client(cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
@@ -630,6 +652,10 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
     snode.spdk_image = spdk_image or ""
     snode.spdk_debug = spdk_debug or 0
     snode.write_to_db(kv_store)
+    snode.app_thread_mask = app_thread_mask or ""
+    snode.pollers_mask = pollers_mask or ""
+    snode.dev_cpu_mask = dev_cpu_mask or ""
+    snode.os_cores = os_cores or []
 
     snode.iobuf_small_pool_count = small_pool_count or 0
     snode.iobuf_large_pool_count = large_pool_count or 0
@@ -656,13 +682,41 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask,
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- start spdk framework
+    # 2- set socket implementation options
+    ret = rpc_client.sock_impl_set_options()
+    if not ret:
+        logger.error("Failed socket implement set options")
+        return False
+
+    # 3- set nvme config
+    if snode.pollers_mask:
+        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
+        if not ret:
+            logger.error("Failed to set pollers mask")
+            return False
+
+    # 4- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 3- set nvme bdev options
+    # 5- set app_thread cpu mask
+    if snode.app_thread_mask:
+        ret = rpc_client.thread_get_stats()
+        app_thread_process_id = 0
+        if ret.get("threads"):
+            for entry in ret["threads"]:
+                if entry['name'] == 'app_thread':
+                    app_thread_process_id = entry['id']
+                    break
+
+        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
+        if not ret:
+            logger.error("Failed to set app thread mask")
+            return False
+
+    # 6- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")
@@ -1020,10 +1074,6 @@ def restart_storage_node(
     logger.info(f"Node info: {node_info}")
 
     logger.info("Restarting SPDK")
-    cpu = snode.spdk_cpu_mask
-    if spdk_cpu_mask:
-        cpu = spdk_cpu_mask
-        snode.spdk_cpu_mask = cpu
     mem = snode.spdk_mem
     if spdk_mem:
         mem = spdk_mem
@@ -1039,7 +1089,7 @@ def restart_storage_node(
 
     cluster_docker = utils.get_docker_client(snode.cluster_id)
     cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-    results, err = snode_api.spdk_process_start(cpu, mem, img, spdk_debug, cluster_ip)
+    results, err = snode_api.spdk_process_start(snode.spdk_cpu_mask, mem, img, spdk_debug, cluster_ip)
 
     if not results:
         logger.error(f"Failed to start spdk: {err}")
@@ -1073,13 +1123,41 @@ def restart_storage_node(
             logger.error("Failed to set iobuf options")
             return False
 
-    # 2- start spdk framework
+    # 2- set socket implementation options
+    ret = rpc_client.sock_impl_set_options()
+    if not ret:
+        logger.error("Failed socket implement set options")
+        return False
+
+    # 3- set nvme config
+    if snode.pollers_mask:
+        ret = rpc_client.nvmf_set_config(snode.pollers_mask)
+        if not ret:
+            logger.error("Failed to set pollers mask")
+            return False
+
+    # 4- start spdk framework
     ret = rpc_client.framework_start_init()
     if not ret:
         logger.error("Failed to start framework")
         return False
 
-    # 3- set nvme bdev options
+    # 5- set app_thread cpu mask
+    if snode.app_thread_mask:
+        ret = rpc_client.thread_get_stats()
+        app_thread_process_id = 0
+        if ret.get("threads"):
+            for entry in ret["threads"]:
+                if entry['name'] == 'app_thread':
+                    app_thread_process_id = entry['id']
+                    break
+
+        ret = rpc_client.thread_set_cpumask(app_thread_process_id, snode.app_thread_mask)
+        if not ret:
+            logger.error("Failed to set app thread mask")
+            return False
+
+    # 6- set nvme bdev options
     ret = rpc_client.bdev_nvme_set_options()
     if not ret:
         logger.error("Failed to set nvme options")

{sbcli_pre-1.1.0 → sbcli_pre-1.1.2}/simplyblock_core/utils.py

@@ -280,3 +280,48 @@ def sum_records(records):
 
 def get_random_vuid():
     return 1 + int(random.random() * 10000)
+
+
+def calculate_core_allocation(cpu_count):
+    '''
+    If number of cpu cores >= 8, tune cpu core mask
+        1. Never use core 0 for spdk.
+        2. For every 8 cores, leave one core to the operating system
+        3. Do not use more than 15% of remaining available cores for nvme pollers
+        4. Use one dedicated core for app_thread
+        5. distribute distrib bdevs and alceml bdevs to all other cores
+    JIRA ticket link/s
+    https://simplyblock.atlassian.net/browse/SFAM-885
+    '''
+
+    all_cores = list(range(0, cpu_count))
+    # Calculate the number of cores to exclude for the OS
+    if cpu_count == 8:
+        os_cores_count = 1
+    else:
+        os_cores_count = 1 + (cpu_count // 8)
+
+    # Calculate os cores
+    os_cores = all_cores[0:os_cores_count]
+
+    # Calculate available cores
+    available_cores_count = cpu_count - os_cores_count
+
+    # Calculate NVMe pollers
+    nvme_pollers_count = int(available_cores_count * 0.15)
+    nvme_pollers_cores = all_cores[os_cores_count:os_cores_count + nvme_pollers_count]
+
+    # Allocate core for app_thread
+    app_thread_core = all_cores[os_cores_count + nvme_pollers_count:os_cores_count + nvme_pollers_count + 1]
+
+    # Calculate bdb_lcpu cores
+    bdb_lcpu_cores = all_cores[os_cores_count + nvme_pollers_count + 1:]
+
+    return os_cores, nvme_pollers_cores, app_thread_core, bdb_lcpu_cores
+
+
+def generate_mask(cores):
+    mask = 0
+    for core in cores:
+        mask |= (1 << core)
+    return f'0x{mask:X}'

{sbcli_pre-1.1.0 → sbcli_pre-1.1.2}/simplyblock_web/blueprints/snode_ops.py

@@ -81,23 +81,10 @@ def spdk_process_start():
     spdk_cpu_mask = None
     if 'spdk_cpu_mask' in data:
         spdk_cpu_mask = data['spdk_cpu_mask']
+
     spdk_mem = None
     if 'spdk_mem' in data:
         spdk_mem = data['spdk_mem']
-    node_cpu_count = os.cpu_count()
-
-    if spdk_cpu_mask:
-        try:
-            requested_cpu_count = len(format(int(spdk_cpu_mask, 16), 'b'))
-            if requested_cpu_count > node_cpu_count:
-                return utils.get_response(
-                    False,
-                    f"The requested cpu count: {requested_cpu_count} "
-                    f"is larger than the node's cpu count: {node_cpu_count}")
-        except:
-            spdk_cpu_mask = hex(int(math.pow(2, node_cpu_count)) - 1)
-    else:
-        spdk_cpu_mask = hex(int(math.pow(2, node_cpu_count)) - 1)
 
     if spdk_mem:
         spdk_mem = int(utils.parse_size(spdk_mem) / (1000 * 1000))

{sbcli_pre-1.1.0 → sbcli_pre-1.1.2}/simplyblock_web/blueprints/web_api_storage_node.py

@@ -180,14 +180,6 @@ def storage_node_add():
         data_nics = req_data['data_nics']
         data_nics = data_nics.split(",")
 
-    spdk_cpu_mask = None
-    if 'spdk_cpu_mask' in req_data:
-        msk = req_data['spdk_cpu_mask']
-        if utils.validate_cpu_mask(msk):
-            spdk_cpu_mask = msk
-        else:
-            return utils.get_response_error(f"Invalid cpu mask value: {msk}", 400)
-
     spdk_mem = None
     if 'spdk_mem' in req_data:
         mem = req_data['spdk_mem']
@@ -196,7 +188,7 @@ def storage_node_add():
             return utils.get_response_error(f"SPDK memory:{mem} must be larger than 1G", 400)
 
     out = storage_node_ops.add_node(
-        cluster_id, node_ip, ifname, data_nics, spdk_cpu_mask, spdk_mem,
+        cluster_id, node_ip, ifname, data_nics, spdk_mem,
         spdk_image=spdk_image, spdk_debug=spdk_debug)
 
     return utils.get_response(out)