sbcli-pre 25.6.2__tar.gz → 25.6.4__tar.gz

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 25.6.2
+Version: 25.6.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 25.6.2
+Version: 25.6.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/cluster_ops.py

@@ -3,8 +3,6 @@ import datetime
 import json
 import os
 import socket
-import re
-import tempfile
 import subprocess
 import time
 import uuid
@@ -13,7 +11,6 @@ import typing as t
 
 import docker
 import requests
-from jinja2 import Environment, FileSystemLoader
 
 from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops
 from simplyblock_core.controllers import cluster_events, device_controller, pool_controller, \
@@ -30,7 +27,6 @@ from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.utils import pull_docker_image_with_retry
 
 logger = utils.get_logger(__name__)
-TOP_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 db_controller = DBController()
 
@@ -216,56 +212,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     c.inflight_io_threshold = inflight_io_threshold
     c.enable_qos = enable_qos
     c.strict_node_anti_affinity = strict_node_anti_affinity
+    c.contact_point = contact_point
 
-    alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")
-    alert_resources_file = "alert_resources.yaml"
-
-    env = Environment(loader=FileSystemLoader(alerts_template_folder), trim_blocks=True, lstrip_blocks=True)
-    template = env.get_template(f'{alert_resources_file}.j2')
-
-    slack_pattern = re.compile(r"https://hooks\.slack\.com/services/\S+")
-    email_pattern = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
-
-    if slack_pattern.match(contact_point):
-        ALERT_TYPE = "slack"
-    elif email_pattern.match(contact_point):
-        ALERT_TYPE = "email"
-    else:
-        ALERT_TYPE = "slack"
-        contact_point = 'https://hooks.slack.com/services/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'
-
-    values = {
-        'CONTACT_POINT': contact_point,
-        'GRAFANA_ENDPOINT': c.grafana_endpoint,
-        'ALERT_TYPE': ALERT_TYPE,
-    }
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_file_path = os.path.join(temp_dir, alert_resources_file)
-        with open(temp_file_path, 'w') as file:
-            file.write(template.render(values))
-
-        destination_file_path = os.path.join(alerts_template_folder, alert_resources_file)
-        subprocess.check_call(['sudo', '-v'])  # sudo -v checks if the current user has sudo permissions
-        subprocess.check_call(['sudo', 'mv', temp_file_path, destination_file_path])
-        print(f"File moved to {destination_file_path} successfully.")
-
-    scripts_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/")
-    prometheus_file = "prometheus.yml"
-    env = Environment(loader=FileSystemLoader(scripts_folder), trim_blocks=True, lstrip_blocks=True)
-    template = env.get_template(f'{prometheus_file}.j2')
-    values = {
-        'CLUSTER_ID': c.uuid,
-        'CLUSTER_SECRET': c.secret}
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        file_path = os.path.join(temp_dir, prometheus_file)
-        with open(file_path, 'w') as file:
-            file.write(template.render(values))
-
-        prometheus_file_path = os.path.join(scripts_folder, prometheus_file)
-        subprocess.check_call(['sudo', 'mv', file_path, prometheus_file_path])
-        print(f"File moved to {prometheus_file_path} successfully.")
+    utils.render_and_deploy_alerting_configs(contact_point, c.grafana_endpoint, c.uuid, c.secret)
 
     logger.info("Deploying swarm stack ...")
     log_level = "DEBUG" if constants.LOG_WEB_DEBUG else "INFO"
@@ -1194,11 +1143,11 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None,
     service_image = constants.SIMPLY_BLOCK_DOCKER_IMAGE
     if mgmt_image:
         service_image = mgmt_image
-        for service in cluster_docker.services.list():
-            if image_parts in service.attrs['Spec']['Labels']['com.docker.stack.image'] or \
-            "simplyblock" in service.attrs['Spec']['Labels']['com.docker.stack.image']:
-                logger.info(f"Updating service {service.name}")
-                service.update(image=service_image, force_update=True)
+    for service in cluster_docker.services.list():
+        if image_parts in service.attrs['Spec']['Labels']['com.docker.stack.image'] or \
+        "simplyblock" in service.attrs['Spec']['Labels']['com.docker.stack.image']:
+            logger.info(f"Updating service {service.name}")
+            service.update(image=service_image, force_update=True)
     logger.info("Done updating mgmt cluster")
 
     if mgmt_only:

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/constants.py

@@ -125,7 +125,7 @@ MAX_SNAP_COUNT = 100
 SPDK_PROXY_MULTI_THREADING_ENABLED=True
 SPDK_PROXY_TIMEOUT=60*5
 LVOL_NVME_CONNECT_RECONNECT_DELAY=2
-LVOL_NVME_CONNECT_CTRL_LOSS_TMO=60
+LVOL_NVME_CONNECT_CTRL_LOSS_TMO=60*60
 LVOL_NVME_CONNECT_NR_IO_QUEUES=6
 LVOL_NVME_KEEP_ALIVE_TO=5
 LVOL_NVMF_PORT_START=9100

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/controllers/device_controller.py

@@ -903,13 +903,15 @@ def restart_jm_device(device_id, force=False, format_alceml=False):
 
             if not ret:
                 logger.error(f"Failed to create alceml bdev: {jm_device.alceml_bdev}")
-                return False
+                if not force:
+                    return False
 
             jm_bdev = f"jm_{snode.get_id()}"
             ret = rpc_client.bdev_jm_create(jm_bdev, jm_device.alceml_bdev, jm_cpu_mask=snode.jm_cpu_mask)
             if not ret:
                 logger.error(f"Failed to create {jm_bdev}")
-                return False
+                if not force:
+                    return False
 
             if snode.enable_ha_jm:
                 # add pass through
@@ -917,7 +919,8 @@ def restart_jm_device(device_id, force=False, format_alceml=False):
                 ret = rpc_client.bdev_PT_NoExcl_create(pt_name, jm_bdev)
                 if not ret:
                     logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
-                    return False
+                    if not force:
+                        return False
 
                 subsystem_nqn = snode.subsystem + ":dev:" + jm_bdev
                 logger.info("creating subsystem %s", subsystem_nqn)
@@ -936,7 +939,8 @@ def restart_jm_device(device_id, force=False, format_alceml=False):
                 ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
                 if not ret:
                     logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
-                    return False
+                    if not force:
+                        return False
 
                 set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
 

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/controllers/lvol_controller.py

@@ -932,6 +932,17 @@ def delete_lvol(id_or_name, force_delete=False):
             logger.error(msg)
             return False, msg
 
+        # 1- delete subsystem from secondary
+        if secondary_node:
+            secondary_node = db_controller.get_storage_node_by_id(secondary_node.get_id())
+            if secondary_node.status == StorageNode.STATUS_ONLINE:
+                logger.info(f"Deleting subsystem for lvol:{lvol.get_id()} from node:{secondary_node.get_id()}")
+                secondary_rpc_client = secondary_node.rpc_client()
+                ret = secondary_rpc_client.subsystem_delete(lvol.nqn)
+                if not ret:
+                    logger.warning(f"Failed to delete subsystem from node: {secondary_node.get_id()}")
+
+        # 2- delete subsystem and lvol bdev from primary
         if primary_node:
 
             ret = delete_lvol_from_node(lvol.get_id(), primary_node.get_id())
@@ -940,6 +951,7 @@ def delete_lvol(id_or_name, force_delete=False):
                 if not force_delete:
                     return False
 
+        # 3- delete lvol bdev from secondary
         if secondary_node:
             secondary_node = db_controller.get_storage_node_by_id(secondary_node.get_id())
             if secondary_node.status == StorageNode.STATUS_ONLINE:

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/controllers/tasks_controller.py

@@ -96,11 +96,11 @@ def add_device_to_auto_restart(device):
 
 def add_node_to_auto_restart(node):
     cluster = db.get_cluster_by_id(node.cluster_id)
-    if cluster.status != Cluster.STATUS_ACTIVE:
-        logger.info("Cluster is not active, skip node auto restart")
+    if cluster.status not in [Cluster.STATUS_ACTIVE, Cluster.STATUS_DEGRADED, Cluster.STATUS_READONLY]:
+        logger.warning(f"Cluster is not active, skip node auto restart, status: {cluster.status}")
         return False
     for sn in db.get_storage_nodes_by_cluster_id(node.cluster_id):
-        if node.get_id() != sn.get_id() and sn.status != StorageNode.STATUS_ONLINE:
+        if node.get_id() != sn.get_id() and sn.status != StorageNode.STATUS_ONLINE and node.mgmt_ip != sn.mgmt_ip:
             logger.info("Node found that is not online, skip node auto restart")
             return False
     return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/distr_controller.py

@@ -49,7 +49,7 @@ def send_dev_status_event(device, status, target_node=None):
     else:
         snodes = db_controller.get_storage_nodes_by_cluster_id(device.cluster_id)
     for node in snodes:
-        if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]:
+        if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]:
             continue
 
         dev_status = status

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=25.6.2
+SIMPLY_BLOCK_VERSION=25.6.4
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:R25.6-PRE

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/mgmt_node_ops.py

@@ -60,6 +60,9 @@ def deploy_mgmt_node(cluster_ip, cluster_id, ifname, cluster_secret):
             logger.error("Node already exists in the cluster")
             return False
 
+    utils.render_and_deploy_alerting_configs(cluster_data['contact_point'], cluster_data['grafana_endpoint'],
+                                                                    cluster_data['uuid'], cluster_data['secret'])
+
     logger.info("Joining docker swarm...")
     try:
         cluster_docker = utils.get_docker_client(cluster_id)

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/models/cluster.py

@@ -47,6 +47,7 @@ class Cluster(BaseModel):
     enable_qos: bool = False
     grafana_endpoint: str = ""
     grafana_secret: str = ""
+    contact_point: str = ""
     ha_type: str = "single"
     inflight_io_threshold: int = 4
     iscsi: str = ""

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/models/storage_node.py

@@ -236,12 +236,15 @@ class StorageNode(BaseNodeObject):
         remote_bdev = f"{primary_node.hublvol.bdev_name}n1"
 
         if not rpc_client.get_bdevs(remote_bdev):
-
+            ip_lst = []
             for ip in (iface.ip4_address for iface in primary_node.data_nics):
+                ip_lst.append(ip)
+            multipath = bool(len(ip_lst) > 1)
+            for ip in ip_lst:
                 ret = rpc_client.bdev_nvme_attach_controller_tcp(
                         primary_node.hublvol.bdev_name, primary_node.hublvol.nqn,
-                        ip, primary_node.hublvol.nvmf_port)
-                if not ret:
+                        ip, primary_node.hublvol.nvmf_port, multipath=multipath)
+                if not ret and not multipath:
                     logger.warning(f'Failed to connect to hublvol on {ip}')
 
         if not rpc_client.bdev_lvol_set_lvs_opts(

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/rpc_client.py

@@ -151,7 +151,8 @@ class RPCClient:
             "buf_cache_size": 512,
             "dif_insert_or_strip": False,
             "c2h_success": True,
-            "sock_priority": 0
+            "sock_priority": 0,
+            "ack_timeout": 4096,
         }
         return self._request("nvmf_create_transport", params)
 
@@ -571,7 +572,10 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "num_io_queues": 128
+            "num_io_queues": 128,
+            "fast_io_fail_timeout_sec": 2,
+            "ctrlr_loss_timeout_sec": 10,
+            "reconnect_delay_sec": 2,
         }
         if multipath:
             params["multipath"] = "failover"
@@ -639,7 +643,8 @@ class RPCClient:
             "fast_io_fail_timeout_sec" : 0,
             "reconnect_delay_sec": 1,
             "keep_alive_timeout_ms": 10000,
-            "timeout_us": constants.NVME_TIMEOUT_US
+            "timeout_us": constants.NVME_TIMEOUT_US,
+            "transport_ack_timeout": 13,
         }
         return self._request("bdev_nvme_set_options", params)
 

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/services/tasks_runner_port_allow.py

@@ -2,8 +2,8 @@
 import time
 
 
-from simplyblock_core import db_controller, utils
-from simplyblock_core.controllers import tasks_events, tcp_ports_events
+from simplyblock_core import db_controller, utils, storage_node_ops, distr_controller
+from simplyblock_core.controllers import tasks_events, tcp_ports_events, health_controller
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.storage_node import StorageNode
@@ -57,8 +57,23 @@ while True:
                             task.write_to_db(db.kv_store)
                             continue
 
-                        if node.health_check is False:
-                            msg = f"Node health check fail: {node.get_id()}, retry later"
+                        logger.info("connecting remote devices")
+                        node.remote_devices = storage_node_ops._connect_to_remote_devs(
+                            node, force_conect_restarting_nodes=True)
+                        node.write_to_db()
+
+                        logger.info("Sending device status event")
+                        for db_dev in node.nvme_devices:
+                            distr_controller.send_dev_status_event(db_dev, db_dev.status)
+
+                        lvstore_check = True
+                        if node.lvstore_status == "ready":
+                            lvstore_check &= health_controller._check_node_lvstore(node.lvstore_stack, node, auto_fix=True)
+                            if node.secondary_node_id:
+                                lvstore_check &= health_controller._check_node_hublvol(node)
+
+                        if lvstore_check is False:
+                            msg = f"Node LVolStore check fail, retry later"
                             logger.warning(msg)
                             task.function_result = msg
                             task.status = JobSchedule.STATUS_SUSPENDED

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/storage_node_ops.py

@@ -1502,11 +1502,11 @@ def restart_storage_node(
     # Calculate pool count
 
     number_of_alceml_devices = snode.number_of_alceml_devices
-    small_pool_count, large_pool_count = utils.calculate_pool_count(
-        number_of_alceml_devices, snode.number_of_distribs * 2, snode.cpu, len(snode.poller_cpu_cores) or snode.cpu)
+    #small_pool_count, large_pool_count = utils.calculate_pool_count(
+    #    number_of_alceml_devices, snode.number_of_distribs * 2, snode.cpu, len(snode.poller_cpu_cores) or snode.cpu)
 
     # Calculate minimum huge page memory
-    minimum_hp_memory = utils.calculate_minimum_hp_memory(small_pool_count, large_pool_count, snode.max_lvol,
+    minimum_hp_memory = utils.calculate_minimum_hp_memory(snode.iobuf_small_pool_count, snode.iobuf_large_pool_count, snode.max_lvol,
                                                           snode.max_prov,
                                                           len(utils.hexa_to_cpu_list(snode.spdk_cpu_mask)))
 
@@ -1546,6 +1546,7 @@ def restart_storage_node(
     for n in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
         if n.api_endpoint == snode.api_endpoint:
             total_mem += n.spdk_mem
+    total_mem+= utils.parse_size("500m")
 
     results = None
     try:

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_core/utils.py

@@ -16,6 +16,9 @@ import docker
 from prettytable import PrettyTable
 from docker.errors import APIError, DockerException, ImageNotFound
 
+import tempfile
+from jinja2 import Environment, FileSystemLoader
+
 from simplyblock_core import constants
 from simplyblock_core import shell_utils
 from simplyblock_core.models.job_schedule import JobSchedule
@@ -1409,7 +1412,7 @@ def regenerate_config(new_config, old_config, force=False):
                 logger.error(f"The nic {nic} is not a member of system nics {all_nics}")
                 return False
 
-        small_pool_count, large_pool_count = calculate_pool_count(number_of_alcemls, 2 * number_of_distribs,
+        small_pool_count, large_pool_count = calculate_pool_count(number_of_alcemls + 1, 2 * number_of_distribs,
                                                                   len(isolated_cores),
                                                                   number_of_poller_cores or len(
                                                                       isolated_cores),)
@@ -1478,7 +1481,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a
     nvme_by_numa = {nid: [] for nid in sockets_to_use}
     nvme_numa_neg1 = []
     for nvme_name, val in nvmes.items():
-        numa = val["numa_node"]
+        numa = int(val["numa_node"])
         if numa in sockets_to_use:
             nvme_by_numa[numa].append(nvme_name)
         elif int(numa) == -1:
@@ -1760,3 +1763,55 @@ def remove_container(client: docker.DockerClient, name, timeout=3):
     except docker.errors.APIError as e:
         if 'Conflict ("removal of container {container.id} is already in progress")' != e.response.reason:
             raise
+
+def render_and_deploy_alerting_configs(contact_point, grafana_endpoint, cluster_uuid, cluster_secret):
+    TOP_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")
+    alert_resources_file = "alert_resources.yaml"
+
+    env = Environment(loader=FileSystemLoader(alerts_template_folder), trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template(f'{alert_resources_file}.j2')
+
+    slack_pattern = re.compile(r"https://hooks\.slack\.com/services/\S+")
+    email_pattern = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+
+    if slack_pattern.match(contact_point):
+        ALERT_TYPE = "slack"
+    elif email_pattern.match(contact_point):
+        ALERT_TYPE = "email"
+    else:
+        ALERT_TYPE = "slack"
+        contact_point = 'https://hooks.slack.com/services/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'
+
+    values = {
+        'CONTACT_POINT': contact_point,
+        'GRAFANA_ENDPOINT': grafana_endpoint,
+        'ALERT_TYPE': ALERT_TYPE,
+    }
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_file_path = os.path.join(temp_dir, alert_resources_file)
+        with open(temp_file_path, 'w') as file:
+            file.write(template.render(values))
+
+        destination_file_path = os.path.join(alerts_template_folder, alert_resources_file)
+        subprocess.check_call(['sudo', '-v'])  # sudo -v checks if the current user has sudo permissions
+        subprocess.check_call(['sudo', 'mv', temp_file_path, destination_file_path])
+        print(f"File moved to {destination_file_path} successfully.")
+
+    scripts_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/")
+    prometheus_file = "prometheus.yml"
+    env = Environment(loader=FileSystemLoader(scripts_folder), trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template(f'{prometheus_file}.j2')
+    values = {
+        'CLUSTER_ID': cluster_uuid,
+        'CLUSTER_SECRET': cluster_secret}
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        file_path = os.path.join(temp_dir, prometheus_file)
+        with open(file_path, 'w') as file:
+            file.write(template.render(values))
+
+        prometheus_file_path = os.path.join(scripts_folder, prometheus_file)
+        subprocess.check_call(['sudo', 'mv', file_path, prometheus_file_path])
+        print(f"File moved to {prometheus_file_path} successfully.")

{sbcli-pre-25.6.2 → sbcli-pre-25.6.4}/simplyblock_web/blueprints/snode_ops.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # encoding: utf-8
 import json
+import math
 import os
 import time
 from typing import List, Optional, Union
@@ -633,7 +634,7 @@ def set_hugepages():
         numa = node_config["socket"]
         huge_page_memory_dict[numa] = huge_page_memory_dict.get(numa, 0) + node_config["huge_page_memory"]
     for numa, huge_page_memory in huge_page_memory_dict.items():
-        num_pages = huge_page_memory // (2048 * 1024)
+        num_pages = math.ceil(huge_page_memory / (2048 * 1024))
         core_utils.set_hugepages_if_needed(numa, num_pages)
 
     return utils.get_response(True)