sbcli-dev 4.0.4__zip → 4.0.6__zip

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.4
+Version: 4.0.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.4
+SIMPLY_BLOCK_VERSION=4.0.6
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.4
+Version: 4.0.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_core/constants.py

@@ -100,3 +100,6 @@ INSTANCE_STORAGE_DATA = {
     }
 
 MAX_SNAP_COUNT = 15
+
+SPDK_PROXY_MULTI_THREADING_ENABLED=False
+SPDK_PROXY_TIMEOUT=60*5

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_core/controllers/device_controller.py

@@ -672,7 +672,7 @@ def add_device(device_id):
     if snode.jm_device and snode.jm_device.raid_bdev:
         # looking for jm partition
         rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-        jm_dev_part = f"{dev.nvme_bdev[:-1]}1"
+        jm_dev_part = f"{dev.nvme_bdev[:-2]}p1"
         ret = rpc_client.get_bdevs(jm_dev_part)
         if ret:
             logger.info(f"JM part found: {jm_dev_part}")
@@ -714,6 +714,10 @@ def set_jm_device_state(device_id, state):
         for node_index, node in enumerate(snodes):
             if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
+            for jm_dev in node.remote_jm_devices:
+                if jm_dev.get_id() == device_id and jm_device.status != JMDevice.STATUS_ONLINE:
+                    node.remote_jm_devices.remove(jm_dev)
+                    break
             logger.info(f"Connecting to node: {node.get_id()}")
             node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
             node.write_to_db(db_controller.kv_store)
@@ -796,7 +800,7 @@ def restart_jm_device(device_id, force=False):
         for dev in snode.nvme_devices:
             if dev.status != NVMeDevice.STATUS_ONLINE:
                 continue
-            dev_part = f"{dev.nvme_bdev[:-1]}1"
+            dev_part = f"{dev.nvme_bdev[:-2]}p1"
             if dev_part in bdevs_names:
                 if dev_part not in jm_nvme_bdevs:
                     jm_nvme_bdevs.append(dev_part)

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_core/services/spdk_http_proxy_server.py

@@ -13,7 +13,6 @@ from http.server import ThreadingHTTPServer
 from http.server import BaseHTTPRequestHandler
 
 
-TIMEOUT = 5*60  # 5 min
 rpc_sock = '/var/tmp/spdk.sock'
 logger_handler = logging.StreamHandler(stream=sys.stdout)
 logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
@@ -34,7 +33,7 @@ def get_env_var(name, default=None, is_required=False):
 
 def rpc_call(req):
     req_data = json.loads(req.decode('ascii'))
-    print(req_data)
+    logger.debug(f"Request data: {str(req_data)}")
     sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
     sock.settimeout(TIMEOUT)
     sock.connect(rpc_sock)
@@ -64,7 +63,7 @@ def rpc_call(req):
     if not response and len(buf) > 0:
         raise
 
-    print(buf)
+    logger.debug(f"Response data: {buf}")
 
     return buf
 
@@ -122,24 +121,28 @@ class ServerHandler(BaseHTTPRequestHandler):
                 self.do_INTERNALERROR()
 
 
-def run_server(host, port, user, password, cert=None):
+def run_server(host, port, user, password, cert=None, is_threading_enabled=False):
     # encoding user and password
     key = base64.b64encode((user+':'+password).encode(encoding='ascii')).decode('ascii')
 
     try:
         ServerHandler.key = key
-        # httpd = ThreadingHTTPServer((host, port), ServerHandler)
-        httpd = HTTPServer((host, port), ServerHandler)
+        if is_threading_enabled:
+            httpd = ThreadingHTTPServer((host, port), ServerHandler)
+        else:
+            httpd = HTTPServer((host, port), ServerHandler)
         httpd.timeout = TIMEOUT
         if cert is not None:
             httpd.socket = ssl.wrap_socket(httpd.socket, certfile=cert, server_side=True)
-        print('Started RPC http proxy server')
+        logger.info('Started RPC http proxy server')
         httpd.serve_forever()
     except KeyboardInterrupt:
-        print('Shutting down server')
+        logger.info('Shutting down server')
         httpd.socket.close()
 
 
+TIMEOUT = int(get_env_var("TIMEOUT", is_required=False, default=60*5))
+is_threading_enabled = get_env_var("MULTI_THREADING_ENABLED", is_required=False, default=False)
 server_ip = get_env_var("SERVER_IP", is_required=True)
 rpc_port = get_env_var("RPC_PORT", is_required=True)
 rpc_username = get_env_var("RPC_USERNAME", is_required=True)
@@ -150,4 +153,5 @@ try:
 except Exception:
     rpc_port = 8080
 
-run_server(server_ip, rpc_port, rpc_username, rpc_password)
+is_threading_enabled = bool(is_threading_enabled)
+run_server(server_ip, rpc_port, rpc_username, rpc_password, is_threading_enabled=is_threading_enabled)

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_core/snode_client.py

@@ -72,7 +72,7 @@ class SNodeClient:
 
     def spdk_process_start(self, spdk_cpu_mask, spdk_mem, spdk_image=None, spdk_debug=None, cluster_ip=None,
                            fdb_connection=None, namespace=None, server_ip=None, rpc_port=None,
-                           rpc_username=None, rpc_password=None):
+                           rpc_username=None, rpc_password=None, multi_threading_enabled=False, timeout=0):
         params = {
             "cluster_ip": cluster_ip,
             "server_ip": server_ip,
@@ -92,6 +92,10 @@ class SNodeClient:
             params['fdb_connection'] = fdb_connection
         if namespace:
             params["namespace"] = namespace
+        if multi_threading_enabled:
+            params["multi_threading_enabled"] = multi_threading_enabled
+        if timeout:
+            params["timeout"] = timeout
         return self._request("POST", "spdk_process_start", params)
 
     def join_swarm(self, cluster_ip, join_token, db_connection, cluster_id):

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_core/storage_node_ops.py

@@ -798,16 +798,24 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
     if jm_ids:
         for jm_id in jm_ids:
             jm_dev = db_controller.get_jm_device_by_id(jm_id)
-            remote_devices.append(jm_dev)
+            if jm_dev:
+                remote_devices.append(jm_dev)
     elif len(this_node.remote_jm_devices) > 0:
         remote_devices = this_node.remote_jm_devices
     else:
         for node in db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id):
             if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
-            if node.jm_device:
+            if node.jm_device and node.jm_device.status == JMDevice.STATUS_ONLINE:
                 remote_devices.append(node.jm_device)
 
+    if len(remote_devices) < 2:
+        online_jms = get_next_ha_jms(this_node)
+        for jm_id in online_jms:
+            jm_dev = db_controller.get_jm_device_by_id(jm_id)
+            if jm_dev:
+                remote_devices.append(jm_dev)
+
     for jm_dev in remote_devices:
         name = f"remote_{jm_dev.jm_bdev}"
         bdev_name = f"{name}n1"
@@ -1004,7 +1012,8 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     try:
         results, err = snode_api.spdk_process_start(
             spdk_cpu_mask, spdk_mem, spdk_image, spdk_debug, cluster_ip, fdb_connection,
-            namespace, mgmt_ip, constants.RPC_HTTP_PROXY_PORT, rpc_user, rpc_pass)
+            namespace, mgmt_ip, constants.RPC_HTTP_PROXY_PORT, rpc_user, rpc_pass,
+            multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT)
     except Exception as e:
         logger.error(e)
         return False
@@ -1353,6 +1362,10 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
                 distr_controller.disconnect_device(dev)
             device_controller.device_set_failed(dev.get_id())
 
+    if snode.jm_device:
+        logger.info("Removing JM")
+        device_controller.remove_jm_device(snode.jm_device.get_id(), force=True)
+
     logger.info("Removing storage node")
 
     logger.debug("Leaving swarm...")
@@ -1535,7 +1548,8 @@ def restart_storage_node(
         fdb_connection = cluster.db_connection
         results, err = snode_api.spdk_process_start(
             snode.spdk_cpu_mask, spdk_mem, snode.spdk_image, spdk_debug, cluster_ip, fdb_connection,
-            snode.namespace, snode.mgmt_ip, constants.RPC_HTTP_PROXY_PORT, snode.rpc_username, snode.rpc_password)
+            snode.namespace, snode.mgmt_ip, constants.RPC_HTTP_PROXY_PORT, snode.rpc_username, snode.rpc_password,
+            multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT)
     except Exception as e:
         logger.error(e)
         return False
@@ -1685,6 +1699,7 @@ def restart_storage_node(
 
     if snode.enable_ha_jm:
         logger.info("Connecting to remote JMs")
+        snode.remote_jm_devices = []
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode)
 
     logger.info("Setting node status to Online")
@@ -1712,6 +1727,9 @@ def restart_storage_node(
             continue
         tasks_controller.add_device_mig_task(dev.get_id())
 
+    if cluster.status == cluster.STATUS_UNREADY:
+        logger.info("Done")
+        return "Success"
     # Create distribs, raid0, and lvstore and expose lvols to the fabrics
     if snode.lvstore_stack:
         temp_rpc_client = RPCClient(
@@ -2560,12 +2578,15 @@ def get_next_ha_jms(current_node):
     for node in db_controller.get_storage_nodes_by_cluster_id(current_node.cluster_id):
         if node.get_id() == current_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
-        if node.jm_device:
+        if node.jm_device and node.jm_device.status == JMDevice.STATUS_ONLINE:
             jm_count[node.jm_device.get_id()] = 1 + jm_count.get(node.jm_device.get_id(), 0)
         for rem_jm_device in node.remote_jm_devices:
             if rem_jm_device.get_id() != current_node.jm_device.get_id():
-                jm_count[rem_jm_device.get_id()] = 1 + jm_count.get(rem_jm_device.get_id(), 0)
-
+                try:
+                    if db_controller.get_jm_device_by_id(rem_jm_device.get_id()).status == JMDevice.STATUS_ONLINE:
+                        jm_count[rem_jm_device.get_id()] = 1 + jm_count.get(rem_jm_device.get_id(), 0)
+                except :
+                    pass
     jm_count = dict(sorted(jm_count.items(), key=lambda x: x[1]))
     return list(jm_count.keys())[:2]
 
@@ -2595,6 +2616,7 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
     if snode.enable_ha_jm:
         jm_vuid = utils.get_random_vuid()
         online_jms = get_next_ha_jms(snode)
+        logger.debug(f"online_jms: {str(online_jms)}")
         snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, online_jms)
         snode.write_to_db()
 

{sbcli_dev-4.0.4 → sbcli_dev-4.0.6}/simplyblock_web/blueprints/snode_ops.py

@@ -96,6 +96,17 @@ def spdk_process_start():
     if 'spdk_mem' in data:
         spdk_mem = data['spdk_mem']
 
+    multi_threading_enabled = False
+    if 'multi_threading_enabled' in data:
+        multi_threading_enabled = bool(data['multi_threading_enabled'])
+
+    timeout = 60*5
+    if 'timeout' in data:
+        try:
+            timeout = int(data['timeout'])
+        except:
+            pass
+
     if spdk_mem:
         spdk_mem = int(utils.parse_size(spdk_mem) / (1000 * 1000))
     else:
@@ -157,6 +168,8 @@ def spdk_process_start():
             f"RPC_PORT={data['rpc_port']}",
             f"RPC_USERNAME={data['rpc_username']}",
             f"RPC_PASSWORD={data['rpc_password']}",
+            f"MULTI_THREADING_ENABLED={multi_threading_enabled}",
+            f"TIMEOUT={timeout}",
         ]
         # restart_policy={"Name": "always"}
     )