sbcli-pre 1.0.4__zip → 1.0.6__zip

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.0.4
+Version: 1.0.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=1.0.4
+SIMPLY_BLOCK_VERSION=1.0.6
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
 

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.0.4
+Version: 1.0.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_cli/cli.py

@@ -472,7 +472,7 @@ class CLIWrapper:
         # mgmt-node ops
         subparser = self.add_command('mgmt', 'Management node commands')
 
-        sub_command = self.add_sub_command(subparser, 'add', 'Add Management node to the cluster')
+        sub_command = self.add_sub_command(subparser, 'add', 'Add Management node to the cluster (local run)')
         sub_command.add_argument("cluster_ip", help='the cluster IP address')
         sub_command.add_argument("cluster_id", help='the cluster UUID')
         sub_command.add_argument("ifname", help='Management interface name')
@@ -823,8 +823,6 @@ class CLIWrapper:
             sub_command = args_dict[args.command]
             if sub_command == 'create':
                 ret = self.cluster_create(args)
-            elif sub_command == 'join':
-                ret = self.cluster_join(args)
             elif sub_command == 'add':
                 ret = self.cluster_add(args)
             elif sub_command == 'status':
@@ -990,7 +988,7 @@ class CLIWrapper:
                 cluster_id = args.cluster_id
                 cluster_ip = args.cluster_ip
                 ifname = args.ifname
-                ret = cluster_ops.join_cluster(cluster_ip, cluster_id, "management", ifname, [], None, None)
+                ret = mgmt_ops.deploy_mgmt_node(cluster_ip, cluster_id, ifname)
             elif sub_command == "list":
                 ret = mgmt_ops.list_mgmt_nodes(args.json)
             elif sub_command == "remove":
@@ -1174,27 +1172,6 @@ class CLIWrapper:
             CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
             ifname, log_del_interval, metrics_retention_period)
 
-    def cluster_join(self, args):
-        cluster_id = args.cluster_id
-        cluster_ip = args.cluster_ip
-        role = args.role
-        ifname = args.ifname
-        data_nics = args.data_nics
-        spdk_cpu_mask = None
-        if args.spdk_cpu_mask:
-            if self.validate_cpu_mask(args.spdk_cpu_mask):
-                spdk_cpu_mask = args.spdk_cpu_mask
-            else:
-                return f"Invalid cpu mask value: {args.spdk_cpu_mask}"
-
-        spdk_mem = None
-        if args.spdk_mem:
-            spdk_mem = self.parse_size(args.spdk_mem)
-            if spdk_mem < 1 * 1024 * 1024:
-                return f"SPDK memory:{args.spdk_mem} must be larger than 1G"
-
-        return cluster_ops.join_cluster(cluster_ip, cluster_id, role, ifname, data_nics, spdk_cpu_mask, spdk_mem)
-
     def query_yes_no(self, question, default="yes"):
         """Ask a yes/no question via raw_input() and return their answer.
 

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_core/cluster_ops.py

@@ -1,7 +1,6 @@
 # coding=utf-8
 import json
 import logging
-import math
 import os
 import time
 import uuid
@@ -9,7 +8,7 @@ import uuid
 import docker
 import requests
 
-from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops, shell_utils
+from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops
 from simplyblock_core.controllers import cluster_events, device_controller
 from simplyblock_core.kv_store import DBController
 from simplyblock_core.models.cluster import Cluster
@@ -199,183 +198,6 @@ def deploy_spdk(node_docker, spdk_cpu_mask, spdk_mem):
             break
 
 
-def join_cluster(cluster_ip, cluster_id, role, ifname, data_nics,  spdk_cpu_mask, spdk_mem):  # role: ["management", "storage"]
-
-    if role not in ["management", "storage", "storage-alloc"]:
-        logger.error(f"Unknown role: {role}")
-        return False
-
-    try:
-        resp = requests.get(f"http://{cluster_ip}/cluster/{cluster_id}")
-        resp_json = resp.json()
-        cluster_data = resp_json['results'][0]
-        logger.info(f"Cluster found! NQN:{cluster_data['nqn']}")
-        logger.debug(cluster_data)
-    except Exception as e:
-        logger.error("Error getting cluster data!")
-        logger.error(e)
-        return ""
-
-    logger.info("Installing dependencies...")
-    ret = scripts.install_deps()
-    logger.info("Installing dependencies > Done")
-
-    if not ifname:
-        ifname = "eth0"
-
-    DEV_IP = utils.get_iface_ip(ifname)
-    if not DEV_IP:
-        logger.error(f"Error getting interface ip: {ifname}")
-        return False
-
-    logger.info(f"Node IP: {DEV_IP}")
-    ret = scripts.configure_docker(DEV_IP)
-
-    db_connection = cluster_data['db_connection']
-    ret = scripts.set_db_config(db_connection)
-
-    if role == "storage":
-        logger.info("Deploying SPDK")
-        node_cpu_count = os.cpu_count()
-        if spdk_cpu_mask:
-            requested_cpu_count = len(format(int(spdk_cpu_mask, 16), 'b'))
-            if requested_cpu_count > node_cpu_count:
-                logger.error(f"The requested cpu count: {requested_cpu_count} "
-                             f"is larger than the node's cpu count: {node_cpu_count}")
-                return False
-        else:
-            spdk_cpu_mask = hex(int(math.pow(2, node_cpu_count))-1)
-        if spdk_mem:
-            spdk_mem = int(spdk_mem/(1024*1024))
-        else:
-            spdk_mem = 4096
-        node_docker = docker.DockerClient(base_url=f"tcp://{DEV_IP}:2375", version="auto", timeout=60*5)
-        deploy_spdk(node_docker, spdk_cpu_mask, spdk_mem)
-        time.sleep(5)
-
-    logger.info("Joining docker swarm...")
-    db_controller = DBController()
-    nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
-    if not nodes:
-        logger.error("No mgmt nodes was found in the cluster!")
-        exit(1)
-
-    try:
-        cluster_docker = utils.get_docker_client(cluster_id)
-        docker_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-
-        if role == 'management':
-            join_token = cluster_docker.swarm.attrs['JoinTokens']['Manager']
-        else:
-            join_token = cluster_docker.swarm.attrs['JoinTokens']['Worker']
-
-        node_docker = docker.DockerClient(base_url=f"tcp://{DEV_IP}:2375", version="auto")
-        if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
-            logger.info("Node is part of another swarm, leaving swarm")
-            try:
-                cluster_docker.nodes.get(node_docker.info()["Swarm"]["NodeID"]).remove(force=True)
-            except:
-                pass
-            node_docker.swarm.leave(force=True)
-            time.sleep(5)
-        node_docker.swarm.join([f"{docker_ip}:2377"], join_token)
-
-        retries = 10
-        while retries > 0:
-            if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
-                break
-            logger.info("Waiting for node to be active...")
-            retries -= 1
-            time.sleep(2)
-        logger.info("Joining docker swarm > Done")
-        time.sleep(5)
-
-    except Exception as e:
-        raise e
-
-    if role == 'management':
-        mgmt_node_ops.add_mgmt_node(DEV_IP, cluster_id)
-        cluster_obj = db_controller.get_cluster_by_id(cluster_id)
-        nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
-        if len(nodes) >= 3:
-            logger.info("Waiting for FDB container to be active...")
-            fdb_cont = None
-            retries = 30
-            while retries > 0 and fdb_cont is None:
-                logger.info("Looking for FDB container...")
-                for cont in node_docker.containers.list(all=True):
-                    logger.debug(cont.attrs['Name'])
-                    if cont.attrs['Name'].startswith("/app_fdb"):
-                        fdb_cont = cont
-                        break
-                if fdb_cont:
-                    logger.info("FDB container found")
-                    break
-                else:
-                    retries -= 1
-                    time.sleep(5)
-
-            if not fdb_cont:
-                logger.warning("FDB container was not found")
-            else:
-                retries = 10
-                while retries > 0:
-                    info = node_docker.containers.get(fdb_cont.attrs['Id'])
-                    status = info.attrs['State']["Status"]
-                    is_running = info.attrs['State']["Running"]
-                    if not is_running:
-                        logger.info("Container is not running, waiting...")
-                        time.sleep(3)
-                        retries -= 1
-                    else:
-                        logger.info(f"Container status: {status}, Is Running: {is_running}")
-                    break
-
-            logger.info("Configuring Double DB...")
-            time.sleep(3)
-            out = scripts.set_db_config_double()
-            cluster_obj.ha_type = "ha"
-            cluster_obj.write_to_db(db_controller.kv_store)
-            logger.info("Configuring DB > Done")
-
-    elif role == "storage":
-        # add storage node
-        fdb_cont = None
-        retries = 30
-        while retries > 0 and fdb_cont is None:
-            logger.info("Looking for SpdkAppProxy container...")
-            for cont in node_docker.containers.list(all=True):
-                logger.debug(cont.attrs['Name'])
-                if cont.attrs['Name'].startswith("/app_SpdkAppProxy"):
-                    fdb_cont = cont
-                    break
-            if fdb_cont:
-                logger.info("SpdkAppProxy container found")
-                break
-            else:
-                retries -= 1
-                time.sleep(5)
-
-        if not fdb_cont:
-            logger.warning("SpdkAppProxy container was not found")
-        else:
-            retries = 10
-            while retries > 0:
-                info = node_docker.containers.get(fdb_cont.attrs['Id'])
-                status = info.attrs['State']["Status"]
-                is_running = info.attrs['State']["Running"]
-                if not is_running:
-                    logger.info("Container is not running, waiting...")
-                    time.sleep(3)
-                    retries -= 1
-                else:
-                    logger.info(f"Container status: {status}, Is Running: {is_running}")
-                break
-        storage_node_ops.add_storage_node(cluster_id, ifname, data_nics)
-
-    logger.info("Node joined the cluster")
-
-
 def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit):
     db_controller = DBController()
     clusters = db_controller.get_clusters()

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_core/controllers/device_controller.py

@@ -36,7 +36,7 @@ def device_set_state(device_id, state):
 
     old_status = dev.status
     device.status = state
-    distr_controller.send_dev_status_event(device.cluster_device_order, device.status)
+    distr_controller.send_dev_status_event(device, device.status)
     snode.write_to_db(db_controller.kv_store)
     device_events.device_status_change(device, device.status, old_status)
     return True
@@ -268,7 +268,7 @@ def device_remove(device_id, force=True):
             break
 
     logger.info("Sending device event")
-    distr_controller.send_dev_status_event(device.cluster_device_order, "removed")
+    distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_REMOVED)
 
     logger.info("Disconnecting device from all nodes")
     distr_controller.disconnect_device(device)

sbcli_pre-1.0.6/simplyblock_core/mgmt_node_ops.py

@@ -0,0 +1,205 @@
+# coding=utf-8
+import json
+import logging
+import uuid
+import time
+import requests
+
+import docker
+
+from simplyblock_core import utils, scripts
+from simplyblock_core.controllers import mgmt_events
+from simplyblock_core.kv_store import DBController
+from simplyblock_core.models.mgmt_node import MgmtNode
+
+
+logger = logging.getLogger()
+
+
+def deploy_mgmt_node(cluster_ip, cluster_id, ifname):
+
+    try:
+        resp = requests.get(f"http://{cluster_ip}/cluster/{cluster_id}")
+        resp_json = resp.json()
+        cluster_data = resp_json['results'][0]
+        logger.info(f"Cluster found! NQN:{cluster_data['nqn']}")
+        logger.debug(cluster_data)
+    except Exception as e:
+        logger.error("Error getting cluster data!")
+        logger.error(e)
+        return ""
+
+    logger.info("Installing dependencies...")
+    scripts.install_deps()
+    logger.info("Installing dependencies > Done")
+
+    if not ifname:
+        ifname = "eth0"
+
+    DEV_IP = utils.get_iface_ip(ifname)
+    if not DEV_IP:
+        logger.error(f"Error getting interface ip: {ifname}")
+        return False
+
+    logger.info(f"Node IP: {DEV_IP}")
+    ret = scripts.configure_docker(DEV_IP)
+
+    db_connection = cluster_data['db_connection']
+    scripts.set_db_config(db_connection)
+    time.sleep(1)
+    hostname = utils.get_hostname()
+    db_controller = DBController()
+    nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
+    if not nodes:
+        logger.error("No mgmt nodes was found in the cluster!")
+        return False
+    for node in nodes:
+        if node.hostname == hostname:
+            logger.error("Node already exists in the cluster")
+            return False
+
+    logger.info("Joining docker swarm...")
+    try:
+        cluster_docker = utils.get_docker_client(cluster_id)
+        docker_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
+        join_token = cluster_docker.swarm.attrs['JoinTokens']['Manager']
+        node_docker = docker.DockerClient(base_url=f"tcp://{DEV_IP}:2375", version="auto")
+        if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
+            logger.info("Node is part of another swarm, leaving swarm")
+            try:
+                cluster_docker.nodes.get(node_docker.info()["Swarm"]["NodeID"]).remove(force=True)
+            except:
+                pass
+            node_docker.swarm.leave(force=True)
+            time.sleep(5)
+
+        node_docker.swarm.join([f"{docker_ip}:2377"], join_token)
+
+        retries = 10
+        while retries > 0:
+            if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
+                break
+            logger.info("Waiting for node to be active...")
+            retries -= 1
+            time.sleep(2)
+        logger.info("Joining docker swarm > Done")
+        time.sleep(5)
+
+    except Exception as e:
+        raise e
+
+    logger.info("Adding management node object")
+    node_id = add_mgmt_node(DEV_IP, cluster_id)
+
+    # check if ha setting is required
+    nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
+    if len(nodes) >= 3:
+        logger.info("Waiting for FDB container to be active...")
+        fdb_cont = None
+        retries = 30
+        while retries > 0 and fdb_cont is None:
+            logger.info("Looking for FDB container...")
+            for cont in node_docker.containers.list(all=True):
+                logger.debug(cont.attrs['Name'])
+                if cont.attrs['Name'].startswith("/app_fdb"):
+                    fdb_cont = cont
+                    break
+            if fdb_cont:
+                logger.info("FDB container found")
+                break
+            else:
+                retries -= 1
+                time.sleep(5)
+
+        if not fdb_cont:
+            logger.warning("FDB container was not found")
+        else:
+            retries = 10
+            while retries > 0:
+                info = node_docker.containers.get(fdb_cont.attrs['Id'])
+                status = info.attrs['State']["Status"]
+                is_running = info.attrs['State']["Running"]
+                if not is_running:
+                    logger.info("Container is not running, waiting...")
+                    time.sleep(3)
+                    retries -= 1
+                else:
+                    logger.info(f"Container status: {status}, Is Running: {is_running}")
+                break
+
+        logger.info("Configuring Double DB...")
+        time.sleep(3)
+        scripts.set_db_config_double()
+        for cl in db_controller.get_clusters():
+            cl.ha_type = "ha"
+            cl.write_to_db(db_controller.kv_store)
+
+    logger.info("Node joined the cluster")
+    return node_id
+
+
+def add_mgmt_node(mgmt_ip, cluster_id=None):
+    db_controller = DBController()
+    hostname = utils.get_hostname()
+    node = db_controller.get_mgmt_node_by_hostname(hostname)
+    if node:
+        logger.error("Node already exists in the cluster")
+        return False
+
+    node = MgmtNode()
+    node.uuid = str(uuid.uuid4())
+    node.hostname = hostname
+    node.docker_ip_port = f"{mgmt_ip}:2375"
+    node.cluster_id = cluster_id
+    node.mgmt_ip = mgmt_ip
+    node.status = MgmtNode.STATUS_ONLINE
+    node.write_to_db(db_controller.kv_store)
+
+    mgmt_events.mgmt_add(node)
+    logger.info("Done")
+    return node.uuid
+
+
+def list_mgmt_nodes(is_json):
+    db_controller = DBController()
+    nodes = db_controller.get_mgmt_nodes()
+    data = []
+    output = ""
+
+    for node in nodes:
+        logging.debug(node)
+        logging.debug("*" * 20)
+        data.append({
+            "UUID": node.get_id(),
+            "Hostname": node.hostname,
+            "IP": node.mgmt_ip,
+            "Status": node.status,
+        })
+
+    if not data:
+        return output
+
+    if is_json:
+        output = json.dumps(data, indent=2)
+    else:
+        output = utils.print_table(data)
+    return output
+
+
+def remove_mgmt_node(uuid):
+    db_controller = DBController()
+    snode = db_controller.get_mgmt_node_by_id(uuid)
+    if not snode:
+        logger.error("can not find node")
+        return False
+
+    logging.info("Removing mgmt node")
+    snode.remove(db_controller.kv_store)
+
+    logger.info("Leaving swarm...")
+    node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto")
+    node_docker.swarm.leave()
+
+    mgmt_events.mgmt_remove(snode)
+    logging.info("done")
+

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_core/storage_node_ops.py

@@ -1112,7 +1112,7 @@ def restart_storage_node(
         else:
             logger.info(f"Device not found: {db_dev.get_id()}")
             db_dev.status = NVMeDevice.STATUS_REMOVED
-            distr_controller.send_dev_status_event(db_dev.cluster_device_order, db_dev.status)
+            distr_controller.send_dev_status_event(db_dev, db_dev.status)
 
     # todo: handle new devices
     # for dev in nvme_devs:

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_web/blueprints/web_api_lvol.py

@@ -32,7 +32,8 @@ def list_lvols(uuid, cluster_id):
     elif cluster_id:
         lvols = db_controller.get_lvols(cluster_id)
     else:
-        lvols = db_controller.get_lvols()  # pass
+        cluster_id = utils.get_cluster_id(request)
+        lvols = db_controller.get_lvols(cluster_id)
     data = []
     for lvol in lvols:
         data.append(lvol.get_clean_dict())

{sbcli_pre-1.0.4 → sbcli_pre-1.0.6}/simplyblock_web/blueprints/web_api_pool.py

@@ -30,7 +30,8 @@ def list_pools(uuid, cluster_id):
     elif cluster_id:
         pools = db_controller.get_pools(cluster_id)
     else:
-        pools = db_controller.get_pools()
+        cluster_id = utils.get_cluster_id(request)
+        pools = db_controller.get_pools(cluster_id)
     data = []
     for pool in pools:
         data.append(pool.get_clean_dict())
@@ -42,6 +43,7 @@ def add_pool():
     """
         Params:
         | name (required) | LVol name or id
+        | cluster_id (required) | Cluster uuid
         | pool_max        | Pool maximum size: 10M, 10G, 10(bytes)
         | lvol_max        | LVol maximum size: 10M, 10G, 10(bytes)
         | no_secret       | pool is created with a secret

sbcli_pre-1.0.4/simplyblock_core/mgmt_node_ops.py

@@ -1,80 +0,0 @@
-# coding=utf-8
-import json
-import logging
-import uuid
-
-import docker
-
-from simplyblock_core import utils
-from simplyblock_core.controllers import mgmt_events
-from simplyblock_core.kv_store import DBController
-from simplyblock_core.models.mgmt_node import MgmtNode
-
-logger = logging.getLogger()
-
-
-def add_mgmt_node(mgmt_ip, cluster_id=None):
-    db_controller = DBController()
-    hostname = utils.get_hostname()
-    node = db_controller.get_mgmt_node_by_hostname(hostname)
-    if node:
-        logger.error("Node already exists in the cluster")
-        return False
-
-    node = MgmtNode()
-    node.uuid = str(uuid.uuid4())
-    node.hostname = hostname
-    node.docker_ip_port = f"{mgmt_ip}:2375"
-    node.cluster_id = cluster_id
-    node.mgmt_ip = mgmt_ip
-    node.status = MgmtNode.STATUS_ONLINE
-    node.write_to_db(db_controller.kv_store)
-
-    mgmt_events.mgmt_add(node)
-    logger.info("Done")
-    return True
-
-
-def list_mgmt_nodes(is_json):
-    db_controller = DBController()
-    nodes = db_controller.get_mgmt_nodes()
-    data = []
-    output = ""
-
-    for node in nodes:
-        logging.debug(node)
-        logging.debug("*" * 20)
-        data.append({
-            "UUID": node.get_id(),
-            "Hostname": node.hostname,
-            "IP": node.mgmt_ip,
-            "Status": node.status,
-        })
-
-    if not data:
-        return output
-
-    if is_json:
-        output = json.dumps(data, indent=2)
-    else:
-        output = utils.print_table(data)
-    return output
-
-
-def remove_mgmt_node(uuid):
-    db_controller = DBController()
-    snode = db_controller.get_mgmt_node_by_id(uuid)
-    if not snode:
-        logger.error("can not find node")
-        return False
-
-    logging.info("Removing mgmt node")
-    snode.remove(db_controller.kv_store)
-
-    logger.info("Leaving swarm...")
-    node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto")
-    node_docker.swarm.leave()
-
-    mgmt_events.mgmt_remove(snode)
-    logging.info("done")
-