sbcli-dev 19.2.27__tar.gz → 19.2.30__tar.gz

{sbcli_dev-19.2.27/sbcli_dev.egg-info → sbcli_dev-19.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 19.2.27
+Version: 19.2.30
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy
@@ -33,6 +33,7 @@ Requires-Dist: flask-openapi3
 Requires-Dist: jsonschema
 Requires-Dist: fastapi
 Requires-Dist: uvicorn
+Requires-Dist: prometheus_api_client
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30}/requirements.txt

@@ -24,3 +24,4 @@ flask-openapi3
 jsonschema
 fastapi
 uvicorn
+prometheus_api_client

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30/sbcli_dev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 19.2.27
+Version: 19.2.30
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy
@@ -33,6 +33,7 @@ Requires-Dist: flask-openapi3
 Requires-Dist: jsonschema
 Requires-Dist: fastapi
 Requires-Dist: uvicorn
+Requires-Dist: prometheus_api_client
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30}/sbcli_dev.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ simplyblock_core/distr_controller.py
 simplyblock_core/env_var
 simplyblock_core/fw_api_client.py
 simplyblock_core/mgmt_node_ops.py
+simplyblock_core/prom_client.py
 simplyblock_core/rpc_client.py
 simplyblock_core/shell_utils.py
 simplyblock_core/snode_client.py
@@ -87,6 +88,13 @@ simplyblock_core/scripts/charts/values.yaml
 simplyblock_core/scripts/charts/crds/apps.foundationdb.org_foundationdbbackups.yaml
 simplyblock_core/scripts/charts/crds/apps.foundationdb.org_foundationdbclusters.yaml
 simplyblock_core/scripts/charts/crds/apps.foundationdb.org_foundationdbrestores.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockdevices.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocklvols.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockpools.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstorageclusters.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstoragenodes.yaml
+simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocktasks.yaml
+simplyblock_core/scripts/charts/templates/_helpers.tpl
 simplyblock_core/scripts/charts/templates/app_configmap.yaml
 simplyblock_core/scripts/charts/templates/app_ingress.yaml
 simplyblock_core/scripts/charts/templates/app_k8s.yaml
@@ -103,6 +111,8 @@ simplyblock_core/scripts/charts/templates/monitoring_ingress.yaml
 simplyblock_core/scripts/charts/templates/monitoring_k8s.yaml
 simplyblock_core/scripts/charts/templates/monitoring_secret.yaml
 simplyblock_core/scripts/charts/templates/monitoring_svc.yaml
+simplyblock_core/scripts/charts/templates/simplyblock-manager.yaml
+simplyblock_core/scripts/charts/templates/simplyblock_customresource.yaml
 simplyblock_core/scripts/charts/templates/storage_class.yaml
 simplyblock_core/scripts/dashboards/cluster.json
 simplyblock_core/scripts/dashboards/devices.json
@@ -188,6 +198,7 @@ simplyblock_web/static/deploy_cnode.yaml
 simplyblock_web/static/deploy_spdk.yaml
 simplyblock_web/static/is_up.py
 simplyblock_web/static/list_deps.py
+simplyblock_web/static/openapi.json
 simplyblock_web/static/rpac.yaml
 simplyblock_web/static/tst.py
 simplyblock_web/templates/caching_deploy_spdk.yaml.j2

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30}/sbcli_dev.egg-info/requires.txt

@@ -24,3 +24,4 @@ flask-openapi3
 jsonschema
 fastapi
 uvicorn
+prometheus_api_client

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30}/simplyblock_cli/cli.py

@@ -5,7 +5,7 @@ import logging
 import sys
 import traceback
 
-from simplyblock_cli.clibase import CLIWrapperBase, range_type, regex_type, size_type
+from simplyblock_cli.clibase import CLIWrapperBase, range_type, size_type
 from simplyblock_core import utils
 
 class CLIWrapper(CLIWrapperBase):
@@ -36,6 +36,7 @@ class CLIWrapper(CLIWrapperBase):
         self.init_storage_node__configure(subparser)
         self.init_storage_node__configure_upgrade(subparser)
         self.init_storage_node__deploy_cleaner(subparser)
+        self.init_storage_node__clean_devices(subparser)
         self.init_storage_node__add_node(subparser)
         self.init_storage_node__delete(subparser)
         self.init_storage_node__remove(subparser)
@@ -51,7 +52,6 @@ class CLIWrapper(CLIWrapperBase):
         if self.developer_mode:
             self.init_storage_node__device_testing_mode(subparser)
         self.init_storage_node__get_device(subparser)
-        self.init_storage_node__reset_device(subparser)
         self.init_storage_node__restart_device(subparser)
         self.init_storage_node__add_device(subparser)
         self.init_storage_node__remove_device(subparser)
@@ -77,6 +77,7 @@ class CLIWrapper(CLIWrapperBase):
             self.init_storage_node__dump_lvstore(subparser)
         if self.developer_mode:
             self.init_storage_node__set(subparser)
+        self.init_storage_node__new_device_from_failed(subparser)
 
 
     def init_storage_node__deploy(self, subparser):
@@ -87,11 +88,15 @@ class CLIWrapper(CLIWrapperBase):
     def init_storage_node__configure(self, subparser):
         subcommand = self.add_sub_command(subparser, 'configure', 'Prepare a configuration file to be used when adding the storage node')
         argument = subcommand.add_argument('--max-lvol', help='Max logical volume per storage node', type=int, dest='max_lvol', required=True)
-        argument = subcommand.add_argument('--max-size', help='Maximum amount of GB to be utilized on this storage node', type=str, dest='max_prov', required=True)
+        argument = subcommand.add_argument('--max-size', help='Maximum amount of GB to be utilized on this storage node', type=str, dest='max_prov', required=False)
         argument = subcommand.add_argument('--nodes-per-socket', help='number of each node to be added per each socket.', type=int, default=1, dest='nodes_per_socket')
         argument = subcommand.add_argument('--sockets-to-use', help='The system socket to use when adding the storage nodes', type=str, default='0', dest='sockets_to_use')
+        argument = subcommand.add_argument('--cores-percentage', help='The percentage of cores to be used for spdk (0-99)', type=range_type(0, 99), default=0, dest='cores_percentage')
         argument = subcommand.add_argument('--pci-allowed', help='Comma separated list of PCI addresses of Nvme devices to use for storage devices.', type=str, default='', dest='pci_allowed', required=False)
         argument = subcommand.add_argument('--pci-blocked', help='Comma separated list of PCI addresses of Nvme devices to not use for storage devices', type=str, default='', dest='pci_blocked', required=False)
+        argument = subcommand.add_argument('--device-model', help='NVMe SSD model string, example: --model PM1628, --device-model and --size-range must be set together', type=str, default='', dest='device_model', required=False)
+        argument = subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987, --device-model and --size-range must be set together', type=str, default='', dest='size_range', required=False)
+        argument = subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions', dest='force', action='store_true')
 
     def init_storage_node__configure_upgrade(self, subparser):
         subcommand = self.add_sub_command(subparser, 'configure-upgrade', 'Upgrade the automated configuration file with new changes of cpu mask or storage devices')
@@ -99,6 +104,10 @@ class CLIWrapper(CLIWrapperBase):
     def init_storage_node__deploy_cleaner(self, subparser):
         subcommand = self.add_sub_command(subparser, 'deploy-cleaner', 'Cleans a previous simplyblock deploy (local run)')
 
+    def init_storage_node__clean_devices(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'clean-devices', 'clean devices stored in /etc/simplyblock/sn_config_file (local run)')
+        argument = subcommand.add_argument('--config-path', help='Config path to read stored nvme devices from', type=str, default='/etc/simplyblock/sn_config_file', dest='config_path', required=False)
+
     def init_storage_node__add_node(self, subparser):
         subcommand = self.add_sub_command(subparser, 'add-node', 'Adds a storage node by its IP address')
         subcommand.add_argument('cluster_id', help='Cluster id', type=str)
@@ -210,13 +219,10 @@ class CLIWrapper(CLIWrapperBase):
         subcommand = self.add_sub_command(subparser, 'get-device', 'Gets storage device by its id')
         subcommand.add_argument('device_id', help='Device id', type=str)
 
-    def init_storage_node__reset_device(self, subparser):
-        subcommand = self.add_sub_command(subparser, 'reset-device', 'Resets a storage device')
-        subcommand.add_argument('device_id', help='Device id', type=str)
-
     def init_storage_node__restart_device(self, subparser):
         subcommand = self.add_sub_command(subparser, 'restart-device', 'Restarts a storage device')
         subcommand.add_argument('device_id', help='Device id', type=str)
+        argument = subcommand.add_argument('--force', help='Force remove', dest='force', action='store_true')
 
     def init_storage_node__add_device(self, subparser):
         subcommand = self.add_sub_command(subparser, 'add-device', 'Adds a new storage device')
@@ -276,6 +282,7 @@ class CLIWrapper(CLIWrapperBase):
         subcommand = self.add_sub_command(subparser, 'restart-jm-device', 'Restarts a journaling device')
         subcommand.add_argument('jm_device_id', help='Journaling device id', type=str)
         argument = subcommand.add_argument('--force', help='Force device remove', dest='force', action='store_true')
+        argument = subcommand.add_argument('--format', help='Format the Alceml device used for JM device', dest='format', action='store_true')
 
     def init_storage_node__send_cluster_map(self, subparser):
         subcommand = self.add_sub_command(subparser, 'send-cluster-map', 'Sends a new cluster map')
@@ -299,6 +306,10 @@ class CLIWrapper(CLIWrapperBase):
         subcommand.add_argument('attr_name', help='attr_name', type=str)
         subcommand.add_argument('attr_value', help='attr_value', type=str)
 
+    def init_storage_node__new_device_from_failed(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'new-device-from-failed', 'Adds a new device to from failed device information')
+        subcommand.add_argument('device_id', help='Device id', type=str)
+
 
     def init_cluster(self):
         subparser = self.add_command('cluster', 'Cluster commands')
@@ -809,6 +820,8 @@ class CLIWrapper(CLIWrapperBase):
                     ret = self.storage_node__configure_upgrade(sub_command, args)
                 elif sub_command in ['deploy-cleaner']:
                     ret = self.storage_node__deploy_cleaner(sub_command, args)
+                elif sub_command in ['clean-devices']:
+                    ret = self.storage_node__clean_devices(sub_command, args)
                 elif sub_command in ['add-node']:
                     if not self.developer_mode:
                         args.jm_percent = 3
@@ -860,8 +873,6 @@ class CLIWrapper(CLIWrapperBase):
                         ret = self.storage_node__device_testing_mode(sub_command, args)
                 elif sub_command in ['get-device']:
                     ret = self.storage_node__get_device(sub_command, args)
-                elif sub_command in ['reset-device']:
-                    ret = self.storage_node__reset_device(sub_command, args)
                 elif sub_command in ['restart-device']:
                     ret = self.storage_node__restart_device(sub_command, args)
                 elif sub_command in ['add-device']:
@@ -924,6 +935,8 @@ class CLIWrapper(CLIWrapperBase):
                         ret = False
                     else:
                         ret = self.storage_node__set(sub_command, args)
+                elif sub_command in ['new-device-from-failed']:
+                    ret = self.storage_node__new_device_from_failed(sub_command, args)
                 else:
                     self.parser.print_help()
 

{sbcli_dev-19.2.27 → sbcli_dev-19.2.30}/simplyblock_cli/clibase.py

@@ -87,8 +87,7 @@ class CLIWrapperBase:
     def storage_node__configure(self, sub_command, args):
         if not args.max_lvol:
             self.parser.error(f"Mandatory argument '--max-lvol' not provided for {sub_command}")
-        if not args.max_prov:
-            self.parser.error(f"Mandatory argument '--max-size' not provided for {sub_command}")
+        max_size = getattr(args, "max_prov") or 0
         sockets_to_use = [0]
         if args.sockets_to_use:
             try:
@@ -101,21 +100,38 @@ class CLIWrapperBase:
             self.parser.error(f"nodes_per_socket {args.nodes_per_socket}must be either 1 or 2")
         if args.pci_allowed and args.pci_blocked:
             self.parser.error("pci-allowed and pci-blocked cannot be both specified")
-        max_prov = utils.parse_size(args.max_prov, assume_unit='G')
+        max_prov = utils.parse_size(max_size, assume_unit='G')
         pci_allowed = []
         pci_blocked = []
         if args.pci_allowed:
             pci_allowed = [str(x) for x in args.pci_allowed.split(',')]
         if args.pci_blocked:
             pci_blocked = [str(x) for x in args.pci_blocked.split(',')]
-
-        return storage_ops.generate_automated_deployment_config(args.max_lvol, max_prov, sockets_to_use,
-                                                                args.nodes_per_socket, pci_allowed, pci_blocked)
+        if (args.device_model and not args.size_range) or (not args.device_model and args.size_range):
+            self.parser.error("device_model and size_range must be set together")
+        use_pci_allowed = bool(args.pci_allowed)
+        use_pci_blocked = bool(args.pci_blocked)
+        use_model_range = bool(args.device_model and args.size_range)
+        if sum([use_pci_allowed, use_pci_blocked, use_model_range]) > 1:
+            self.parser.error(
+                "Options --pci-allowed, --pci-blocked, and "
+                "(--device-model with --size-range) are mutually exclusive; choose only one."
+            )
+        cores_percentage = int(args.cores_percentage)
+
+        return storage_ops.generate_automated_deployment_config(
+            args.max_lvol, max_prov, sockets_to_use,args.nodes_per_socket,
+            pci_allowed, pci_blocked, force=args.force, device_model=args.device_model,
+            size_range=args.size_range, cores_percentage=cores_percentage)
 
     def storage_node__deploy_cleaner(self, sub_command, args):
         storage_ops.deploy_cleaner()
         return True  # remove once CLI changed to exceptions
 
+    def storage_node__clean_devices(self, sub_command, args):
+        storage_ops.clean_devices(args.config_path)
+        return True  # remove once CLI changed to exceptions
+
     def storage_node__add_node(self, sub_command, args):
         cluster_id = args.cluster_id
         node_addr = args.node_addr
@@ -134,26 +150,29 @@ class CLIWrapperBase:
         enable_ha_jm = args.enable_ha_jm
         namespace = args.namespace
         ha_jm_count = args.ha_jm_count
-
-        out = storage_ops.add_node(
-            cluster_id=cluster_id,
-            node_addr=node_addr,
-            iface_name=ifname,
-            data_nics_list=data_nics,
-            max_snap=max_snap,
-            spdk_image=spdk_image,
-            spdk_debug=spdk_debug,
-            small_bufsize=small_bufsize,
-            large_bufsize=large_bufsize,
-            num_partitions_per_dev=num_partitions_per_dev,
-            jm_percent=jm_percent,
-            enable_test_device=enable_test_device,
-            namespace=namespace,
-            enable_ha_jm=enable_ha_jm,
-            id_device_by_nqn=args.id_device_by_nqn,
-            partition_size=args.partition_size,
-            ha_jm_count=ha_jm_count,
-        )
+        try:
+            out = storage_ops.add_node(
+                cluster_id=cluster_id,
+                node_addr=node_addr,
+                iface_name=ifname,
+                data_nics_list=data_nics,
+                max_snap=max_snap,
+                spdk_image=spdk_image,
+                spdk_debug=spdk_debug,
+                small_bufsize=small_bufsize,
+                large_bufsize=large_bufsize,
+                num_partitions_per_dev=num_partitions_per_dev,
+                jm_percent=jm_percent,
+                enable_test_device=enable_test_device,
+                namespace=namespace,
+                enable_ha_jm=enable_ha_jm,
+                id_device_by_nqn=args.id_device_by_nqn,
+                partition_size=args.partition_size,
+                ha_jm_count=ha_jm_count,
+            )
+        except Exception as e:
+            print(e)
+            return False
 
         return out
 
@@ -184,11 +203,15 @@ class CLIWrapperBase:
         large_bufsize = args.large_bufsize
         ssd_pcie = args.ssd_pcie
 
-        return storage_ops.restart_storage_node(
-            node_id, max_lvol, max_snap, max_prov,
-            spdk_image, spdk_debug,
-            small_bufsize, large_bufsize, node_ip=args.node_ip, reattach_volume=reattach_volume, force=args.force,
-            new_ssd_pcie=ssd_pcie, force_lvol_recreate=args.force_lvol_recreate)
+        try:
+            return storage_ops.restart_storage_node(
+                node_id, max_lvol, max_snap, max_prov,
+                spdk_image, spdk_debug,
+                small_bufsize, large_bufsize, node_ip=args.node_ip, reattach_volume=reattach_volume, force=args.force,
+                new_ssd_pcie=ssd_pcie, force_lvol_recreate=args.force_lvol_recreate)
+        except Exception as e:
+            print(e)
+            return False
 
     def storage_node__shutdown(self, sub_command, args):
         return storage_ops.shutdown_storage_node(args.node_id, args.force)
@@ -233,7 +256,7 @@ class CLIWrapperBase:
         return device_controller.reset_storage_device(args.device_id)
 
     def storage_node__restart_device(self, sub_command, args):
-        return device_controller.restart_device(args.device_id)
+        return device_controller.restart_device(args.device_id, args.force)
 
     def storage_node__add_device(self, sub_command, args):
         return device_controller.add_device(args.device_id)
@@ -292,7 +315,7 @@ class CLIWrapperBase:
         return device_controller.remove_jm_device(args.jm_device_id, args.force)
 
     def storage_node__restart_jm_device(self, sub_command, args):
-        return device_controller.restart_jm_device(args.jm_device_id, args.force)
+        return device_controller.restart_jm_device(args.jm_device_id, args.force, args.format)
 
     def storage_node__send_cluster_map(self, sub_command, args):
         node_id = args.node_id
@@ -310,6 +333,9 @@ class CLIWrapperBase:
         node_id = args.node_id
         return storage_ops.dump_lvstore(node_id)
 
+    def storage_node__new_device_from_failed(self, sub_command, args):
+        return device_controller.new_device_from_failed(args.device_id)
+
     def storage_node__set(self, sub_command, args):
         return storage_ops.set_value(args.node_id, args.attr_name, args.attr_value)