sbcli-dev 5.3.1__tar.gz → 6.0.0__tar.gz

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 5.3.1
+Version: 6.0.0
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=5.3.1
+SIMPLY_BLOCK_VERSION=6.0.0
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main
 DOCKER_USER=hamdysimplyblock

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 5.3.1
+Version: 6.0.0
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/sbcli_dev.egg-info/SOURCES.txt

@@ -66,12 +66,15 @@ simplyblock_core/scripts/dashboard.yml
 simplyblock_core/scripts/datasource.yml
 simplyblock_core/scripts/db_config_double.sh
 simplyblock_core/scripts/db_config_single.sh
+simplyblock_core/scripts/deploy_fdb.sh
 simplyblock_core/scripts/deploy_stack.sh
 simplyblock_core/scripts/docker-compose-swarm-monitoring.yml
 simplyblock_core/scripts/docker-compose-swarm.yml
+simplyblock_core/scripts/foundation.yml
 simplyblock_core/scripts/haproxy.cfg
 simplyblock_core/scripts/install_deps.sh
 simplyblock_core/scripts/objstore.yml
+simplyblock_core/scripts/prepare_fdb.sh
 simplyblock_core/scripts/prometheus.yml
 simplyblock_core/scripts/run_ssh.sh
 simplyblock_core/scripts/set_db_config.sh

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_cli/cli.py

@@ -46,7 +46,7 @@ class CLIWrapper:
         sub_command.add_argument("--max-lvol", help='Max lvol per storage node', dest='max_lvol', type=int)
         sub_command.add_argument("--max-snap", help='Max snapshot per storage node', dest='max_snap', type=int, default=500)
         sub_command.add_argument("--max-prov", help='Maximum amount of GB to be provisioned via all storage nodes', dest='max_prov')
-        sub_command.add_argument("--number-of-distribs", help='The number of distirbs to be created on the node', dest='number_of_distribs', type=int, default=4)
+        sub_command.add_argument("--number-of-distribs", help='The number of distirbs to be created on the node', dest='number_of_distribs', type=int, default=2)
         sub_command.add_argument("--number-of-devices", help='Number of devices per storage node if it\'s not supported EC2 instance', dest='number_of_devices', type=int)
         sub_command.add_argument("--cpu-mask", help='SPDK app CPU mask, default is all cores found', dest='spdk_cpu_mask')
 
@@ -57,6 +57,7 @@ class CLIWrapper:
         sub_command.add_argument("--iobuf_large_bufsize", help='bdev_set_options param', dest='large_bufsize',  type=int, default=0)
         sub_command.add_argument("--enable-test-device", help='Enable creation of test device', action='store_true')
         sub_command.add_argument("--disable-ha-jm", help='Disable HA JM for distrib creation', action='store_false', dest='enable_ha_jm', default=True)
+        sub_command.add_argument("--is-secondary-node", help='add as secondary node', action='store_true', dest='is_secondary_node', default=False)
         sub_command.add_argument("--namespace", help='k8s namespace to deploy on',)
 
 
@@ -124,6 +125,7 @@ class CLIWrapper:
         sub_command.add_argument("node_id", help='Node ID')
         sub_command.add_argument("--history", help='list history records -one for every 15 minutes- '
                                                    'for XX days and YY hours -up to 10 days in total-, format: XXdYYh')
+        sub_command.add_argument("--records", help='Number of records, default: 20', type=int, default=20)
 
         sub_command = self.add_sub_command(
             subparser, 'get-capacity', 'Get node capacity statistics')
@@ -206,6 +208,7 @@ class CLIWrapper:
         sub_command.add_argument("device_id", help='Storage device ID')
         sub_command.add_argument("--history", help='list history records -one for every 15 minutes- '
                                                    'for XX days and YY hours -up to 10 days in total-, format: XXdYYh')
+        sub_command.add_argument("--records", help='Number of records, default: 20', type=int, default=20)
 
         sub_command = self.add_sub_command(subparser, 'port-list', 'Get Data interfaces list for a node')
         sub_command.add_argument("node_id", help='Storage node ID')
@@ -292,13 +295,15 @@ class CLIWrapper:
         sub_command.add_argument("--distr-chunk-bs", help='(Dev) distrb bdev chunk block size, default: 4096', type=int,
                                  default=4096)
         sub_command.add_argument("--ha-type", help='LVol HA type (single, ha), default is cluster HA type',
-                                 dest='ha_type', choices=["single", "ha", "default"], default='single')
+                                 dest='ha_type', choices=["single", "ha"], default='single')
         sub_command.add_argument("--enable-node-affinity", help='Enable node affinity for storage nodes', action='store_true')
         sub_command.add_argument("--qpair-count", help='tcp transport qpair count', type=int, dest='qpair_count',
-                                 default=6, choices=range(128))
+                                 default=20, choices=range(128))
         sub_command.add_argument("--max-queue-size", help='The max size the queue will grow', type=int, default=128)
         sub_command.add_argument("--inflight-io-threshold", help='The number of inflight IOs allowed before the IO queuing starts', type=int, default=4)
         sub_command.add_argument("--enable-qos", help='Enable qos bdev for storage nodes', action='store_true', dest='enable_qos')
+        sub_command.add_argument("--strict-node-anti-affinity", help='Enable strict node anti affinity for storage nodes', action='store_true')
+
 
 
         # add cluster
@@ -320,13 +325,14 @@ class CLIWrapper:
         sub_command.add_argument("--distr-chunk-bs", help='(Dev) distrb bdev chunk block size, default: 4096', type=int,
                                  default=4096)
         sub_command.add_argument("--ha-type", help='LVol HA type (single, ha), default is cluster HA type',
-                                 dest='ha_type', choices=["single", "ha", "default"], default='default')
+                                 dest='ha_type', choices=["single", "ha"], default='single')
         sub_command.add_argument("--enable-node-affinity", help='Enable node affinity for storage nodes', action='store_true')
         sub_command.add_argument("--qpair-count", help='tcp transport qpair count', type=int, dest='qpair_count',
                                  default=6, choices=range(128))
         sub_command.add_argument("--max-queue-size", help='The max size the queue will grow', type=int, default=128)
         sub_command.add_argument("--inflight-io-threshold", help='The number of inflight IOs allowed before the IO queuing starts', type=int, default=4)
         sub_command.add_argument("--enable-qos", help='Enable qos bdev for storage nodes', action='store_true', dest='enable_qos')
+        sub_command.add_argument("--strict-node-anti-affinity", help='Enable strict node anti affinity for storage nodes', action='store_true')
 
         # Activate cluster
         sub_command = self.add_sub_command(subparser, 'activate', 'Create distribs and raid0 bdevs on all the storage node and move the cluster to active state')
@@ -530,6 +536,7 @@ class CLIWrapper:
         sub_command.add_argument("id", help='LVol id')
         sub_command.add_argument("--history", help='(XXdYYh), list history records (one for every 15 minutes) '
                                                    'for XX days and YY hours (up to 10 days in total).')
+        sub_command.add_argument("--records", help='Number of records, default: 20', type=int, default=20)
 
         sub_command = self.add_sub_command(subparser, "check", 'Health check LVol')
         sub_command.add_argument("id", help='UUID of LVol')
@@ -624,6 +631,7 @@ class CLIWrapper:
         sub_command.add_argument("id", help='Pool id')
         sub_command.add_argument("--history", help='(XXdYYh), list history records (one for every 15 minutes) '
                                                    'for XX days and YY hours (up to 10 days in total).')
+        sub_command.add_argument("--records", help='Number of records, default: 20', type=int, default=20)
 
         subparser = self.add_command('snapshot', 'Snapshot commands')
 
@@ -686,6 +694,9 @@ class CLIWrapper:
         sub_command.add_argument("--history", help='(XXdYYh), list history records (one for every 15 minutes) '
                                                    'for XX days and YY hours (up to 10 days in total).')
 
+        self.parser.add_argument("--cmd", help='cmd', nargs = '+')
+
+
 
     def init_parser(self):
         self.parser = argparse.ArgumentParser(prog=constants.SIMPLY_BLOCK_CLI_NAME, description='SimplyBlock management CLI')
@@ -709,6 +720,13 @@ class CLIWrapper:
             self.logger.setLevel(logging.INFO)
         logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
 
+        if  args.cmd:
+            cmd = args.cmd
+            func = cmd[0]
+            if func == "deploy-fdb":
+                cluster_ops.open_db_from_zip(" ".join(cmd[1:]))
+            return
+
         args_dict = args.__dict__
         ret = ""
         if args.command in ['storage-node', 'sn']:
@@ -773,7 +791,8 @@ class CLIWrapper:
                     enable_test_device=enable_test_device,
                     namespace=namespace,
                     number_of_distribs=number_of_distribs,
-                    enable_ha_jm=enable_ha_jm
+                    enable_ha_jm=enable_ha_jm,
+                    is_secondary_node=args.is_secondary_node,
                 )
 
                 return out
@@ -854,7 +873,8 @@ class CLIWrapper:
             elif sub_command == "get-io-stats-device":
                 device_id = args.device_id
                 history = args.history
-                data = device_controller.get_device_iostats(device_id, history)
+                records = args.records
+                data = device_controller.get_device_iostats(device_id, history, records_count=records)
                 if data:
                     ret = utils.print_table(data)
                 else:
@@ -872,7 +892,8 @@ class CLIWrapper:
             elif sub_command == "get-io-stats":
                 node_id = args.node_id
                 history = args.history
-                data = storage_ops.get_node_iostats_history(node_id, history)
+                records = args.records
+                data = storage_ops.get_node_iostats_history(node_id, history, records_count=records)
 
                 if data:
                     ret = utils.print_table(data)
@@ -1070,7 +1091,8 @@ class CLIWrapper:
             elif sub_command == "get-io-stats":
                 id = args.id
                 history = args.history
-                data = lvol_controller.get_io_stats(id, history)
+                records = args.records
+                data = lvol_controller.get_io_stats(id, history, records_count=records)
                 if data:
                     ret = utils.print_table(data)
                 else:
@@ -1079,6 +1101,10 @@ class CLIWrapper:
                 id = args.id
                 history = args.history
                 ret = lvol_controller.get_capacity(id, history)
+                if ret:
+                    ret = utils.print_table(ret)
+                else:
+                    return False
             elif sub_command == "check":
                 id = args.id
                 ret = health_controller.check_lvol(id)
@@ -1160,7 +1186,7 @@ class CLIWrapper:
                 ret = pool_controller.get_capacity(args.pool_id)
 
             elif sub_command == "get-io-stats":
-                ret = pool_controller.get_io_stats(args.id, args.history)
+                ret = pool_controller.get_io_stats(args.id, args.history, args.records)
 
             else:
                 self.parser.print_help()
@@ -1269,11 +1295,13 @@ class CLIWrapper:
         max_queue_size = args.max_queue_size
         inflight_io_threshold = args.inflight_io_threshold
         enable_qos = args.enable_qos
+        strict_node_anti_affinity = args.strict_node_anti_affinity
+
 
         return cluster_ops.add_cluster(
             blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
             distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity,
-            qpair_count, max_queue_size, inflight_io_threshold, enable_qos)
+            qpair_count, max_queue_size, inflight_io_threshold, enable_qos, strict_node_anti_affinity)
 
 
     def cluster_create(self, args):
@@ -1299,6 +1327,7 @@ class CLIWrapper:
         max_queue_size = args.max_queue_size
         inflight_io_threshold = args.inflight_io_threshold
         enable_qos = args.enable_qos
+        strict_node_anti_affinity = args.strict_node_anti_affinity
 
 
         return cluster_ops.create_cluster(
@@ -1306,7 +1335,7 @@ class CLIWrapper:
             CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
             ifname, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint,
             distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity,
-            qpair_count, max_queue_size, inflight_io_threshold, enable_qos)
+            qpair_count, max_queue_size, inflight_io_threshold, enable_qos, strict_node_anti_affinity)
 
 
     def query_yes_no(self, question, default="yes"):

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_core/cluster_ops.py

@@ -14,7 +14,8 @@ import requests
 from jinja2 import Environment, FileSystemLoader
 
 from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops, distr_controller
-from simplyblock_core.controllers import cluster_events, device_controller, storage_events
+from simplyblock_core.controllers import cluster_events, device_controller, storage_events, pool_controller, \
+    lvol_controller
 from simplyblock_core.kv_store import DBController, KVStore
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.rpc_client import RPCClient
@@ -103,7 +104,7 @@ def _add_graylog_input(cluster_ip, password):
 def create_cluster(blk_size, page_size_in_blocks, cli_pass,
                    cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period,
                    contact_point, grafana_endpoint, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type,
-                   enable_node_affinity, qpair_count, max_queue_size, inflight_io_threshold, enable_qos):
+                   enable_node_affinity, qpair_count, max_queue_size, inflight_io_threshold, enable_qos, strict_node_anti_affinity):
 
     logger.info("Installing dependencies...")
     ret = scripts.install_deps()
@@ -171,11 +172,12 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     else:
         c.grafana_endpoint = f"http://{DEV_IP}/grafana"
     c.enable_node_affinity = enable_node_affinity
-    c.qpair_count = qpair_count or 256
+    c.qpair_count = qpair_count or 6
 
     c.max_queue_size = max_queue_size
     c.inflight_io_threshold = inflight_io_threshold
     c.enable_qos = enable_qos
+    c.strict_node_anti_affinity = strict_node_anti_affinity
 
     alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")
     alert_resources_file = "alert_resources.yaml"
@@ -295,7 +297,7 @@ def deploy_spdk(node_docker, spdk_cpu_mask, spdk_mem):
 
 def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
                 distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity, qpair_count,
-                max_queue_size, inflight_io_threshold, enable_qos):
+                max_queue_size, inflight_io_threshold, enable_qos, strict_node_anti_affinity):
     db_controller = DBController()
     clusters = db_controller.get_clusters()
     if not clusters:
@@ -314,6 +316,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn
     cluster.secret = utils.generate_string(20)
     cluster.db_connection = default_cluster.db_connection
     cluster.grafana_endpoint = default_cluster.grafana_endpoint
+    cluster.strict_node_anti_affinity = strict_node_anti_affinity
 
     _create_update_user(cluster.uuid, cluster.grafana_endpoint, default_cluster.secret, cluster.secret)
 
@@ -327,7 +330,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn
     cluster.distr_chunk_bs = distr_chunk_bs
     cluster.ha_type = ha_type
     cluster.enable_node_affinity = enable_node_affinity
-    cluster.qpair_count = qpair_count or 256
+    cluster.qpair_count = qpair_count or 6
     cluster.max_queue_size = max_queue_size
     cluster.inflight_io_threshold = inflight_io_threshold
     cluster.enable_qos = enable_qos
@@ -360,11 +363,13 @@ def cluster_activate(cl_id, force=False):
             logger.warning(f"Failed to activate cluster, Cluster is in an ACTIVE state, use --force to reactivate")
             return False
     set_cluster_status(cl_id, Cluster.STATUS_IN_ACTIVATION)
-    snodes = db_controller.get_storage_nodes_by_cluster_id(cl_id)
+    snodes = db_controller.get_primary_storage_nodes_by_cluster_id(cl_id)
     online_nodes = []
     dev_count = 0
 
     for node in snodes:
+        if node.is_secondary_node:
+            continue
         if node.status == node.STATUS_ONLINE:
             online_nodes.append(node)
             for dev in node.nvme_devices:
@@ -378,7 +383,21 @@ def cluster_activate(cl_id, force=False):
     records = db_controller.get_cluster_capacity(cluster)
     max_size = records[0]['size_total']
 
+    if cluster.ha_type == "ha":
+        for snode in snodes:
+            if snode.is_secondary_node or snode.secondary_node_id:
+                continue
+            secondary_nodes = storage_node_ops.get_secondary_nodes(snode)
+            if not secondary_nodes:
+                logger.error(f"Failed to activate cluster, No enough secondary nodes")
+                set_cluster_status(cl_id, Cluster.STATUS_UNREADY)
+                return False
+            snode.secondary_node_id = secondary_nodes[0]
+            snode.write_to_db()
+
     for snode in snodes:
+        if snode.is_secondary_node:
+            continue
         if snode.lvstore:
             logger.warning(f"Node {snode.get_id()} already has lvstore {snode.lvstore}")
             ret = storage_node_ops.recreate_lvstore(snode)
@@ -391,6 +410,8 @@ def cluster_activate(cl_id, force=False):
             return False
     if not cluster.cluster_max_size:
         cluster.cluster_max_size = max_size
+        cluster.cluster_max_devices = dev_count
+        cluster.cluster_max_nodes = len(online_nodes)
     cluster.write_to_db(db_controller.kv_store)
     set_cluster_status(cl_id, Cluster.STATUS_ACTIVE)
     logger.info("Cluster activated successfully")
@@ -660,12 +681,16 @@ def get_logs(cluster_id, is_json=False):
         if 'vuid' in record.object_dict:
             vuid = record.object_dict['vuid']
 
+        msg =  record.message
+        if record.event in ["device_status", "node_status"]:
+            msg = msg+f" ({record.count})"
+
         out.append({
             "Date": record.get_date_string(),
             "NodeId": record.node_id,
             "Event": record.event,
             "Level": record.event_level,
-            "Message": record.message,
+            "Message":msg,
             "Storage_ID": str(Storage_ID),
             "VUID": str(vuid),
             "Status": record.status,
@@ -713,14 +738,11 @@ def update_cluster(cl_id):
     except Exception as e:
         print(e)
 
-    # for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
-    #     node_docker = docker.DockerClient(base_url=f"tcp://{node.mgmt_ip}:2375", version="auto")
-    #     logger.info(f"Pulling image {constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}")
-    #     node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
-    #     if node.status == StorageNode.STATUS_ONLINE:
-    #         storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
-    #         time.sleep(3)
-    #     storage_node_ops.restart_storage_node(node.get_id())
+    for node in db_controller.get_storage_nodes_by_cluster_id(cl_id):
+        try:
+            storage_node_ops.start_storage_node_api_container(node.mgmt_ip)
+        except:
+            pass
 
     logger.info("Done")
     return True
@@ -787,3 +809,42 @@ def delete_cluster(cl_id):
     cluster_events.cluster_delete(cluster)
     cluster.remove(db_controller.kv_store)
     logger.info("Done")
+
+def open_db_from_zip(fip_path):
+    import boto3
+    s3 = boto3.client('s3')
+
+
+    out = '/tmp/fdb.zip'
+    try:
+        os.remove(out)
+    except:
+        pass
+
+    buket_name = 'simplyblock-e2e-test-logs'
+    file_name = ""
+    if fip_path.startswith('s3://'):
+        # s3://simplyblock-e2e-test-logs/12220160320/mgmt/fdb.zip
+        buket_name = fip_path.split("/")[2]
+        file_name = "/".join(fip_path.split("/")[3:])
+
+
+    elif len(fip_path.split('/'))<=3:
+        # /12220160320/mgmt/fdb.zip
+        file_name = fip_path
+
+    elif fip_path.startswith('https://'):
+        #https://simplyblock-e2e-test-logs.s3.us-east-2.amazonaws.com/12220160320/mgmt/fdb.zip
+        buket_name = fip_path.split("/")[2]
+        buket_name = buket_name.split(".")[0]
+        file_name = "/".join(fip_path.split("/")[3:])
+    else:
+        file_name = fip_path
+
+    try:
+        ret = s3.download_file(buket_name, file_name, out)
+    except Exception as e:
+        logger.error(e)
+
+    if os.path.exists(out):
+        scripts.deploy_fdb_from_file_service(out)

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_core/constants.py

@@ -46,7 +46,7 @@ weights = {
 }
 
 
-HEALTH_CHECK_INTERVAL_SEC = 10
+HEALTH_CHECK_INTERVAL_SEC = 30
 
 GRAYLOG_CHECK_INTERVAL_SEC = 60
 
@@ -67,7 +67,7 @@ GELF_PORT = 12202
 MIN_HUGE_PAGE_MEMORY_FOR_LVOL = 209715200
 MIN_SYS_MEMORY_FOR_LVOL = 524288000
 EXTRA_SMALL_POOL_COUNT = 1024
-EXTRA_LARGE_POOL_COUNT = 128
+EXTRA_LARGE_POOL_COUNT = 16000
 # EXTRA_SMALL_POOL_COUNT = 0
 # EXTRA_LARGE_POOL_COUNT = 0
 EXTRA_HUGE_PAGE_MEMORY = 2147483648

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_core/controllers/device_controller.py

@@ -24,13 +24,15 @@ def device_set_state(device_id, state):
         logger.error("node not found")
         return False
 
+    device = None
     for dev in snode.nvme_devices:
         if dev.get_id() == device_id:
             device = dev
             break
 
-    if device.status == state:
-        return True
+    if not device:
+        logger.error("device not found")
+        return False
 
     if state == NVMeDevice.STATUS_ONLINE:
         device.retries_exhausted = False
@@ -38,26 +40,26 @@ def device_set_state(device_id, state):
     if state == NVMeDevice.STATUS_REMOVED:
         device.deleted = True
 
-    old_status = dev.status
-    device.status = state
-    snode.write_to_db(db_controller.kv_store)
+    if state == NVMeDevice.STATUS_READONLY and device.status == NVMeDevice.STATUS_UNAVAILABLE:
+        return False
+
+    if device.status != state:
+        old_status = dev.status
+        device.status = state
+        snode.write_to_db(db_controller.kv_store)
+        device_events.device_status_change(device, device.status, old_status)
 
-    distr_controller.send_cluster_map_to_node(snode)
+    distr_controller.send_dev_status_event(device, device.status)
 
     if state == NVMeDevice.STATUS_ONLINE:
-        snode = db_controller.get_storage_node_by_id(dev.node_id)
         logger.info("Make other nodes connect to the node devices")
         snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
         for node in snodes:
             if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
             node.remote_devices = storage_node_ops._connect_to_remote_devs(node)
-            if node.enable_ha_jm:
-                node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
             node.write_to_db()
-            distr_controller.send_cluster_map_to_node(node)
 
-    device_events.device_status_change(device, device.status, old_status)
     return True
 
 
@@ -553,6 +555,7 @@ def device_set_retries_exhausted(device_id, retries_exhausted):
     dev = db_controller.get_storage_devices(device_id)
     if not dev:
         logger.error("device not found")
+        return False
 
     snode = db_controller.get_storage_node_by_id(dev.node_id)
     if not snode:
@@ -737,16 +740,22 @@ def set_jm_device_state(device_id, state):
         jm_device.status = state
         snode.write_to_db(db_controller.kv_store)
 
-    if snode.enable_ha_jm and state in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]:
+    if snode.enable_ha_jm and state == NVMeDevice.STATUS_ONLINE:
+        rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5)
+        jm_bdev = f"jm_{snode.get_id()}"
+        subsystem_nqn = snode.subsystem + ":dev:" + jm_bdev
+
+        for iface in snode.data_nics:
+            if iface.ip4_address:
+                ret = rpc_client.nvmf_subsystem_listener_set_ana_state(
+                    subsystem_nqn, iface.ip4_address, "4420", True)
+                break
+
         # make other nodes connect to the new devices
         snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
         for node_index, node in enumerate(snodes):
             if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
                 continue
-            for jm_dev in node.remote_jm_devices:
-                if jm_dev.get_id() == device_id and jm_device.status != JMDevice.STATUS_ONLINE:
-                    node.remote_jm_devices.remove(jm_dev)
-                    break
             logger.info(f"Connecting to node: {node.get_id()}")
             node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
             node.write_to_db(db_controller.kv_store)
@@ -848,32 +857,32 @@ def restart_jm_device(device_id, force=False, format_alceml=False):
                     snode.write_to_db(db_controller.kv_store)
                     set_jm_device_state(snode.jm_device.get_id(), JMDevice.STATUS_ONLINE)
         else:
-            nvme_bdev = jm_device.nvme_bdev
-            if snode.enable_test_device:
-                ret = rpc_client.bdev_passtest_create(jm_device.testing_bdev, jm_device.nvme_bdev)
-                if not ret:
-                    logger.error(f"Failed to create passtest bdev {jm_device.testing_bdev}")
-                    return False
-                nvme_bdev = jm_device.testing_bdev
-            alceml_cpu_mask = ""
-            alceml_worker_cpu_mask = ""
-
-            if snode.alceml_cpu_cores:
-                alceml_cpu_mask = utils.decimal_to_hex_power_of_2(snode.alceml_cpu_cores[snode.alceml_cpu_index])
-                snode.alceml_cpu_index = (snode.alceml_cpu_index + 1) % len(snode.alceml_cpu_cores)
-
-            if snode.alceml_worker_cpu_cores:
-                alceml_worker_cpu_mask = utils.decimal_to_hex_power_of_2(
-                    snode.alceml_worker_cpu_cores[snode.alceml_worker_cpu_index])
-                snode.alceml_worker_cpu_index = (snode.alceml_worker_cpu_index + 1) % len(snode.alceml_worker_cpu_cores)
-
-            ret = rpc_client.bdev_alceml_create(jm_device.alceml_bdev, nvme_bdev, jm_device.get_id(),
-                                                pba_init_mode=1, alceml_cpu_mask=alceml_cpu_mask,
-                                                alceml_worker_cpu_mask=alceml_worker_cpu_mask)
-
-            if not ret:
-                logger.error(f"Failed to create alceml bdev: {jm_device.alceml_bdev}")
-                return False
+            # nvme_bdev = jm_device.nvme_bdev
+            # if snode.enable_test_device:
+            #     ret = rpc_client.bdev_passtest_create(jm_device.testing_bdev, jm_device.nvme_bdev)
+            #     if not ret:
+            #         logger.error(f"Failed to create passtest bdev {jm_device.testing_bdev}")
+            #         # return False
+            #     nvme_bdev = jm_device.testing_bdev
+            # alceml_cpu_mask = ""
+            # alceml_worker_cpu_mask = ""
+            #
+            # if snode.alceml_cpu_cores:
+            #     alceml_cpu_mask = utils.decimal_to_hex_power_of_2(snode.alceml_cpu_cores[snode.alceml_cpu_index])
+            #     snode.alceml_cpu_index = (snode.alceml_cpu_index + 1) % len(snode.alceml_cpu_cores)
+            #
+            # if snode.alceml_worker_cpu_cores:
+            #     alceml_worker_cpu_mask = utils.decimal_to_hex_power_of_2(
+            #         snode.alceml_worker_cpu_cores[snode.alceml_worker_cpu_index])
+            #     snode.alceml_worker_cpu_index = (snode.alceml_worker_cpu_index + 1) % len(snode.alceml_worker_cpu_cores)
+            #
+            # ret = rpc_client.bdev_alceml_create(jm_device.alceml_bdev, nvme_bdev, jm_device.get_id(),
+            #                                     pba_init_mode=1, alceml_cpu_mask=alceml_cpu_mask,
+            #                                     alceml_worker_cpu_mask=alceml_worker_cpu_mask)
+            #
+            # if not ret:
+            #     logger.error(f"Failed to create alceml bdev: {jm_device.alceml_bdev}")
+            #     return False
 
             jm_bdev = f"jm_{snode.get_id()}"
             ret = rpc_client.bdev_jm_create(jm_bdev, jm_device.alceml_bdev, jm_cpu_mask=snode.jm_cpu_mask)

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_core/controllers/device_events.py

@@ -37,7 +37,7 @@ def device_status_change(device, new_state, old_status, caused_by=ec.CAUSED_BY_C
 
 
 def device_restarted(device, caused_by=ec.CAUSED_BY_CLI):
-    _device_event(device, f"Device restarted", caused_by, ec.EVENT_STATUS_CHANGE)
+    _device_event(device, f"Device restarted, status: {device.status}", caused_by, ec.EVENT_STATUS_CHANGE)
 
 
 def device_reset(device, caused_by=ec.CAUSED_BY_CLI):

{sbcli_dev-5.3.1 → sbcli_dev-6.0.0}/simplyblock_core/controllers/events_controller.py

@@ -62,7 +62,7 @@ def log_distr_event(cluster_id, node_id, event_dict):
 
     db_controller = DBController()
     ds.write_to_db(db_controller.kv_store)
-    return ds.get_id()
+    return ds
 
 
 def log_event_cluster(cluster_id, domain, event, db_object, caused_by, message,