sbcli-pre 1.2.5__zip → 1.2.7__zip

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/controllers/device_controller.py

@@ -1,8 +1,8 @@
 import time
 import logging
 
-from simplyblock_core import distr_controller, utils
-from simplyblock_core.controllers import device_events, lvol_controller
+from simplyblock_core import distr_controller, utils, storage_node_ops
+from simplyblock_core.controllers import device_events, lvol_controller, tasks_controller
 from simplyblock_core.kv_store import DBController
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.rpc_client import RPCClient
@@ -16,6 +16,7 @@ def device_set_state(device_id, state):
     dev = db_controller.get_storage_devices(device_id)
     if not dev:
         logger.error("device not found")
+        return False
 
     snode = db_controller.get_storage_node_by_id(dev.node_id)
     if not snode:
@@ -35,7 +36,7 @@ def device_set_state(device_id, state):
 
     old_status = dev.status
     device.status = state
-    distr_controller.send_dev_status_event(device.cluster_device_order, device.status)
+    distr_controller.send_dev_status_event(device, device.status)
     snode.write_to_db(db_controller.kv_store)
     device_events.device_status_change(device, device.status, old_status)
     return True
@@ -78,14 +79,20 @@ def device_set_read_only(device_id):
 
 
 def device_set_online(device_id):
-    return device_set_state(device_id, NVMeDevice.STATUS_ONLINE)
+    ret = device_set_state(device_id, NVMeDevice.STATUS_ONLINE)
+    if ret:
+        logger.info("Adding task to device data migration")
+        task_id = tasks_controller.add_device_mig_task(device_id)
+        if task_id:
+            logger.info(f"Task id: {task_id}")
+    return ret
 
 
 def get_alceml_name(alceml_id):
     return f"alceml_{alceml_id}"
 
 
-def _def_create_device_stack(device_obj, snode):
+def _def_create_device_stack(device_obj, snode, force=False):
 
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
@@ -98,22 +105,26 @@ def _def_create_device_stack(device_obj, snode):
     ret = rpc_client.bdev_passtest_create(test_name, device_obj.nvme_bdev)
     if not ret:
         logger.error(f"Failed to create bdev: {test_name}")
-        return False
+        if not force:
+            return False
 
     alceml_id = device_obj.get_id()
     alceml_name = get_alceml_name(alceml_id)
     logger.info(f"adding {alceml_name}")
-    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=2)
+    ret = rpc_client.bdev_alceml_create(alceml_name, test_name, alceml_id, pba_init_mode=2,
+                                        dev_cpu_mask=snode.dev_cpu_mask)
     if not ret:
         logger.error(f"Failed to create alceml bdev: {alceml_name}")
-        return False
+        if not force:
+            return False
 
     # add pass through
     pt_name = f"{alceml_name}_PT"
     ret = rpc_client.bdev_PT_NoExcl_create(pt_name, alceml_name)
     if not ret:
         logger.error(f"Failed to create pt noexcl bdev: {pt_name}")
-        return False
+        if not force:
+            return False
 
     subsystem_nqn = snode.subsystem + ":dev:" + alceml_id
     logger.info("Creating subsystem %s", subsystem_nqn)
@@ -137,11 +148,13 @@ def _def_create_device_stack(device_obj, snode):
     logger.info(f"Adding {pt_name} to the subsystem")
     ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
 
-    if device_obj.jm_bdev:
-        ret = rpc_client.bdev_jm_create(device_obj.jm_bdev, device_obj.alceml_bdev)
+    if hasattr(device_obj, 'jm_bdev') and device_obj.jm_bdev:
+        ret = rpc_client.bdev_jm_create(device_obj.jm_bdev, device_obj.alceml_bdev,
+                                        dev_cpu_mask=snode.dev_cpu_mask)
         if not ret:
-            logger.error(f"Failed to create bdev: {device_obj.jm_bdev}")
-            return False
+            logger.error(f"Failed to create jm bdev: {device_obj.jm_bdev}")
+            if not force:
+                return False
 
     device_obj.testing_bdev = test_name
     device_obj.alceml_bdev = alceml_name
@@ -174,23 +187,15 @@ def restart_device(device_id, force=False):
             device_obj = dev
             break
 
-    device_obj.status = 'restarting'
-    snode.write_to_db(db_controller.kv_store)
-
     logger.info(f"Restarting device {device_id}")
+    device_set_unavailable(device_id)
 
-    ret = _def_create_device_stack(device_obj, snode)
+    ret = _def_create_device_stack(device_obj, snode, force=force)
 
     if not ret:
         logger.error("Failed to create device stack")
-        device_obj.status = NVMeDevice.STATUS_UNAVAILABLE
-        snode.write_to_db(db_controller.kv_store)
-        return False
-
-    device_obj.io_error = False
-    device_obj.retries_exhausted = False
-    device_obj.status = NVMeDevice.STATUS_ONLINE
-    snode.write_to_db(db_controller.kv_store)
+        if not force:
+            return False
 
     logger.info("Make other nodes connect to the device")
     snodes = db_controller.get_storage_nodes()
@@ -221,10 +226,11 @@ def restart_device(device_id, force=False):
         node.write_to_db(db_controller.kv_store)
         time.sleep(3)
 
-    logger.info("Sending device event")
-    distr_controller.send_dev_status_event(device_obj.cluster_device_order, "online")
+    logger.info("Setting device io_error to False")
+    device_set_io_error(device_id, False)
+    logger.info("Setting device online")
+    device_set_online(device_id)
     device_events.device_restarted(device_obj)
-
     return "Done"
 
 
@@ -267,15 +273,8 @@ def device_remove(device_id, force=True):
             device = dev
             break
 
-    if device.jm_bdev:
-        if snode.lvols:
-            logger.error(f"Failed to remove device: {device.get_id()}, "
-                         f"there are LVols that uses JM from this device, delete LVol to continue")
-            # if not force:
-            return False
-
     logger.info("Sending device event")
-    distr_controller.send_dev_status_event(device.cluster_device_order, "removed")
+    distr_controller.send_dev_status_event(device, NVMeDevice.STATUS_REMOVED)
 
     logger.info("Disconnecting device from all nodes")
     distr_controller.disconnect_device(device)
@@ -291,13 +290,6 @@ def device_remove(device_id, force=True):
         if not force:
             return False
 
-    if device.jm_bdev:
-        ret = rpc_client.bdev_jm_delete(f"jm_{snode.get_id()}")
-        if not ret:
-            logger.error(f"Failed to remove journal manager: jm_{snode.get_id()}")
-            if not force:
-                return False
-
     logger.info("Removing device bdevs")
     ret = rpc_client.bdev_PT_NoExcl_delete(f"{device.alceml_bdev}_PT")
     if not ret:
@@ -405,19 +397,12 @@ def get_device_iostats(device_id, history, records_count=20, parse_sizes=True):
 
 def reset_storage_device(dev_id):
     db_controller = DBController()
-    device = None
-    snode = None
-    for node in db_controller.get_storage_nodes():
-        for dev in node.nvme_devices:
-            if dev.get_id() == dev_id:
-                device = dev
-                snode = node
-                break
-
+    device = db_controller.get_storage_devices(dev_id)
     if not device:
         logger.error(f"Device not found: {dev_id}")
         return False
 
+    snode = db_controller.get_storage_node_by_id(device.node_id)
     if not snode:
         logger.error(f"Node not found {device.node_id}")
         return False
@@ -426,30 +411,38 @@ def reset_storage_device(dev_id):
         logger.error(f"Device status: {device.status} is removed")
         return False
 
-    logger.info("Setting device to unavailable")
-    old_status = device.status
-    device.status = NVMeDevice.STATUS_UNAVAILABLE
-    distr_controller.send_dev_status_event(device.cluster_device_order, device.status)
-    snode.write_to_db(db_controller.kv_store)
-    device_events.device_status_change(device, device.status, old_status)
+    logger.info("Setting devices to unavailable")
+    device_set_unavailable(dev_id)
+    devs = []
+    for dev in snode.nvme_devices:
+        if dev.get_id() == device.get_id():
+            continue
+        if dev.status == NVMeDevice.STATUS_ONLINE and dev.physical_label == device.physical_label:
+            devs.append(dev)
+            device_set_unavailable(dev.get_id())
 
     logger.info("Resetting device")
     rpc_client = RPCClient(
         snode.mgmt_ip, snode.rpc_port,
         snode.rpc_username, snode.rpc_password)
 
-    controller_name = device.nvme_bdev[:-2]
+    controller_name = device.nvme_controller
     response = rpc_client.reset_device(controller_name)
     if not response:
         logger.error(f"Failed to reset NVMe BDev {controller_name}")
         return False
+    time.sleep(3)
 
-    device.io_error = False
-    device.retries_exhausted = False
-    snode.write_to_db(db_controller.kv_store)
+    logger.info("Setting devices online")
+    for dev in devs:
+        device_set_online(dev.get_id())
 
-    device_events.device_reset(device)
+    # set io_error flag False
+    device_set_io_error(dev_id, False)
+    device_set_retries_exhausted(dev_id, False)
+    # set device to online
     device_set_online(dev_id)
+    device_events.device_reset(device)
     return True
 
 

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/controllers/events_controller.py

@@ -40,7 +40,7 @@ def log_distr_event(cluster_id, node_id, event_dict):
     ds.uuid = str(uuid.uuid4())
     ds.cluster_uuid = cluster_id
     ds.node_id = node_id
-    ds.date = int(time.time())
+    ds.date = round(time.time()*1000)
     ds.domain = DOMAIN_DISTR
     ds.event_level = EventObj.LEVEL_ERROR
     ds.caused_by = CAUSED_BY_MONITOR
@@ -66,7 +66,7 @@ def log_distr_event(cluster_id, node_id, event_dict):
 
 
 def log_event_cluster(cluster_id, domain, event, db_object, caused_by, message,
-                      node_id=None, event_level=EventObj.LEVEL_INFO):
+                      node_id=None, event_level=EventObj.LEVEL_INFO, status=None):
     """
     uuid:
     cluster_uuid: 1234
@@ -83,7 +83,7 @@ def log_event_cluster(cluster_id, domain, event, db_object, caused_by, message,
     ds = EventObj()
     ds.uuid = str(uuid.uuid4())
     ds.cluster_uuid = cluster_id
-    ds.date = int(time.time())
+    ds.date = round(time.time()*1000)
     ds.node_id = node_id
     ds.event_level = event_level
 
@@ -93,12 +93,14 @@ def log_event_cluster(cluster_id, domain, event, db_object, caused_by, message,
     ds.object_dict = db_object.get_clean_dict()
     ds.caused_by = caused_by
     ds.message = message
+    ds.status = status
 
     log_event_based_on_level(cluster_id, event, db_object.name, message, caused_by, event_level)
 
     db_controller = DBController()
     ds.write_to_db(db_controller.kv_store)
 
+
 def log_event_based_on_level(cluster_id, event, db_object, message, caused_by, event_level):
     json_str = json.dumps({
         "cluster_id": cluster_id,

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/controllers/health_controller.py

@@ -16,11 +16,13 @@ logger = log.getLogger()
 
 def check_cluster(cluster_id):
     db_controller = DBController()
-    st = db_controller.get_storage_nodes()
+    st = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     data = []
+    result = True
     for node in st:
         # check if node is online, unavailable, restarting
         ret = check_node(node.get_id(), with_devices=False)
+        result &= ret
         print("*"*100)
         data.append({
             "Kind": "Node",
@@ -28,30 +30,32 @@ def check_cluster(cluster_id):
             "Status": "ok" if ret else "failed"
         })
 
-    for device in db_controller.get_storage_devices():
-        ret = check_device(device.get_id())
-        print("*" * 100)
-        data.append({
-            "Kind": "Device",
-            "UUID": device.get_id(),
-            "Status": "ok" if ret else "failed"
-        })
-
-    for lvol in db_controller.get_lvols():
-        ret = check_lvol(lvol.get_id())
-        print("*" * 100)
-        data.append({
-            "Kind": "LVol",
-            "UUID": lvol.get_id(),
-            "Status": "ok" if ret else "failed"
-        })
+        for device in node.nvme_devices:
+            ret = check_device(device.get_id())
+            result &= ret
+            print("*" * 100)
+            data.append({
+                "Kind": "Device",
+                "UUID": device.get_id(),
+                "Status": "ok" if ret else "failed"
+            })
+
+        for lvol in db_controller.get_lvols(cluster_id):
+            ret = check_lvol(lvol.get_id())
+            result &= ret
+            print("*" * 100)
+            data.append({
+                "Kind": "LVol",
+                "UUID": lvol.get_id(),
+                "Status": "ok" if ret else "failed"
+            })
     print(utils.print_table(data))
-    return True
+    return result
 
 
 def _check_node_docker_api(ip):
     try:
-        node_docker = docker.DockerClient(base_url=f"tcp://{ip}:2375", version="auto")
+        node_docker = docker.DockerClient(base_url=f"tcp://{ip}:2375", version="auto", timeout=3)
         ret = node_docker.info()
         if ret:
             logger.debug(ret)
@@ -65,7 +69,7 @@ def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password):
     try:
         rpc_client = RPCClient(
             rpc_ip, rpc_port, rpc_username, rpc_password,
-            timeout=5, retry=3)
+            timeout=10, retry=1)
         ret = rpc_client.get_version()
         if ret:
             logger.debug(f"SPDK version: {ret['version']}")
@@ -167,7 +171,7 @@ def check_node(node_id, with_devices=True):
 
 def check_device(device_id):
     db_controller = DBController()
-    device = db_controller.get_storage_devices(device_id)
+    device = db_controller.get_storage_device_by_id(device_id)
     if not device:
         logger.error("device not found")
         return False
@@ -192,8 +196,8 @@ def check_device(device_id):
             snode.rpc_username, snode.rpc_password)
 
         bdevs_stack = [device.nvme_bdev, device.testing_bdev, device.alceml_bdev, device.pt_bdev]
-        if device.jm_bdev:
-            bdevs_stack.append(device.jm_bdev)
+        # if device.jm_bdev:
+        #     bdevs_stack.append(device.jm_bdev)
         logger.info(f"Checking Device: {device_id}, status:{device.status}")
         problems = 0
         for bdev in bdevs_stack:
@@ -231,7 +235,7 @@ def check_device(device_id):
 
 def check_remote_device(device_id):
     db_controller = DBController()
-    device = db_controller.get_storage_devices(device_id)
+    device = db_controller.get_storage_device_by_id(device_id)
     if not device:
         logger.error("device not found")
         return False
@@ -241,7 +245,7 @@ def check_remote_device(device_id):
         return False
 
     result = True
-    for node in db_controller.get_storage_nodes():
+    for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id):
         if node.status == StorageNode.STATUS_ONLINE:
             if node.get_id() == snode.get_id():
                 continue
@@ -306,20 +310,6 @@ def check_lvol_on_node(lvol_id, node_id):
         logger.exception(e)
         return False
 
-    # check ndcs+npcs <= online devices
-    # then change its status to offline if fails this check
-    online_devices = 0
-    for node in db_controller.get_storage_nodes():
-        for dev in node.nvme_devices:
-            if dev.status == dev.STATUS_ONLINE:
-                online_devices += 1
-
-    # if lvol.ndcs + lvol.npcs < online_devices:
-    #     logger.info(f"Checking Distr ndcs+npcs: {lvol.ndcs}+{lvol.npcs}, online devices: {online_devices} ... ok")
-    # else:
-    #     logger.info(f"Checking Distr ndcs+npcs: {lvol.ndcs}+{lvol.npcs}, online devices: {online_devices} ... failed")
-        # passed = False
-
     return passed