sbcli-dev 3.9.2__zip → 3.9.4__zip

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.2
+Version: 3.9.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.9.2
+SIMPLY_BLOCK_VERSION=3.9.4
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.2
+Version: 3.9.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/sbcli_dev.egg-info/SOURCES.txt

@@ -90,7 +90,6 @@ simplyblock_core/services/caching_node_monitor.py
 simplyblock_core/services/cap_monitor.py
 simplyblock_core/services/capacity_and_stats_collector.py
 simplyblock_core/services/device_monitor.py
-simplyblock_core/services/distr_event_collector.py
 simplyblock_core/services/health_check_service.py
 simplyblock_core/services/install_service.sh
 simplyblock_core/services/log_agg_service.py

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/cluster_ops.py

@@ -170,7 +170,10 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     c.distr_bs = distr_bs
     c.distr_chunk_bs = distr_chunk_bs
     c.ha_type = ha_type
-    c.grafana_endpoint = grafana_endpoint
+    if grafana_endpoint:
+        c.grafana_endpoint = grafana_endpoint
+    else:
+        c.grafana_endpoint = f"http://{DEV_IP}/grafana"
     c.enable_node_affinity = enable_node_affinity
     
     alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/controllers/lvol_controller.py

@@ -165,25 +165,6 @@ def validate_add_lvol_func(name, size, host_id_or_name, pool_id_or_name,
     return True, ""
 
 
-def get_jm_names(snode):
-    return [snode.jm_device.jm_bdev] if snode.jm_device else []
-
-
-def get_ha_jm_names(current_node, snode_list):
-    jm_list = []
-    if current_node.jm_device:
-        jm_list.append(current_node.jm_device.jm_bdev)
-    else:
-        jm_list.append("JM_LOCAL")
-
-    for node in snode_list:
-        if node.get_id() == current_node.get_id():
-            continue
-        name = f"remote_{node.jm_device.jm_bdev}n1"
-        jm_list.append(name)
-    return jm_list[:3]
-
-
 def _get_next_3_nodes(cluster_id, lvol_size=0):
     snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
     # online_nodes = []
@@ -205,7 +186,7 @@ def _get_next_3_nodes(cluster_id, lvol_size=0):
             # node_stat_list = db_controller.get_node_stats(node, limit=1000)
             # combined_record = utils.sum_records(node_stat_list)
             node_st = {
-                "lvol": len(node.lvols) or 1,
+                "lvol": len(node.lvols)+1,
                 # "cpu": 1 + (node.cpu * node.cpu_hz),
                 # "r_io": combined_record.read_io_ps,
                 # "w_io": combined_record.write_io_ps,
@@ -365,6 +346,11 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp,
         logger.error("Storage nodes are less than 3 in ha cluster")
         return False, "Storage nodes are less than 3 in ha cluster"
 
+    if host_node and host_node.status != StorageNode.STATUS_ONLINE:
+        mgs = f"Storage node is not online. ID: {host_node.get_id()} status: {host_node.status}"
+        logger.error(mgs)
+        return False, mgs
+
     cluster_size_prov_util = int(((cluster_size_prov+size) / cluster_size_total) * 100)
 
     if cl.prov_cap_crit and cl.prov_cap_crit < cluster_size_prov_util:

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/distr_controller.py

@@ -27,14 +27,13 @@ def send_node_status_event(node, node_status):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=10)
         ret = rpc_client.distr_status_events_update(events)
 
 
 def send_dev_status_event(device, dev_status):
     db_controller = DBController()
     storage_ID = device.cluster_device_order
-    logging.info(f"Sending event updates, device: {storage_ID}, status: {dev_status}")
     node_status_event = {
         "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
         "event_type": "device_status",
@@ -46,8 +45,8 @@ def send_dev_status_event(device, dev_status):
     for node in snodes:
         if node.status != node.STATUS_ONLINE:
             continue
-        logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+        logging.debug(f"Sending event updates, device: {storage_ID}, status: {dev_status}, node: {node.get_id()}")
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=1)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -82,7 +81,6 @@ def get_distr_cluster_map(snodes, target_node):
         dev_w_map = []
         node_w = 0
         for i, dev in enumerate(snode.nvme_devices):
-            logger.debug(f"Device: {dev.get_id()}, status: {dev.status}")
             if dev.status in [NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW]:
                 continue
             dev_w = int(dev.size/(1024*1024*1024)) or 1
@@ -100,6 +98,8 @@ def get_distr_cluster_map(snodes, target_node):
                         break
             if not name:
                 name = f"remote_{dev.alceml_bdev}n1"
+                dev_status = NVMeDevice.STATUS_UNAVAILABLE
+            logger.debug(f"Device: {dev.get_id()}, status: {dev_status}, bdev_name: {name}")
             dev_map[dev.cluster_device_order] = {
                 "UUID": dev.get_id(),
                 "bdev_name": name,
@@ -194,7 +194,7 @@ def parse_distr_cluster_map(map_string):
 def send_cluster_map_to_node(node):
     db_controller = DBController()
     snodes = db_controller.get_storage_nodes_by_cluster_id(node.cluster_id)
-    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+    rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=10)
     cluster_map_data = get_distr_cluster_map(snodes, node)
     cluster_map_data['UUID_node_target'] = node.get_id()
     ret = rpc_client.distr_send_cluster_map(cluster_map_data)
@@ -212,7 +212,7 @@ def send_cluster_map_add_node(snode):
         if node.status != node.STATUS_ONLINE:
             continue
         logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5)
 
         cluster_map_data = get_distr_cluster_map([snode], node)
         cl_map = {

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/kv_store.py

@@ -316,3 +316,9 @@ class DBController:
         for dev in snode.nvme_devices:
             total_node_capacity += dev.size
         return total_node_capacity
+
+    def get_jm_device_by_id(self, jm_id):
+        for node in self.get_storage_nodes():
+            if node.jm_device and node.jm_device.get_id() == jm_id:
+                return node.jm_device
+        return None

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/models/base_model.py

@@ -101,7 +101,10 @@ class BaseModel(object):
             return objects[0]
         return None
 
-    def write_to_db(self, kv_store):
+    def write_to_db(self, kv_store=None):
+        if not kv_store:
+            from simplyblock_core.kv_store import KVStore
+            kv_store = KVStore()
         try:
             prefix = "%s/%s/%s" % (self.object_type, self.name, self.get_id())
             st = json.dumps(self.to_dict())

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/services/health_check_service.py

@@ -55,6 +55,9 @@ while True:
 
             if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
                 logger.info(f"Node status is: {snode.status}, skipping")
+                set_node_health_check(snode, False)
+                for dev in snode.nvme_devices:
+                    set_device_health_check(cluster_id, dev, False)
                 continue
 
             # 1- check node ping
@@ -65,10 +68,6 @@ while True:
             node_api_check = health_controller._check_node_api(snode.mgmt_ip)
             logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
-            if snode.status == StorageNode.STATUS_OFFLINE:
-                set_node_health_check(snode, ping_check & node_api_check)
-                continue
-
             # 3- check node RPC
             node_rpc_check = health_controller._check_node_rpc(
                 snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
@@ -117,7 +116,7 @@ while True:
 
                 if snode.jm_device:
                     jm_device = snode.jm_device
-                    logger.info(f"Node node jm: {jm_device}")
+                    logger.info(f"Node JM: {jm_device}")
                     ret = health_controller.check_jm_device(jm_device.get_id())
                     if ret:
                         logger.info(f"Checking jm bdev: {jm_device.jm_bdev} ... ok")
@@ -133,7 +132,7 @@ while True:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                         else:
                             logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                        node_remote_devices_check &= bool(ret)
+                        # node_remote_devices_check &= bool(ret)
 
                 lvstore_check = True
                 if snode.lvstore and snode.lvstore_stack:

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/services/main_distr_event_collector.py

@@ -56,7 +56,7 @@ def process_device_event(event):
                     new_remote_devices.append(rem_dev)
             node.remote_devices = new_remote_devices
             node.write_to_db(db_controller.kv_store)
-            distr_controller.send_cluster_map_to_node(node)
+            device_controller.device_set_unavailable(device_id)
 
         else:
             if event.message == 'SPDK_BDEV_EVENT_REMOVE':

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_core/storage_node_ops.py

@@ -781,37 +781,45 @@ def _connect_to_remote_devs(this_node):
     return remote_devices
 
 
-def _connect_to_remote_jm_devs(this_node):
+def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
     db_controller = DBController()
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
         this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
 
+    node_bdevs = rpc_client.get_bdevs()
+    node_bdev_names = [b['name'] for b in node_bdevs]
     remote_devices = []
-    # connect to remote devs
-    snodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id)
-    for node_index, node in enumerate(snodes):
-        if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
-            continue
-
-        if node.jm_device and node.jm_device.status == NVMeDevice.STATUS_ONLINE:
-            logger.info(f"Connecting to JM on node: {node.get_id()}")
-            name = f"remote_{node.jm_device.jm_bdev}"
-            bdev_name = f"{name}n1"
-            ret = rpc_client.get_bdevs(bdev_name)
+    if jm_ids:
+        for jm_id in jm_ids:
+            jm_dev = db_controller.get_jm_device_by_id(jm_id)
+            remote_devices.append(jm_dev)
+    elif len(this_node.remote_jm_devices) > 0:
+        remote_devices = this_node.remote_jm_devices
+    else:
+        for node in db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id):
+            if node.get_id() == this_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
+                continue
+            if node.jm_device:
+                remote_devices.append(node.jm_device)
+
+    for jm_dev in remote_devices:
+        name = f"remote_{jm_dev.jm_bdev}"
+        bdev_name = f"{name}n1"
+        jm_dev.remote_bdev = bdev_name
+        if bdev_name in node_bdev_names:
+            logger.info(f"bdev found {bdev_name}")
+            jm_dev.status = JMDevice.STATUS_ONLINE
+        else:
+            logger.info(f"Connecting {this_node.get_id()} to {name}")
+            ret = rpc_client.bdev_nvme_attach_controller_tcp(
+                name, jm_dev.nvmf_nqn, jm_dev.nvmf_ip, jm_dev.nvmf_port)
             if ret:
-                logger.info(f"bdev found {bdev_name}")
+                jm_dev.status = JMDevice.STATUS_ONLINE
             else:
-                logger.info(f"Connecting {this_node.get_id()} to {name}")
-                ret = rpc_client.bdev_nvme_attach_controller_tcp(
-                    name, node.jm_device.nvmf_nqn, node.jm_device.nvmf_ip, node.jm_device.nvmf_port)
-                if not ret:
-                    logger.error(f"failed to connect to remote JM {node.jm_device.jm_bdev}")
-                    continue
-
-            node.jm_device.remote_bdev = bdev_name
-            remote_devices.append(node.jm_device)
+                logger.error(f"failed to connect to remote JM {name}")
+                jm_dev.status = JMDevice.STATUS_UNAVAILABLE
 
     return remote_devices
 
@@ -1209,7 +1217,8 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
             else:
                 node.remote_devices.append(dev)
 
-        if node.enable_ha_jm:
+        if snode.jm_device and node.enable_ha_jm:
+            node.remote_jm_devices.append(snode.jm_device)
             node.remote_jm_devices = _connect_to_remote_jm_devs(node)
         node.write_to_db(kv_store)
         logger.info(f"connected to devices count: {len(node.remote_devices)}")
@@ -2540,28 +2549,32 @@ def recreate_lvstore(snode):
     return True
 
 
-def get_next_ha_jm_names(current_node):
+def get_next_ha_jms(current_node):
     db_controller = DBController(KVStore())
-    jm_list = []
-    if current_node.jm_device:
-        jm_list.append(current_node.jm_device.jm_bdev)
-    else:
-        jm_list.append("JM_LOCAL")
-
     jm_count = {}
     for node in db_controller.get_storage_nodes_by_cluster_id(current_node.cluster_id):
         if node.get_id() == current_node.get_id() or node.status != StorageNode.STATUS_ONLINE:
             continue
         if node.jm_device:
-            jm_count[node.jm_device.jm_bdev] = 1 + jm_count.get(node.jm_device.jm_bdev, 0)
+            jm_count[node.jm_device.get_id()] = 1 + jm_count.get(node.jm_device.get_id(), 0)
         for rem_jm_device in node.remote_jm_devices:
-            jm_count[rem_jm_device.jm_bdev] = 1 + jm_count.get(rem_jm_device.jm_bdev, 0)
+            if rem_jm_device.get_id() != current_node.jm_device.get_id():
+                jm_count[rem_jm_device.get_id()] = 1 + jm_count.get(rem_jm_device.get_id(), 0)
 
     jm_count = dict(sorted(jm_count.items(), key=lambda x: x[1]))
+    return list(jm_count.keys())[:2]
+
 
-    jm_list.append(f"remote_{list(jm_count.keys())[0].jm_bdev}n1")
-    jm_list.append(f"remote_{list(jm_count.keys())[1].jm_bdev}n1")
+def get_node_jm_names(current_node):
+    jm_list = []
+    if current_node.jm_device:
+        jm_list.append(current_node.jm_device.jm_bdev)
+    else:
+        jm_list.append("JM_LOCAL")
 
+    if current_node.enable_ha_jm:
+        for jm_dev in current_node.remote_jm_devices[:2]:
+            jm_list.append(f"remote_{jm_dev.jm_bdev}n1")
     return jm_list
 
 
@@ -2575,13 +2588,10 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
     strip_size_kb = utils.nearest_upper_power_of_2(strip_size_kb)
     jm_vuid = 0
     if snode.enable_ha_jm:
-        if len(nodes) > 3:
-            nodes = nodes[:3]
-        jm_names = lvol_controller.get_ha_jm_names(snode, nodes)
         jm_vuid = utils.get_random_vuid()
-    else:
-        jm_names = lvol_controller.get_jm_names(snode)
-
+        online_jms = get_next_ha_jms(snode)
+        snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, online_jms)
+        snode.write_to_db()
 
     for _ in range(snode.number_of_distribs):
         distrib_vuid = utils.get_random_vuid()
@@ -2594,7 +2604,6 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
                     "name": distrib_name,
                     "params": {
                         "name": distrib_name,
-                        "jm_names": jm_names,
                         "jm_vuid": jm_vuid,
                         "vuid": distrib_vuid,
                         "ndcs": ndcs,
@@ -2622,14 +2631,6 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
                 },
                 "distribs_list": distrib_list
             },
-            # {
-            #     "type": "bdev_ptnonexcl",
-            #     "name": "raid_PT",
-            #     "params": {
-            #         "name": "raid_PT",
-            #         "base_bdev_name": raid_device
-            #     }
-            # },
             {
                 "type": "bdev_lvstore",
                 "name": lvs_name,
@@ -2651,13 +2652,11 @@ def create_lvstore(snode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blo
         logger.error(err)
         return False
 
-
     snode.lvstore = lvs_name
     snode.lvstore_stack = lvstore_stack
     snode.raid = raid_device
-    db_controller = DBController(KVStore())
-    snode.write_to_db(db_controller.kv_store)
-    time.sleep(3)
+    snode.write_to_db()
+    time.sleep(1)
     return True
 
 
@@ -2677,6 +2676,7 @@ def _create_bdev_stack(snode, lvstore_stack=None):
         params = bdev['params']
 
         if type == "bdev_distr":
+            params['jm_names'] = get_node_jm_names(snode)
             if snode.distrib_cpu_cores:
                 distrib_cpu_mask = utils.decimal_to_hex_power_of_2(snode.distrib_cpu_cores[snode.distrib_cpu_index])
                 params['distrib_cpu_mask'] = distrib_cpu_mask
@@ -2693,26 +2693,6 @@ def _create_bdev_stack(snode, lvstore_stack=None):
 
         elif type == "bdev_ptnonexcl":
             ret = rpc_client.bdev_PT_NoExcl_create(**params)
-            if ret:
-                try:
-                    # add pass through
-                    pt_name = "raid_PT"
-                    subsystem_nqn = snode.subsystem + ":dev:raid"
-                    logger.info("creating raid subsystem %s", subsystem_nqn)
-                    ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', 'sbcli-cn')
-                    for iface in snode.data_nics:
-                        if iface.ip4_address:
-                            tr_type = iface.get_transport_type()
-                            logger.info("adding listener for %s on IP %s" % (subsystem_nqn, iface.ip4_address))
-                            ret = rpc_client.listeners_create(subsystem_nqn, tr_type, iface.ip4_address, "4420")
-                            break
-                    logger.info(f"add {pt_name} to subsystem")
-                    ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name)
-                    if not ret:
-                        logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}")
-                        return False
-                except:
-                    pass
 
         elif type == "bdev_raid":
             distribs_list = bdev["distribs_list"]

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_web/blueprints/web_api_lvol.py

@@ -230,7 +230,7 @@ def resize_lvol(uuid):
 
     cl_data = request.get_json()
     if 'size' not in cl_data:
-        return utils.get_response(None, "missing required param: new_size", 400)
+        return utils.get_response(None, "missing required param: size", 400)
 
     new_size = utils.parse_size(cl_data['size'])
 

{sbcli_dev-3.9.2 → sbcli_dev-3.9.4}/simplyblock_web/templates/storage_deploy_spdk.yaml.j2

@@ -90,9 +90,9 @@ spec:
         resources:
           limits:
             hugepages-2Mi: {{ MEM_GEGA }}Gi
-            memory: 10Gi
+            memory: 4Gi
           requests:
-            memory: 10Gi
+            memory: 4Gi
 
       - name: spdk-proxy-container
         image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
@@ -110,33 +110,3 @@ spec:
           value: "{{ RPC_USERNAME }}"
         - name: RPC_PASSWORD
           value: "{{ RPC_PASSWORD }}"
-
-{#      - name: node-exporter#}
-{#        image: prom/node-exporter:v1.7.0#}
-{#        user: root#}
-{#        command:#}
-{#          - '--path.procfs=/host/proc'#}
-{#          - '--path.sysfs=/host/sys'#}
-{#          - '--path.rootfs=/host'#}
-{#          - '--collector.filesystem.ignored-mount-points="^(/rootfs|/host|)/(sys|proc|dev|host|etc)($$|/)"'#}
-{#          - '--collector.filesystem.ignored-fs-types="^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue)(fs|)$$"'#}
-{#          - '--no-collector.ipvs'#}
-{#        volumeMounts:#}
-{#        - name: host-sys#}
-{#          mountPath: /host/sys#}
-{#          readOnly: true#}
-{#        - name: host-proc#}
-{#          mountPath: /host/proc#}
-{#          readOnly: true#}
-{#        - name: host-rootfs#}
-{#          mountPath: /rootfs#}
-{#          readOnly: true#}
-
-      - name: distr-event-collector
-        image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
-        imagePullPolicy: "Always"
-        command: ["python", "simplyblock_core/services/distr_event_collector.py"]
-        volumeMounts:
-        - name: foundationdb
-          mountPath: /etc/foundationdb
-          readOnly: true

sbcli_dev-3.9.2/simplyblock_core/services/distr_event_collector.py

@@ -1,157 +0,0 @@
-# coding=utf-8
-import time
-
-
-from simplyblock_core import constants, kv_store, utils, rpc_client
-from simplyblock_core.controllers import events_controller, device_controller, lvol_events
-from simplyblock_core.models.lvol_model import LVol
-
-
-from simplyblock_core.models.nvme_device import NVMeDevice
-from simplyblock_core.rpc_client import RPCClient
-
-
-logger = utils.get_logger(__name__)
-
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-
-def process_device_event(event):
-    if event.message in ['SPDK_BDEV_EVENT_REMOVE', "error_open", 'error_read', "error_write", "error_unmap"]:
-        node_id = event.node_id
-        storage_id = event.storage_id
-
-        device = None
-        for node in db_controller.get_storage_nodes():
-            for dev in node.nvme_devices:
-                if dev.cluster_device_order == storage_id:
-                    if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
-                        logger.info(f"The storage device is not online, skipping. status: {dev.status}")
-                        # event.status = 'skipped'
-                        # return
-                    device = dev
-                    break
-
-        if not device:
-            logger.info(f"Device not found!, storage id: {storage_id} from node: {node_id}")
-            event.status = 'device_not_found'
-            return
-
-        device_id = device.get_id()
-        if event.message == 'SPDK_BDEV_EVENT_REMOVE':
-            if device.node_id == node_id:
-                logger.info(f"Removing storage id: {storage_id} from node: {node_id}")
-                device_controller.device_remove(device_id)
-            else:
-                logger.info(f"Removing remote storage id: {storage_id} from node: {node_id}")
-                new_remote_devices = []
-                device_node = db_controller.get_storage_node_by_id(node_id)
-                rpc_client = RPCClient(device_node.mgmt_ip, device_node.rpc_port,
-                                       device_node.rpc_username, device_node.rpc_password)
-                for rem_dev in device_node.remote_devices:
-                    if rem_dev.get_id() == device.get_id():
-                        ctrl_name = rem_dev.remote_bdev[:-2]
-                        rpc_client.bdev_nvme_detach_controller(ctrl_name)
-                    else:
-                        new_remote_devices.append(rem_dev)
-                device_node.remote_devices = new_remote_devices
-                device_node.write_to_db(db_controller.kv_store)
-
-        elif event.message in ['error_write', 'error_unmap']:
-            logger.info(f"Setting device to read-only")
-            device_controller.device_set_io_error(device_id, True)
-            device_controller.device_set_read_only(device_id)
-        else:
-            logger.info(f"Setting device to unavailable")
-            device_controller.device_set_io_error(device_id, True)
-            device_controller.device_set_unavailable(device_id)
-
-        event.status = 'processed'
-
-
-def process_lvol_event(event):
-    if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
-        # vuid = event.object_dict['vuid']
-        node_id = event.node_id
-        lvols = []
-        for lv in db_controller.get_lvols():  # pass
-            if lv.node_id == node_id:
-                lvols.append(lv)
-
-        if not lvols:
-            logger.error(f"LVols on node {node_id} not found")
-            event.status = 'lvols_not_found'
-        else:
-            for lvol in lvols:
-                if lvol.status == LVol.STATUS_ONLINE:
-                    logger.info("Setting LVol to offline")
-                    lvol.io_error = True
-                    old_status = lvol.status
-                    lvol.status = LVol.STATUS_OFFLINE
-                    lvol.write_to_db(db_controller.kv_store)
-                    lvol_events.lvol_status_change(lvol, lvol.status, old_status, caused_by="monitor")
-                    lvol_events.lvol_io_error_change(lvol, True, False, caused_by="monitor")
-            event.status = 'processed'
-    else:
-        logger.error(f"Unknown LVol event message: {event.message}")
-        event.status = "event_unknown"
-
-
-def process_event(event_id):
-    event = db_controller.get_events(event_id)[0]
-    if event.event == "device_status":
-        if event.storage_id >= 0:
-            process_device_event(event)
-
-        if event.vuid >= 0:
-            process_lvol_event(event)
-
-    event.write_to_db(db_controller.kv_store)
-
-
-hostname = utils.get_hostname()
-logger.info("Getting node info...")
-while True:
-    time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
-
-    snode = db_controller.get_storage_node_by_hostname(hostname)
-    if not snode:
-        logger.error("This node is not part of the cluster, hostname: %s, Retrying..." % hostname)
-        continue
-
-    logger.info(f"Starting Distr event collector on node: {hostname}")
-
-    client = rpc_client.RPCClient(
-        snode.mgmt_ip,
-        snode.rpc_port,
-        snode.rpc_username,
-        snode.rpc_password,
-        timeout=10, retry=2)
-
-    try:
-        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
-
-        if not events:
-            logger.debug("no events found")
-            continue
-
-        logger.info(f"Found events: {len(events)}")
-        event_ids = []
-        for ev in events:
-            logger.debug(ev)
-            ev_id = events_controller.log_distr_event(snode.cluster_id, snode.get_id(), ev)
-            event_ids.append(ev_id)
-
-        for eid in event_ids:
-            logger.info(f"Processing event: {eid}")
-            process_event(eid)
-
-        logger.info(f"Discarding events: {len(events)}")
-        client.distr_status_events_discard_then_get(len(events), 0)
-
-    except Exception as e:
-        logger.error("Failed to process distr events")
-        logger.exception(e)
-        continue