sbcli-pre 1.2.4__zip → 1.2.5__zip

sbcli_pre-1.2.5/simplyblock_core/services/health_check_service.py

@@ -0,0 +1,136 @@
+# coding=utf-8
+import logging
+
+import time
+import sys
+from datetime import datetime
+
+
+from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.storage_node import StorageNode
+from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core import constants, kv_store
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+def set_node_health_check(snode, health_check_status):
+    snode = db_controller.get_storage_node_by_id(snode.get_id())
+    if snode.health_check == health_check_status:
+        return
+    old_status = snode.health_check
+    snode.health_check = health_check_status
+    snode.updated_at = str(datetime.now())
+    snode.write_to_db(db_store)
+    storage_events.snode_health_check_change(snode, snode.health_check, old_status, caused_by="monitor")
+
+
+def set_device_health_check(cluster_id, device, health_check_status):
+    if device.health_check == health_check_status:
+        return
+    nodes = db_controller.get_storage_nodes()
+    for node in nodes:
+        if node.nvme_devices:
+            for dev in node.nvme_devices:
+                if dev.get_id() == device.get_id():
+                    old_status = dev.health_check
+                    dev.health_check = health_check_status
+                    node.write_to_db(db_store)
+                    device_events.device_health_check_change(
+                        dev, dev.health_check, old_status, caused_by="monitor")
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_store = kv_store.KVStore()
+db_controller = kv_store.DBController()
+
+logger.info("Starting health check service")
+while True:
+    cluster_id = ""
+    cl = db_controller.get_clusters()
+    if cl:
+        cluster_id = cl[0].get_id()
+
+    snodes = db_controller.get_storage_nodes()
+    if not snodes:
+        logger.error("storage nodes list is empty")
+
+    for snode in snodes:
+        logger.info("Node: %s, status %s", snode.get_id(), snode.status)
+
+        if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+            logger.info(f"Node status is: {snode.status}, skipping")
+            continue
+
+        # 1- check node ping
+        ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+        logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+        # 2- check node API
+        node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+        logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+        if snode.status == StorageNode.STATUS_OFFLINE:
+            set_node_health_check(snode, ping_check & node_api_check)
+            continue
+
+        # 3- check node RPC
+        node_rpc_check = health_controller._check_node_rpc(
+            snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+        logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+        # 4- docker API
+        node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+        logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+        is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+
+        health_check_status = is_node_online
+        if not node_rpc_check:
+            logger.info("Putting all devices to unavailable state because RPC check failed")
+            for dev in snode.nvme_devices:
+                if dev.io_error:
+                    logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
+                    continue
+                set_device_health_check(cluster_id, dev, False)
+        else:
+            logger.info(f"Node device count: {len(snode.nvme_devices)}")
+            node_devices_check = True
+            node_remote_devices_check = True
+
+            for dev in snode.nvme_devices:
+                if dev.io_error:
+                    logger.debug(f"Skipping Device check because of io_error {dev.get_id()}")
+                    continue
+                ret = health_controller.check_device(dev.get_id())
+                set_device_health_check(cluster_id, dev, ret)
+                if dev.status == dev.STATUS_ONLINE:
+                    node_devices_check &= ret
+
+            logger.info(f"Node remote device: {len(snode.remote_devices)}")
+            rpc_client = RPCClient(
+                snode.mgmt_ip, snode.rpc_port,
+                snode.rpc_username, snode.rpc_password,
+                timeout=5, retry=3)
+            for remote_device in snode.remote_devices:
+                ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                if ret:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                else:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                node_remote_devices_check &= bool(ret)
+
+            health_check_status = is_node_online and node_devices_check and node_remote_devices_check
+        set_node_health_check(snode, health_check_status)
+
+    time.sleep(constants.HEALTH_CHECK_INTERVAL_SEC)
+

sbcli_pre-1.2.4/simplyblock_core/services/tasks_runner_restart.py → sbcli_pre-1.2.5/simplyblock_core/services/job_tasks.py

@@ -5,7 +5,7 @@ import sys
 
 
 from simplyblock_core import constants, kv_store, storage_node_ops
-from simplyblock_core.controllers import device_controller, tasks_events
+from simplyblock_core.controllers import device_controller
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 
@@ -15,19 +15,6 @@ from graypy import GELFUDPHandler
 from simplyblock_core.models.storage_node import StorageNode
 
 
-# configure logging
-logger_handler = logging.StreamHandler(stream=sys.stdout)
-logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
-logger = logging.getLogger()
-logger.addHandler(gelf_handler)
-logger.addHandler(logger_handler)
-logger.setLevel(logging.DEBUG)
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-
 def _get_node_unavailable_devices_count(node_id):
     node = db_controller.get_storage_node_by_id(node_id)
     devices = []
@@ -44,20 +31,10 @@ def _get_device(task):
             return dev
 
 
-def _validate_no_task_node_restart(cluster_id, node_id):
-    tasks = db_controller.get_job_tasks(cluster_id)
-    for task in tasks:
-        if task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return False
-    return True
-
-
 def task_runner(task):
-    if task.function_name == JobSchedule.FN_DEV_RESTART:
+    if task.function_name == "device_restart":
         return task_runner_device(task)
-    if task.function_name == JobSchedule.FN_NODE_RESTART:
+    if task.function_name == "node_restart":
         return task_runner_node(task)
 
 
@@ -72,55 +49,38 @@ def task_runner_device(task):
         device_controller.device_set_retries_exhausted(device.get_id(), True)
         return True
 
-    if not _validate_no_task_node_restart(task.cluster_id, task.node_id):
-        task.function_result = "canceled: node restart found"
-        task.status = JobSchedule.STATUS_DONE
-        task.write_to_db(db_controller.kv_store)
-        device_controller.device_set_unavailable(device.get_id())
-        return True
-
-    if task.canceled:
-        task.function_result = "canceled"
-        task.status = JobSchedule.STATUS_DONE
-        task.write_to_db(db_controller.kv_store)
-        return True
-
     node = db_controller.get_storage_node_by_id(task.node_id)
     if node.status != StorageNode.STATUS_ONLINE:
-        logger.error(f"Node is not online: {node.get_id()}, retry")
+        logger.error(f"Node is not online: {node.get_id()} , skipping task: {task.get_id()}")
         task.function_result = "Node is offline"
         task.retry += 1
         task.write_to_db(db_controller.kv_store)
         return False
 
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
-        logger.info(f"Device is online: {device.get_id()}")
-        task.function_result = "Device is online"
+        logger.info(f"Device is online: {device.get_id()}, no restart needed")
+        task.function_result = "skipped because dev is online"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    if device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
-        logger.info(f"Device is not unavailable: {device.get_id()}, {device.status} , stopping task")
-        task.function_result = f"stopped because dev is {device.status}"
+    task.status = JobSchedule.STATUS_RUNNING
+    task.write_to_db(db_controller.kv_store)
+
+    # resetting device
+    logger.info(f"Resetting device {device.get_id()}")
+    device_controller.reset_storage_device(device.get_id())
+    time.sleep(5)
+    device = _get_device(task)
+    if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
+        logger.info(f"Device is online: {device.get_id()}")
+        task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    if task.status != JobSchedule.STATUS_RUNNING:
-        task.status = JobSchedule.STATUS_RUNNING
-        task.write_to_db(db_controller.kv_store)
-        tasks_events.task_updated(task)
-
-    # set device online for the first 3 retries
-    if task.retry < 3:
-        logger.info(f"Set device online {device.get_id()}")
-        device_controller.device_set_online(device.get_id())
-    else:
-        logger.info(f"Restarting device {device.get_id()}")
-        device_controller.restart_device(device.get_id(), force=True)
-
-    # check device status
+    logger.info(f"Restarting device {device.get_id()}")
+    device_controller.restart_device(device.get_id(), force=True)
     time.sleep(5)
     device = _get_device(task)
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
@@ -144,37 +104,22 @@ def task_runner_node(task):
         storage_node_ops.set_node_status(task.node_id, StorageNode.STATUS_UNREACHABLE)
         return True
 
-    if node.status == StorageNode.STATUS_REMOVED:
-        logger.info(f"Node is removed: {task.node_id}, stopping task")
-        task.function_result = f"Node is removed"
+    if _get_node_unavailable_devices_count(node.get_id()) == 0:
+        logger.info(f"Node is online: {node.get_id()}, no restart needed")
+        task.function_result = "skipped because node is online"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
-        logger.info(f"Node is online: {node.get_id()}")
-        task.function_result = "Node is online"
-        task.status = JobSchedule.STATUS_DONE
-        task.write_to_db(db_controller.kv_store)
-        return True
-
-    if task.canceled:
-        task.function_result = "canceled"
-        task.status = JobSchedule.STATUS_DONE
-        task.write_to_db(db_controller.kv_store)
-        return True
-
-    if task.status != JobSchedule.STATUS_RUNNING:
-        task.status = JobSchedule.STATUS_RUNNING
-        task.write_to_db(db_controller.kv_store)
-        tasks_events.task_updated(task)
+    task.status = JobSchedule.STATUS_RUNNING
+    task.write_to_db(db_controller.kv_store)
 
     # shutting down node
     logger.info(f"Shutdown node {node.get_id()}")
     ret = storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
     if ret:
         logger.info(f"Node shutdown succeeded")
-    time.sleep(3)
+    time.sleep(5)
 
     # resetting node
     logger.info(f"Restart node {node.get_id()}")
@@ -182,9 +127,8 @@ def task_runner_node(task):
     if ret:
         logger.info(f"Node restart succeeded")
 
-    time.sleep(5)
-    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
-        logger.info(f"Node is online: {node.get_id()}")
+    if _get_node_unavailable_devices_count(node.get_id()) == 0:
+        logger.info(f"Node is online: {node.get_id()}, no restart needed")
         task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
@@ -195,7 +139,19 @@ def task_runner_node(task):
     return False
 
 
-logger.info("Starting Tasks runner...")
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_controller = kv_store.DBController()
+
+logger.info("Starting Jobs runner...")
 while True:
     time.sleep(3)
     clusters = db_controller.get_clusters()
@@ -203,16 +159,11 @@ while True:
         logger.error("No clusters found!")
     else:
         for cl in clusters:
-            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
+            tasks = db_controller.get_job_tasks(cl.get_id())
             for task in tasks:
                 delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
-                if task.function_name in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART]:
-                    while task.status != JobSchedule.STATUS_DONE:
-                        # get new task object because it could be changed from cancel task
-                        task = db_controller.get_task_by_id(task.get_id())
-                        res = task_runner(task)
-                        if res:
-                            tasks_events.task_updated(task)
-                        else:
-                            time.sleep(delay_seconds)
-                            delay_seconds *= 2
+                while task.status != JobSchedule.STATUS_DONE:
+                    res = task_runner(task)
+                    if res is False:
+                        time.sleep(delay_seconds)
+                        delay_seconds *= 2

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/lvol_monitor.py

@@ -49,7 +49,7 @@ db_controller = kv_store.DBController()
 
 logger.info("Starting LVol monitor...")
 while True:
-    lvols = db_controller.get_lvols()  # pass
+    lvols = db_controller.get_lvols()
     if not lvols:
         logger.error("LVols list is empty")
 

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/lvol_stat_collector.py

@@ -115,7 +115,7 @@ logger.info("Starting stats collector...")
 while True:
 
     pools = db_controller.get_pools()
-    all_lvols = db_controller.get_lvols()  # pass
+    all_lvols = db_controller.get_lvols()
     for pool in pools:
         lvols = []
         for lvol in all_lvols:

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/port_stat_collector.py

@@ -74,3 +74,4 @@ while True:
             update_port_stats(snode, nic, stats)
         else:
             logger.error("Error getting port stats: %s", nic.get_id())
+

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/storage_node_monitor.py

@@ -8,7 +8,7 @@ from datetime import datetime
 
 
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, distr_controller
-from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
+from simplyblock_core.controllers import storage_events, health_controller, device_controller
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
@@ -31,8 +31,8 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController(kv_store=db_store)
 
 
-def get_cluster_target_status(cluster_id):
-    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+def get_cluster_target_status(cluster):
+    snodes = db_controller.get_storage_nodes()
 
     online_nodes = 0
     offline_nodes = 0
@@ -66,8 +66,8 @@ def get_cluster_target_status(cluster_id):
     logger.debug(f"online_devices: {online_devices}")
     logger.debug(f"offline_devices: {offline_devices}")
 
-    # if more than two affected nodes then cluster is suspended
-    if affected_nodes > 2 or offline_nodes > 2:
+    # if more than two affected modes then cluster is suspended
+    if affected_nodes > 2:
         return Cluster.STATUS_SUSPENDED
 
     # if any device goes offline then cluster is degraded
@@ -85,7 +85,7 @@ def update_cluster_status(cluster_id):
     cluster = db_controller.get_cluster_by_id(cluster_id)
 
     if cluster.ha_type == "ha":
-        cluster_target_status = get_cluster_target_status(cluster_id)
+        cluster_target_status = get_cluster_target_status(cluster)
         logger.info(f"Target cluster status {cluster_target_status}, current status: {cluster.status}")
         if cluster.status == cluster_target_status:
             return
@@ -111,53 +111,48 @@ def set_node_online(node):
 def set_node_offline(node):
     if node.status != StorageNode.STATUS_UNREACHABLE:
         storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
-        # add node to auto restart
-        tasks_controller.add_node_to_auto_restart(node)
 
 
 logger.info("Starting node monitor")
 while True:
-    clusters = db_controller.get_clusters()
-    for cluster in clusters:
-        cluster_id = cluster.get_id()
-        # get storage nodes
-        nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
-        for snode in nodes:
-            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
-                logger.info(f"Node status is: {snode.status}, skipping")
-                continue
-
-            logger.info(f"Checking node {snode.hostname}")
-
-            # 1- check node ping
-            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
-            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
-
-            # 2- check node API
-            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
-            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
-
-            # 3- check node RPC
-            node_rpc_check = health_controller._check_node_rpc(
-                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
-
-            # 4- docker API
-            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
-            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
-
-            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
-            if is_node_online:
-                set_node_online(snode)
-            else:
-                set_node_offline(snode)
-
-            if not ping_check and not node_rpc_check:
-                # node is dead, set devices offline
-                for dev in snode.nvme_devices:
-                    device_controller.device_set_unavailable(dev.get_id())
-
-        update_cluster_status(cluster_id)
+    # get storage nodes
+    nodes = db_controller.get_storage_nodes()
+    for snode in nodes:
+        if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+            logger.info(f"Node status is: {snode.status}, skipping")
+            continue
+
+        logger.info(f"Checking node {snode.hostname}")
+
+        # 1- check node ping
+        ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+        logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+        # 2- check node API
+        node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+        logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+        # 3- check node RPC
+        node_rpc_check = health_controller._check_node_rpc(
+            snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+        logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+        # 4- docker API
+        node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+        logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+        is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+        if is_node_online:
+            set_node_online(snode)
+        else:
+            set_node_offline(snode)
+
+        if not ping_check and not node_rpc_check:
+            # node is dead, set devices offline
+            for dev in snode.nvme_devices:
+                device_controller.device_set_unavailable(dev.get_id())
+
+        update_cluster_status(snode.cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")
     time.sleep(constants.NODE_MONITOR_INTERVAL_SEC)

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/snode_client.py

@@ -95,15 +95,3 @@ class SNodeClient:
 
     def leave_swarm(self):
         return self._request("GET", "leave_swarm")
-
-    def make_gpt_partitions(self, nbd_device, jm_percent, num_partitions):
-        params = {
-            "nbd_device": nbd_device,
-            "jm_percent": jm_percent,
-            "num_partitions": num_partitions,
-        }
-        return self._request("POST", "make_gpt_partitions", params)
-
-    def delete_dev_gpt_partitions(self, device_pci):
-        params = {"device_pci": device_pci}
-        return self._request("POST", "delete_dev_gpt_partitions", params)