sbcli-pre 1.2.5__zip → 1.2.7__zip

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/storage_node_monitor.py

@@ -8,7 +8,7 @@ from datetime import datetime
 
 
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, distr_controller
-from simplyblock_core.controllers import storage_events, health_controller, device_controller
+from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
@@ -31,8 +31,8 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController(kv_store=db_store)
 
 
-def get_cluster_target_status(cluster):
-    snodes = db_controller.get_storage_nodes()
+def get_cluster_target_status(cluster_id):
+    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
 
     online_nodes = 0
     offline_nodes = 0
@@ -66,8 +66,8 @@ def get_cluster_target_status(cluster):
     logger.debug(f"online_devices: {online_devices}")
     logger.debug(f"offline_devices: {offline_devices}")
 
-    # if more than two affected modes then cluster is suspended
-    if affected_nodes > 2:
+    # if more than two affected nodes then cluster is suspended
+    if affected_nodes > 2 or offline_nodes > 2:
         return Cluster.STATUS_SUSPENDED
 
     # if any device goes offline then cluster is degraded
@@ -85,7 +85,7 @@ def update_cluster_status(cluster_id):
     cluster = db_controller.get_cluster_by_id(cluster_id)
 
     if cluster.ha_type == "ha":
-        cluster_target_status = get_cluster_target_status(cluster)
+        cluster_target_status = get_cluster_target_status(cluster_id)
         logger.info(f"Target cluster status {cluster_target_status}, current status: {cluster.status}")
         if cluster.status == cluster_target_status:
             return
@@ -111,48 +111,53 @@ def set_node_online(node):
 def set_node_offline(node):
     if node.status != StorageNode.STATUS_UNREACHABLE:
         storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
+        # add node to auto restart
+        tasks_controller.add_node_to_auto_restart(node)
 
 
 logger.info("Starting node monitor")
 while True:
-    # get storage nodes
-    nodes = db_controller.get_storage_nodes()
-    for snode in nodes:
-        if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
-            logger.info(f"Node status is: {snode.status}, skipping")
-            continue
-
-        logger.info(f"Checking node {snode.hostname}")
-
-        # 1- check node ping
-        ping_check = health_controller._check_node_ping(snode.mgmt_ip)
-        logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
-
-        # 2- check node API
-        node_api_check = health_controller._check_node_api(snode.mgmt_ip)
-        logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
-
-        # 3- check node RPC
-        node_rpc_check = health_controller._check_node_rpc(
-            snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-        logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
-
-        # 4- docker API
-        node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
-        logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
-
-        is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
-        if is_node_online:
-            set_node_online(snode)
-        else:
-            set_node_offline(snode)
-
-        if not ping_check and not node_rpc_check:
-            # node is dead, set devices offline
-            for dev in snode.nvme_devices:
-                device_controller.device_set_unavailable(dev.get_id())
-
-        update_cluster_status(snode.cluster_id)
+    clusters = db_controller.get_clusters()
+    for cluster in clusters:
+        cluster_id = cluster.get_id()
+        # get storage nodes
+        nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+        for snode in nodes:
+            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+                logger.info(f"Node status is: {snode.status}, skipping")
+                continue
+
+            logger.info(f"Checking node {snode.hostname}")
+
+            # 1- check node ping
+            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+            # 2- check node API
+            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+            # 3- check node RPC
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            # 4- docker API
+            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+            if is_node_online:
+                set_node_online(snode)
+            else:
+                set_node_offline(snode)
+
+            if not ping_check and not node_rpc_check:
+                # node is dead, set devices offline
+                for dev in snode.nvme_devices:
+                    device_controller.device_set_unavailable(dev.get_id())
+
+        update_cluster_status(cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")
     time.sleep(constants.NODE_MONITOR_INTERVAL_SEC)

sbcli_pre-1.2.7/simplyblock_core/services/tasks_runner_migration.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import logging
+import time
+import sys
+
+
+from simplyblock_core import constants, kv_store
+from simplyblock_core.controllers import tasks_events
+from simplyblock_core.models.job_schedule import JobSchedule
+
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+
+def task_runner(task):
+    task.status = JobSchedule.STATUS_RUNNING
+    task.write_to_db(db_controller.kv_store)
+    tasks_events.task_updated(task)
+
+    time.sleep(30)
+
+    task.function_result = "sleep 30"
+    task.status = JobSchedule.STATUS_DONE
+    task.write_to_db(db_controller.kv_store)
+    tasks_events.task_updated(task)
+
+    return True
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_controller = kv_store.DBController()
+
+logger.info("Starting Tasks runner...")
+while True:
+    time.sleep(3)
+    clusters = db_controller.get_clusters()
+    if not clusters:
+        logger.error("No clusters found!")
+    else:
+        for cl in clusters:
+            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
+            for task in tasks:
+                delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
+                if task.function_name == JobSchedule.FN_DEV_MIG:
+                    while task.status != JobSchedule.STATUS_DONE:
+                        res = task_runner(task)
+                        if res:
+                            tasks_events.task_updated(task)
+                        else:
+                            time.sleep(delay_seconds)
+                            delay_seconds *= 2

sbcli_pre-1.2.5/simplyblock_core/services/job_tasks.py → sbcli_pre-1.2.7/simplyblock_core/services/tasks_runner_restart.py

@@ -5,7 +5,7 @@ import sys
 
 
 from simplyblock_core import constants, kv_store, storage_node_ops
-from simplyblock_core.controllers import device_controller
+from simplyblock_core.controllers import device_controller, tasks_events
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 
@@ -15,6 +15,19 @@ from graypy import GELFUDPHandler
 from simplyblock_core.models.storage_node import StorageNode
 
 
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_controller = kv_store.DBController()
+
+
 def _get_node_unavailable_devices_count(node_id):
     node = db_controller.get_storage_node_by_id(node_id)
     devices = []
@@ -31,10 +44,20 @@ def _get_device(task):
             return dev
 
 
+def _validate_no_task_node_restart(cluster_id, node_id):
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return False
+    return True
+
+
 def task_runner(task):
-    if task.function_name == "device_restart":
+    if task.function_name == JobSchedule.FN_DEV_RESTART:
         return task_runner_device(task)
-    if task.function_name == "node_restart":
+    if task.function_name == JobSchedule.FN_NODE_RESTART:
         return task_runner_node(task)
 
 
@@ -49,38 +72,55 @@ def task_runner_device(task):
         device_controller.device_set_retries_exhausted(device.get_id(), True)
         return True
 
+    if not _validate_no_task_node_restart(task.cluster_id, task.node_id):
+        task.function_result = "canceled: node restart found"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        device_controller.device_set_unavailable(device.get_id())
+        return True
+
+    if task.canceled:
+        task.function_result = "canceled"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
     node = db_controller.get_storage_node_by_id(task.node_id)
     if node.status != StorageNode.STATUS_ONLINE:
-        logger.error(f"Node is not online: {node.get_id()} , skipping task: {task.get_id()}")
+        logger.error(f"Node is not online: {node.get_id()}, retry")
         task.function_result = "Node is offline"
         task.retry += 1
         task.write_to_db(db_controller.kv_store)
         return False
 
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
-        logger.info(f"Device is online: {device.get_id()}, no restart needed")
-        task.function_result = "skipped because dev is online"
+        logger.info(f"Device is online: {device.get_id()}")
+        task.function_result = "Device is online"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    task.status = JobSchedule.STATUS_RUNNING
-    task.write_to_db(db_controller.kv_store)
-
-    # resetting device
-    logger.info(f"Resetting device {device.get_id()}")
-    device_controller.reset_storage_device(device.get_id())
-    time.sleep(5)
-    device = _get_device(task)
-    if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
-        logger.info(f"Device is online: {device.get_id()}")
-        task.function_result = "done"
+    if device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
+        logger.info(f"Device is not unavailable: {device.get_id()}, {device.status} , stopping task")
+        task.function_result = f"stopped because dev is {device.status}"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    logger.info(f"Restarting device {device.get_id()}")
-    device_controller.restart_device(device.get_id(), force=True)
+    if task.status != JobSchedule.STATUS_RUNNING:
+        task.status = JobSchedule.STATUS_RUNNING
+        task.write_to_db(db_controller.kv_store)
+        tasks_events.task_updated(task)
+
+    # set device online for the first 3 retries
+    if task.retry < 3:
+        logger.info(f"Set device online {device.get_id()}")
+        device_controller.device_set_online(device.get_id())
+    else:
+        logger.info(f"Restarting device {device.get_id()}")
+        device_controller.restart_device(device.get_id(), force=True)
+
+    # check device status
     time.sleep(5)
     device = _get_device(task)
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
@@ -104,22 +144,37 @@ def task_runner_node(task):
         storage_node_ops.set_node_status(task.node_id, StorageNode.STATUS_UNREACHABLE)
         return True
 
-    if _get_node_unavailable_devices_count(node.get_id()) == 0:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
-        task.function_result = "skipped because node is online"
+    if node.status == StorageNode.STATUS_REMOVED:
+        logger.info(f"Node is removed: {task.node_id}, stopping task")
+        task.function_result = f"Node is removed"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    task.status = JobSchedule.STATUS_RUNNING
-    task.write_to_db(db_controller.kv_store)
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
+        task.function_result = "Node is online"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
+    if task.canceled:
+        task.function_result = "canceled"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
+    if task.status != JobSchedule.STATUS_RUNNING:
+        task.status = JobSchedule.STATUS_RUNNING
+        task.write_to_db(db_controller.kv_store)
+        tasks_events.task_updated(task)
 
     # shutting down node
     logger.info(f"Shutdown node {node.get_id()}")
     ret = storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
     if ret:
         logger.info(f"Node shutdown succeeded")
-    time.sleep(5)
+    time.sleep(3)
 
     # resetting node
     logger.info(f"Restart node {node.get_id()}")
@@ -127,8 +182,9 @@ def task_runner_node(task):
     if ret:
         logger.info(f"Node restart succeeded")
 
-    if _get_node_unavailable_devices_count(node.get_id()) == 0:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
+    time.sleep(5)
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
@@ -139,19 +195,7 @@ def task_runner_node(task):
     return False
 
 
-# configure logging
-logger_handler = logging.StreamHandler(stream=sys.stdout)
-logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
-logger = logging.getLogger()
-logger.addHandler(gelf_handler)
-logger.addHandler(logger_handler)
-logger.setLevel(logging.DEBUG)
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-logger.info("Starting Jobs runner...")
+logger.info("Starting Tasks runner...")
 while True:
     time.sleep(3)
     clusters = db_controller.get_clusters()
@@ -159,11 +203,16 @@ while True:
         logger.error("No clusters found!")
     else:
         for cl in clusters:
-            tasks = db_controller.get_job_tasks(cl.get_id())
+            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
             for task in tasks:
                 delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
-                while task.status != JobSchedule.STATUS_DONE:
-                    res = task_runner(task)
-                    if res is False:
-                        time.sleep(delay_seconds)
-                        delay_seconds *= 2
+                if task.function_name in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART]:
+                    while task.status != JobSchedule.STATUS_DONE:
+                        # get new task object because it could be changed from cancel task
+                        task = db_controller.get_task_by_id(task.uuid)
+                        res = task_runner(task)
+                        if res:
+                            tasks_events.task_updated(task)
+                        else:
+                            time.sleep(delay_seconds)
+                            delay_seconds *= 2

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/snode_client.py

@@ -95,3 +95,15 @@ class SNodeClient:
 
     def leave_swarm(self):
         return self._request("GET", "leave_swarm")
+
+    def make_gpt_partitions(self, nbd_device, jm_percent, num_partitions):
+        params = {
+            "nbd_device": nbd_device,
+            "jm_percent": jm_percent,
+            "num_partitions": num_partitions,
+        }
+        return self._request("POST", "make_gpt_partitions", params)
+
+    def delete_dev_gpt_partitions(self, device_pci):
+        params = {"device_pci": device_pci}
+        return self._request("POST", "delete_dev_gpt_partitions", params)