sbcli-pre 1.2.5__zip → 1.2.6__zip

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/distr_event_collector.py

@@ -88,7 +88,7 @@ def process_lvol_event(event):
     if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
         vuid = event.object_dict['vuid']
         lvol = None
-        for lv in db_controller.get_lvols():
+        for lv in db_controller.get_lvols():  # pass
             if lv.vuid == vuid:
                 lvol = lv
                 break
@@ -127,7 +127,6 @@ def process_event(event_id):
 
 hostname = utils.get_hostname()
 logger.info("Starting Distr event collector...")
-logger.info(f"Node:{hostname}")
 while True:
     time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
 
@@ -141,14 +140,13 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=3, retry=2
-    )
-    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
+        timeout=10, retry=2)
+
     try:
-        # events = client.distr_status_events_get()
-        events = client.distr_status_events_discard_then_get(0, num_of_events)
+        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
+
         if not events:
-            logger.error("Distr events empty")
+            logger.debug("no events found")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -161,10 +159,11 @@ while True:
         for eid in event_ids:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
-        logger.info(f"Discarding events: {num_of_events}")
-        events = client.distr_status_events_discard_then_get(num_of_events, 0)
+
+            logger.info(f"Discarding events: {len(events)}")
+            client.distr_status_events_discard_then_get(len(events), 0)
 
     except Exception as e:
-        logger.error("Failed to get distr events")
+        logger.error("Failed to process distr events")
         logger.exception(e)
         continue

sbcli_pre-1.2.6/simplyblock_core/services/health_check_service.py

@@ -0,0 +1,134 @@
+# coding=utf-8
+import logging
+
+import time
+import sys
+from datetime import datetime
+
+
+from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.storage_node import StorageNode
+from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core import constants, kv_store
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+def set_node_health_check(snode, health_check_status):
+    snode = db_controller.get_storage_node_by_id(snode.get_id())
+    if snode.health_check == health_check_status:
+        return
+    old_status = snode.health_check
+    snode.health_check = health_check_status
+    snode.updated_at = str(datetime.now())
+    snode.write_to_db(db_store)
+    storage_events.snode_health_check_change(snode, snode.health_check, old_status, caused_by="monitor")
+
+
+def set_device_health_check(cluster_id, device, health_check_status):
+    if device.health_check == health_check_status:
+        return
+    nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+    for node in nodes:
+        if node.nvme_devices:
+            for dev in node.nvme_devices:
+                if dev.get_id() == device.get_id():
+                    old_status = dev.health_check
+                    dev.health_check = health_check_status
+                    node.write_to_db(db_store)
+                    device_events.device_health_check_change(
+                        dev, dev.health_check, old_status, caused_by="monitor")
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_store = kv_store.KVStore()
+db_controller = kv_store.DBController()
+
+logger.info("Starting health check service")
+while True:
+    clusters = db_controller.get_clusters()
+    for cluster in clusters:
+        cluster_id = cluster.get_id()
+        snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+        if not snodes:
+            logger.error("storage nodes list is empty")
+
+        for snode in snodes:
+            logger.info("Node: %s, status %s", snode.get_id(), snode.status)
+
+            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+                logger.info(f"Node status is: {snode.status}, skipping")
+                continue
+
+            # 1- check node ping
+            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+            # 2- check node API
+            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+            if snode.status == StorageNode.STATUS_OFFLINE:
+                set_node_health_check(snode, ping_check & node_api_check)
+                continue
+
+            # 3- check node RPC
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            # 4- docker API
+            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+
+            health_check_status = is_node_online
+            if not node_rpc_check:
+                logger.info("Putting all devices to unavailable state because RPC check failed")
+                for dev in snode.nvme_devices:
+                    if dev.io_error:
+                        logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
+                        continue
+                    set_device_health_check(cluster_id, dev, False)
+            else:
+                logger.info(f"Node device count: {len(snode.nvme_devices)}")
+                node_devices_check = True
+                node_remote_devices_check = True
+
+                for dev in snode.nvme_devices:
+                    if dev.io_error:
+                        logger.debug(f"Skipping Device check because of io_error {dev.get_id()}")
+                        continue
+                    ret = health_controller.check_device(dev.get_id())
+                    set_device_health_check(cluster_id, dev, ret)
+                    if dev.status == dev.STATUS_ONLINE:
+                        node_devices_check &= ret
+
+                logger.info(f"Node remote device: {len(snode.remote_devices)}")
+                rpc_client = RPCClient(
+                    snode.mgmt_ip, snode.rpc_port,
+                    snode.rpc_username, snode.rpc_password,
+                    timeout=10, retry=1)
+                for remote_device in snode.remote_devices:
+                    ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                    if ret:
+                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                    else:
+                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                    node_remote_devices_check &= bool(ret)
+
+                health_check_status = is_node_online and node_devices_check and node_remote_devices_check
+            set_node_health_check(snode, health_check_status)
+
+    time.sleep(constants.HEALTH_CHECK_INTERVAL_SEC)
+

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/lvol_monitor.py

@@ -49,7 +49,7 @@ db_controller = kv_store.DBController()
 
 logger.info("Starting LVol monitor...")
 while True:
-    lvols = db_controller.get_lvols()
+    lvols = db_controller.get_lvols()  # pass
     if not lvols:
         logger.error("LVols list is empty")
 

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/lvol_stat_collector.py

@@ -115,7 +115,7 @@ logger.info("Starting stats collector...")
 while True:
 
     pools = db_controller.get_pools()
-    all_lvols = db_controller.get_lvols()
+    all_lvols = db_controller.get_lvols()  # pass
     for pool in pools:
         lvols = []
         for lvol in all_lvols:

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/port_stat_collector.py

@@ -74,4 +74,3 @@ while True:
             update_port_stats(snode, nic, stats)
         else:
             logger.error("Error getting port stats: %s", nic.get_id())
-

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/storage_node_monitor.py

@@ -8,7 +8,7 @@ from datetime import datetime
 
 
 from simplyblock_core import constants, kv_store, cluster_ops, storage_node_ops, distr_controller
-from simplyblock_core.controllers import storage_events, health_controller, device_controller
+from simplyblock_core.controllers import health_controller, device_controller, tasks_controller
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
@@ -31,8 +31,8 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController(kv_store=db_store)
 
 
-def get_cluster_target_status(cluster):
-    snodes = db_controller.get_storage_nodes()
+def get_cluster_target_status(cluster_id):
+    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
 
     online_nodes = 0
     offline_nodes = 0
@@ -66,8 +66,8 @@ def get_cluster_target_status(cluster):
     logger.debug(f"online_devices: {online_devices}")
     logger.debug(f"offline_devices: {offline_devices}")
 
-    # if more than two affected modes then cluster is suspended
-    if affected_nodes > 2:
+    # if more than two affected nodes then cluster is suspended
+    if affected_nodes > 2 or offline_nodes > 2:
         return Cluster.STATUS_SUSPENDED
 
     # if any device goes offline then cluster is degraded
@@ -85,7 +85,7 @@ def update_cluster_status(cluster_id):
     cluster = db_controller.get_cluster_by_id(cluster_id)
 
     if cluster.ha_type == "ha":
-        cluster_target_status = get_cluster_target_status(cluster)
+        cluster_target_status = get_cluster_target_status(cluster_id)
         logger.info(f"Target cluster status {cluster_target_status}, current status: {cluster.status}")
         if cluster.status == cluster_target_status:
             return
@@ -111,48 +111,53 @@ def set_node_online(node):
 def set_node_offline(node):
     if node.status != StorageNode.STATUS_UNREACHABLE:
         storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_UNREACHABLE)
+        # add node to auto restart
+        tasks_controller.add_node_to_auto_restart(node)
 
 
 logger.info("Starting node monitor")
 while True:
-    # get storage nodes
-    nodes = db_controller.get_storage_nodes()
-    for snode in nodes:
-        if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
-            logger.info(f"Node status is: {snode.status}, skipping")
-            continue
-
-        logger.info(f"Checking node {snode.hostname}")
-
-        # 1- check node ping
-        ping_check = health_controller._check_node_ping(snode.mgmt_ip)
-        logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
-
-        # 2- check node API
-        node_api_check = health_controller._check_node_api(snode.mgmt_ip)
-        logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
-
-        # 3- check node RPC
-        node_rpc_check = health_controller._check_node_rpc(
-            snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-        logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
-
-        # 4- docker API
-        node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
-        logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
-
-        is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
-        if is_node_online:
-            set_node_online(snode)
-        else:
-            set_node_offline(snode)
-
-        if not ping_check and not node_rpc_check:
-            # node is dead, set devices offline
-            for dev in snode.nvme_devices:
-                device_controller.device_set_unavailable(dev.get_id())
-
-        update_cluster_status(snode.cluster_id)
+    clusters = db_controller.get_clusters()
+    for cluster in clusters:
+        cluster_id = cluster.get_id()
+        # get storage nodes
+        nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+        for snode in nodes:
+            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+                logger.info(f"Node status is: {snode.status}, skipping")
+                continue
+
+            logger.info(f"Checking node {snode.hostname}")
+
+            # 1- check node ping
+            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+            # 2- check node API
+            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+            # 3- check node RPC
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            # 4- docker API
+            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+            if is_node_online:
+                set_node_online(snode)
+            else:
+                set_node_offline(snode)
+
+            if not ping_check and not node_rpc_check:
+                # node is dead, set devices offline
+                for dev in snode.nvme_devices:
+                    device_controller.device_set_unavailable(dev.get_id())
+
+        update_cluster_status(cluster_id)
 
     logger.info(f"Sleeping for {constants.NODE_MONITOR_INTERVAL_SEC} seconds")
     time.sleep(constants.NODE_MONITOR_INTERVAL_SEC)

sbcli_pre-1.2.6/simplyblock_core/services/tasks_runner_migration.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import logging
+import time
+import sys
+
+
+from simplyblock_core import constants, kv_store
+from simplyblock_core.controllers import tasks_events
+from simplyblock_core.models.job_schedule import JobSchedule
+
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+
+def task_runner(task):
+    task.status = JobSchedule.STATUS_RUNNING
+    task.write_to_db(db_controller.kv_store)
+    tasks_events.task_updated(task)
+
+    time.sleep(30)
+
+    task.function_result = "sleep 30"
+    task.status = JobSchedule.STATUS_DONE
+    task.write_to_db(db_controller.kv_store)
+    tasks_events.task_updated(task)
+
+    return True
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_controller = kv_store.DBController()
+
+logger.info("Starting Tasks runner...")
+while True:
+    time.sleep(3)
+    clusters = db_controller.get_clusters()
+    if not clusters:
+        logger.error("No clusters found!")
+    else:
+        for cl in clusters:
+            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
+            for task in tasks:
+                delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
+                if task.function_name == JobSchedule.FN_DEV_MIG:
+                    while task.status != JobSchedule.STATUS_DONE:
+                        res = task_runner(task)
+                        if res:
+                            tasks_events.task_updated(task)
+                        else:
+                            time.sleep(delay_seconds)
+                            delay_seconds *= 2

sbcli_pre-1.2.5/simplyblock_core/services/job_tasks.py → sbcli_pre-1.2.6/simplyblock_core/services/tasks_runner_restart.py

@@ -5,7 +5,7 @@ import sys
 
 
 from simplyblock_core import constants, kv_store, storage_node_ops
-from simplyblock_core.controllers import device_controller
+from simplyblock_core.controllers import device_controller, tasks_events
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 
@@ -15,6 +15,19 @@ from graypy import GELFUDPHandler
 from simplyblock_core.models.storage_node import StorageNode
 
 
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_controller = kv_store.DBController()
+
+
 def _get_node_unavailable_devices_count(node_id):
     node = db_controller.get_storage_node_by_id(node_id)
     devices = []
@@ -31,10 +44,20 @@ def _get_device(task):
             return dev
 
 
+def _validate_no_task_node_restart(cluster_id, node_id):
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return False
+    return True
+
+
 def task_runner(task):
-    if task.function_name == "device_restart":
+    if task.function_name == JobSchedule.FN_DEV_RESTART:
         return task_runner_device(task)
-    if task.function_name == "node_restart":
+    if task.function_name == JobSchedule.FN_NODE_RESTART:
         return task_runner_node(task)
 
 
@@ -49,38 +72,55 @@ def task_runner_device(task):
         device_controller.device_set_retries_exhausted(device.get_id(), True)
         return True
 
+    if not _validate_no_task_node_restart(task.cluster_id, task.node_id):
+        task.function_result = "canceled: node restart found"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        device_controller.device_set_unavailable(device.get_id())
+        return True
+
+    if task.canceled:
+        task.function_result = "canceled"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
     node = db_controller.get_storage_node_by_id(task.node_id)
     if node.status != StorageNode.STATUS_ONLINE:
-        logger.error(f"Node is not online: {node.get_id()} , skipping task: {task.get_id()}")
+        logger.error(f"Node is not online: {node.get_id()}, retry")
         task.function_result = "Node is offline"
         task.retry += 1
         task.write_to_db(db_controller.kv_store)
         return False
 
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
-        logger.info(f"Device is online: {device.get_id()}, no restart needed")
-        task.function_result = "skipped because dev is online"
+        logger.info(f"Device is online: {device.get_id()}")
+        task.function_result = "Device is online"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    task.status = JobSchedule.STATUS_RUNNING
-    task.write_to_db(db_controller.kv_store)
-
-    # resetting device
-    logger.info(f"Resetting device {device.get_id()}")
-    device_controller.reset_storage_device(device.get_id())
-    time.sleep(5)
-    device = _get_device(task)
-    if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
-        logger.info(f"Device is online: {device.get_id()}")
-        task.function_result = "done"
+    if device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]:
+        logger.info(f"Device is not unavailable: {device.get_id()}, {device.status} , stopping task")
+        task.function_result = f"stopped because dev is {device.status}"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    logger.info(f"Restarting device {device.get_id()}")
-    device_controller.restart_device(device.get_id(), force=True)
+    if task.status != JobSchedule.STATUS_RUNNING:
+        task.status = JobSchedule.STATUS_RUNNING
+        task.write_to_db(db_controller.kv_store)
+        tasks_events.task_updated(task)
+
+    # set device online for the first 3 retries
+    if task.retry < 3:
+        logger.info(f"Set device online {device.get_id()}")
+        device_controller.device_set_online(device.get_id())
+    else:
+        logger.info(f"Restarting device {device.get_id()}")
+        device_controller.restart_device(device.get_id(), force=True)
+
+    # check device status
     time.sleep(5)
     device = _get_device(task)
     if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False:
@@ -104,22 +144,37 @@ def task_runner_node(task):
         storage_node_ops.set_node_status(task.node_id, StorageNode.STATUS_UNREACHABLE)
         return True
 
-    if _get_node_unavailable_devices_count(node.get_id()) == 0:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
-        task.function_result = "skipped because node is online"
+    if node.status == StorageNode.STATUS_REMOVED:
+        logger.info(f"Node is removed: {task.node_id}, stopping task")
+        task.function_result = f"Node is removed"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
         return True
 
-    task.status = JobSchedule.STATUS_RUNNING
-    task.write_to_db(db_controller.kv_store)
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
+        task.function_result = "Node is online"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
+    if task.canceled:
+        task.function_result = "canceled"
+        task.status = JobSchedule.STATUS_DONE
+        task.write_to_db(db_controller.kv_store)
+        return True
+
+    if task.status != JobSchedule.STATUS_RUNNING:
+        task.status = JobSchedule.STATUS_RUNNING
+        task.write_to_db(db_controller.kv_store)
+        tasks_events.task_updated(task)
 
     # shutting down node
     logger.info(f"Shutdown node {node.get_id()}")
     ret = storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
     if ret:
         logger.info(f"Node shutdown succeeded")
-    time.sleep(5)
+    time.sleep(3)
 
     # resetting node
     logger.info(f"Restart node {node.get_id()}")
@@ -127,8 +182,9 @@ def task_runner_node(task):
     if ret:
         logger.info(f"Node restart succeeded")
 
-    if _get_node_unavailable_devices_count(node.get_id()) == 0:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
+    time.sleep(5)
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
@@ -139,19 +195,7 @@ def task_runner_node(task):
     return False
 
 
-# configure logging
-logger_handler = logging.StreamHandler(stream=sys.stdout)
-logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
-logger = logging.getLogger()
-logger.addHandler(gelf_handler)
-logger.addHandler(logger_handler)
-logger.setLevel(logging.DEBUG)
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-logger.info("Starting Jobs runner...")
+logger.info("Starting Tasks runner...")
 while True:
     time.sleep(3)
     clusters = db_controller.get_clusters()
@@ -159,11 +203,16 @@ while True:
         logger.error("No clusters found!")
     else:
         for cl in clusters:
-            tasks = db_controller.get_job_tasks(cl.get_id())
+            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
             for task in tasks:
                 delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
-                while task.status != JobSchedule.STATUS_DONE:
-                    res = task_runner(task)
-                    if res is False:
-                        time.sleep(delay_seconds)
-                        delay_seconds *= 2
+                if task.function_name in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART]:
+                    while task.status != JobSchedule.STATUS_DONE:
+                        # get new task object because it could be changed from cancel task
+                        task = db_controller.get_task_by_id(task.get_id())
+                        res = task_runner(task)
+                        if res:
+                            tasks_events.task_updated(task)
+                        else:
+                            time.sleep(delay_seconds)
+                            delay_seconds *= 2

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/snode_client.py

@@ -95,3 +95,15 @@ class SNodeClient:
 
     def leave_swarm(self):
         return self._request("GET", "leave_swarm")
+
+    def make_gpt_partitions(self, nbd_device, jm_percent, num_partitions):
+        params = {
+            "nbd_device": nbd_device,
+            "jm_percent": jm_percent,
+            "num_partitions": num_partitions,
+        }
+        return self._request("POST", "make_gpt_partitions", params)
+
+    def delete_dev_gpt_partitions(self, device_pci):
+        params = {"device_pci": device_pci}
+        return self._request("POST", "delete_dev_gpt_partitions", params)