sbcli-pre 1.2.4__zip → 1.2.5__zip

sbcli_pre-1.2.4/simplyblock_core/controllers/tasks_controller.py

@@ -1,103 +0,0 @@
-# coding=utf-8
-import logging
-import time
-import uuid
-
-from simplyblock_core import kv_store, constants, utils
-from simplyblock_core.controllers import tasks_events
-from simplyblock_core.models.job_schedule import JobSchedule
-
-logger = logging.getLogger()
-db_controller = kv_store.DBController()
-
-
-def _validate_new_task_node_restart(cluster_id, node_id):
-    tasks = db_controller.get_job_tasks(cluster_id)
-    for task in tasks:
-        if task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return False
-    return True
-
-
-def _validate_new_task_dev_restart(cluster_id, node_id, device_id):
-    tasks = db_controller.get_job_tasks(cluster_id)
-    for task in tasks:
-        if task.function_name == JobSchedule.FN_DEV_RESTART and task.device_id == device_id:
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return False
-        elif task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id:
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return False
-    return True
-
-
-def _add_task(function_name, cluster_id, node_id, device_id):
-
-    if function_name in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_DEV_MIG]:
-        if not _validate_new_task_dev_restart(cluster_id, node_id, device_id):
-            return False
-    elif function_name == JobSchedule.FN_NODE_RESTART:
-        if not _validate_new_task_node_restart(cluster_id, node_id):
-            return False
-
-    task_obj = JobSchedule()
-    task_obj.uuid = str(uuid.uuid4())
-    task_obj.cluster_id = cluster_id
-    task_obj.node_id = node_id
-    task_obj.device_id = device_id
-    task_obj.date = int(time.time())
-    task_obj.function_name = function_name
-    task_obj.status = JobSchedule.STATUS_NEW
-    task_obj.write_to_db(db_controller.kv_store)
-    tasks_events.task_create(task_obj)
-    return task_obj.uuid
-
-
-def add_device_mig_task(device_id):
-    device = db_controller.get_storage_devices(device_id)
-    return _add_task(JobSchedule.FN_DEV_MIG, device.cluster_id, device.node_id, device.get_id())
-
-
-def add_device_to_auto_restart(device):
-    return _add_task(JobSchedule.FN_DEV_RESTART, device.cluster_id, device.node_id, device.get_id())
-
-
-def add_node_to_auto_restart(node):
-    return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "")
-
-
-def list_tasks(cluster_id):
-    cluster = db_controller.get_cluster_by_id(cluster_id)
-    if not cluster:
-        logger.error("Cluster not found: %s", cluster_id)
-        return False
-
-    data = []
-    tasks = db_controller.get_job_tasks(cluster_id)
-    for task in tasks:
-        data.append({
-            "Task ID": task.uuid,
-            "Target ID": task.device_id or task.node_id,
-            "Function": task.function_name,
-            "Retry": f"{task.retry}/{constants.TASK_EXEC_RETRY_COUNT}",
-            "Status": task.status,
-            "Result": task.function_result,
-            "Date": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(task.date)),
-        })
-    return utils.print_table(data)
-
-
-def cancel_task(task_id):
-    task = db_controller.get_task_by_id(task_id)
-    if not task:
-        logger.error("Task not found: %s", task_id)
-        return False
-
-    task.canceled = True
-    task.write_to_db(db_controller.kv_store)
-    tasks_events.task_canceled(task)
-    return True

sbcli_pre-1.2.4/simplyblock_core/controllers/tasks_events.py

@@ -1,37 +0,0 @@
-# coding=utf-8
-import logging
-
-from simplyblock_core.controllers import events_controller as ec
-from simplyblock_core.kv_store import DBController
-
-logger = logging.getLogger()
-db_controller = DBController()
-
-
-def _task_event(task, message, caused_by, event):
-    ec.log_event_cluster(
-        cluster_id=task.cluster_id,
-        domain=ec.DOMAIN_CLUSTER,
-        event=event,
-        db_object=task,
-        caused_by=caused_by,
-        message=message,
-        node_id=task.node_id,
-        status=task.status)
-
-
-def task_create(task, caused_by=ec.CAUSED_BY_CLI):
-    _task_event(task, f"task created: {task.uuid}", caused_by, ec.EVENT_OBJ_CREATED)
-
-
-def task_updated(task, caused_by=ec.CAUSED_BY_CLI):
-    _task_event(task, f"Task updated: {task.uuid}", caused_by, ec.EVENT_STATUS_CHANGE)
-
-
-def task_status_change(task, new_state, old_status, caused_by=ec.CAUSED_BY_CLI):
-    _task_event(task, f"task status changed from: {old_status} to: {new_state}", caused_by, ec.EVENT_STATUS_CHANGE)
-
-
-def task_canceled(task, caused_by=ec.CAUSED_BY_CLI):
-    _task_event(task, f"Task canceled: {task.uuid}", caused_by, ec.EVENT_STATUS_CHANGE)
-

sbcli_pre-1.2.4/simplyblock_core/mgmt_node_ops.py

@@ -1,205 +0,0 @@
-# coding=utf-8
-import json
-import logging
-import uuid
-import time
-import requests
-
-import docker
-
-from simplyblock_core import utils, scripts
-from simplyblock_core.controllers import mgmt_events
-from simplyblock_core.kv_store import DBController
-from simplyblock_core.models.mgmt_node import MgmtNode
-
-
-logger = logging.getLogger()
-
-
-def deploy_mgmt_node(cluster_ip, cluster_id, ifname):
-
-    try:
-        resp = requests.get(f"http://{cluster_ip}/cluster/{cluster_id}")
-        resp_json = resp.json()
-        cluster_data = resp_json['results'][0]
-        logger.info(f"Cluster found! NQN:{cluster_data['nqn']}")
-        logger.debug(cluster_data)
-    except Exception as e:
-        logger.error("Error getting cluster data!")
-        logger.error(e)
-        return ""
-
-    logger.info("Installing dependencies...")
-    scripts.install_deps()
-    logger.info("Installing dependencies > Done")
-
-    if not ifname:
-        ifname = "eth0"
-
-    DEV_IP = utils.get_iface_ip(ifname)
-    if not DEV_IP:
-        logger.error(f"Error getting interface ip: {ifname}")
-        return False
-
-    logger.info(f"Node IP: {DEV_IP}")
-    ret = scripts.configure_docker(DEV_IP)
-
-    db_connection = cluster_data['db_connection']
-    scripts.set_db_config(db_connection)
-    time.sleep(1)
-    hostname = utils.get_hostname()
-    db_controller = DBController()
-    nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
-    if not nodes:
-        logger.error("No mgmt nodes was found in the cluster!")
-        return False
-    for node in nodes:
-        if node.hostname == hostname:
-            logger.error("Node already exists in the cluster")
-            return False
-
-    logger.info("Joining docker swarm...")
-    try:
-        cluster_docker = utils.get_docker_client(cluster_id)
-        docker_ip = cluster_docker.info()["Swarm"]["NodeAddr"]
-        join_token = cluster_docker.swarm.attrs['JoinTokens']['Manager']
-        node_docker = docker.DockerClient(base_url=f"tcp://{DEV_IP}:2375", version="auto")
-        if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
-            logger.info("Node is part of another swarm, leaving swarm")
-            try:
-                cluster_docker.nodes.get(node_docker.info()["Swarm"]["NodeID"]).remove(force=True)
-            except:
-                pass
-            node_docker.swarm.leave(force=True)
-            time.sleep(5)
-
-        node_docker.swarm.join([f"{docker_ip}:2377"], join_token)
-
-        retries = 10
-        while retries > 0:
-            if node_docker.info()["Swarm"]["LocalNodeState"] == "active":
-                break
-            logger.info("Waiting for node to be active...")
-            retries -= 1
-            time.sleep(2)
-        logger.info("Joining docker swarm > Done")
-        time.sleep(5)
-
-    except Exception as e:
-        raise e
-
-    logger.info("Adding management node object")
-    node_id = add_mgmt_node(DEV_IP, cluster_id)
-
-    # check if ha setting is required
-    nodes = db_controller.get_mgmt_nodes(cluster_id=cluster_id)
-    if len(nodes) >= 3:
-        logger.info("Waiting for FDB container to be active...")
-        fdb_cont = None
-        retries = 30
-        while retries > 0 and fdb_cont is None:
-            logger.info("Looking for FDB container...")
-            for cont in node_docker.containers.list(all=True):
-                logger.debug(cont.attrs['Name'])
-                if cont.attrs['Name'].startswith("/app_fdb"):
-                    fdb_cont = cont
-                    break
-            if fdb_cont:
-                logger.info("FDB container found")
-                break
-            else:
-                retries -= 1
-                time.sleep(5)
-
-        if not fdb_cont:
-            logger.warning("FDB container was not found")
-        else:
-            retries = 10
-            while retries > 0:
-                info = node_docker.containers.get(fdb_cont.attrs['Id'])
-                status = info.attrs['State']["Status"]
-                is_running = info.attrs['State']["Running"]
-                if not is_running:
-                    logger.info("Container is not running, waiting...")
-                    time.sleep(3)
-                    retries -= 1
-                else:
-                    logger.info(f"Container status: {status}, Is Running: {is_running}")
-                break
-
-        logger.info("Configuring Double DB...")
-        time.sleep(3)
-        scripts.set_db_config_double()
-        for cl in db_controller.get_clusters():
-            cl.ha_type = "ha"
-            cl.write_to_db(db_controller.kv_store)
-
-    logger.info("Node joined the cluster")
-    return node_id
-
-
-def add_mgmt_node(mgmt_ip, cluster_id=None):
-    db_controller = DBController()
-    hostname = utils.get_hostname()
-    node = db_controller.get_mgmt_node_by_hostname(hostname)
-    if node:
-        logger.error("Node already exists in the cluster")
-        return False
-
-    node = MgmtNode()
-    node.uuid = str(uuid.uuid4())
-    node.hostname = hostname
-    node.docker_ip_port = f"{mgmt_ip}:2375"
-    node.cluster_id = cluster_id
-    node.mgmt_ip = mgmt_ip
-    node.status = MgmtNode.STATUS_ONLINE
-    node.write_to_db(db_controller.kv_store)
-
-    mgmt_events.mgmt_add(node)
-    logger.info("Done")
-    return node.uuid
-
-
-def list_mgmt_nodes(is_json):
-    db_controller = DBController()
-    nodes = db_controller.get_mgmt_nodes()
-    data = []
-    output = ""
-
-    for node in nodes:
-        logging.debug(node)
-        logging.debug("*" * 20)
-        data.append({
-            "UUID": node.get_id(),
-            "Hostname": node.hostname,
-            "IP": node.mgmt_ip,
-            "Status": node.status,
-        })
-
-    if not data:
-        return output
-
-    if is_json:
-        output = json.dumps(data, indent=2)
-    else:
-        output = utils.print_table(data)
-    return output
-
-
-def remove_mgmt_node(uuid):
-    db_controller = DBController()
-    snode = db_controller.get_mgmt_node_by_id(uuid)
-    if not snode:
-        logger.error("can not find node")
-        return False
-
-    logging.info("Removing mgmt node")
-    snode.remove(db_controller.kv_store)
-
-    logger.info("Leaving swarm...")
-    node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto")
-    node_docker.swarm.leave()
-
-    mgmt_events.mgmt_remove(snode)
-    logging.info("done")
-

sbcli_pre-1.2.4/simplyblock_core/services/health_check_service.py

@@ -1,134 +0,0 @@
-# coding=utf-8
-import logging
-
-import time
-import sys
-from datetime import datetime
-
-
-from simplyblock_core.controllers import health_controller, storage_events, device_events
-from simplyblock_core.models.storage_node import StorageNode
-from simplyblock_core.rpc_client import RPCClient
-from simplyblock_core import constants, kv_store
-
-# Import the GELF logger
-from graypy import GELFUDPHandler
-
-def set_node_health_check(snode, health_check_status):
-    snode = db_controller.get_storage_node_by_id(snode.get_id())
-    if snode.health_check == health_check_status:
-        return
-    old_status = snode.health_check
-    snode.health_check = health_check_status
-    snode.updated_at = str(datetime.now())
-    snode.write_to_db(db_store)
-    storage_events.snode_health_check_change(snode, snode.health_check, old_status, caused_by="monitor")
-
-
-def set_device_health_check(cluster_id, device, health_check_status):
-    if device.health_check == health_check_status:
-        return
-    nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
-    for node in nodes:
-        if node.nvme_devices:
-            for dev in node.nvme_devices:
-                if dev.get_id() == device.get_id():
-                    old_status = dev.health_check
-                    dev.health_check = health_check_status
-                    node.write_to_db(db_store)
-                    device_events.device_health_check_change(
-                        dev, dev.health_check, old_status, caused_by="monitor")
-
-
-# configure logging
-logger_handler = logging.StreamHandler(stream=sys.stdout)
-logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
-logger = logging.getLogger()
-logger.addHandler(gelf_handler)
-logger.addHandler(logger_handler)
-logger.setLevel(logging.DEBUG)
-
-# get DB controller
-db_store = kv_store.KVStore()
-db_controller = kv_store.DBController()
-
-logger.info("Starting health check service")
-while True:
-    clusters = db_controller.get_clusters()
-    for cluster in clusters:
-        cluster_id = cluster.get_id()
-        snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
-        if not snodes:
-            logger.error("storage nodes list is empty")
-
-        for snode in snodes:
-            logger.info("Node: %s, status %s", snode.get_id(), snode.status)
-
-            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
-                logger.info(f"Node status is: {snode.status}, skipping")
-                continue
-
-            # 1- check node ping
-            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
-            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
-
-            # 2- check node API
-            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
-            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
-
-            if snode.status == StorageNode.STATUS_OFFLINE:
-                set_node_health_check(snode, ping_check & node_api_check)
-                continue
-
-            # 3- check node RPC
-            node_rpc_check = health_controller._check_node_rpc(
-                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
-            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
-
-            # 4- docker API
-            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
-            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
-
-            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
-
-            health_check_status = is_node_online
-            if not node_rpc_check:
-                logger.info("Putting all devices to unavailable state because RPC check failed")
-                for dev in snode.nvme_devices:
-                    if dev.io_error:
-                        logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
-                        continue
-                    set_device_health_check(cluster_id, dev, False)
-            else:
-                logger.info(f"Node device count: {len(snode.nvme_devices)}")
-                node_devices_check = True
-                node_remote_devices_check = True
-
-                for dev in snode.nvme_devices:
-                    if dev.io_error:
-                        logger.debug(f"Skipping Device check because of io_error {dev.get_id()}")
-                        continue
-                    ret = health_controller.check_device(dev.get_id())
-                    set_device_health_check(cluster_id, dev, ret)
-                    if dev.status == dev.STATUS_ONLINE:
-                        node_devices_check &= ret
-
-                logger.info(f"Node remote device: {len(snode.remote_devices)}")
-                rpc_client = RPCClient(
-                    snode.mgmt_ip, snode.rpc_port,
-                    snode.rpc_username, snode.rpc_password,
-                    timeout=10, retry=1)
-                for remote_device in snode.remote_devices:
-                    ret = rpc_client.get_bdevs(remote_device.remote_bdev)
-                    if ret:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
-                    else:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    node_remote_devices_check &= bool(ret)
-
-                health_check_status = is_node_online and node_devices_check and node_remote_devices_check
-            set_node_health_check(snode, health_check_status)
-
-    time.sleep(constants.HEALTH_CHECK_INTERVAL_SEC)
-

sbcli_pre-1.2.4/simplyblock_core/services/tasks_runner_migration.py

@@ -1,61 +0,0 @@
-# coding=utf-8
-import logging
-import time
-import sys
-
-
-from simplyblock_core import constants, kv_store
-from simplyblock_core.controllers import tasks_events
-from simplyblock_core.models.job_schedule import JobSchedule
-
-
-# Import the GELF logger
-from graypy import GELFUDPHandler
-
-
-def task_runner(task):
-    task.status = JobSchedule.STATUS_RUNNING
-    task.write_to_db(db_controller.kv_store)
-    tasks_events.task_updated(task)
-
-    time.sleep(30)
-
-    task.function_result = "sleep 30"
-    task.status = JobSchedule.STATUS_DONE
-    task.write_to_db(db_controller.kv_store)
-    tasks_events.task_updated(task)
-
-    return True
-
-
-# configure logging
-logger_handler = logging.StreamHandler(stream=sys.stdout)
-logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
-logger = logging.getLogger()
-logger.addHandler(gelf_handler)
-logger.addHandler(logger_handler)
-logger.setLevel(logging.DEBUG)
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-logger.info("Starting Tasks runner...")
-while True:
-    time.sleep(3)
-    clusters = db_controller.get_clusters()
-    if not clusters:
-        logger.error("No clusters found!")
-    else:
-        for cl in clusters:
-            tasks = db_controller.get_job_tasks(cl.get_id(), reverse=False)
-            for task in tasks:
-                delay_seconds = constants.TASK_EXEC_INTERVAL_SEC
-                if task.function_name == JobSchedule.FN_DEV_MIG:
-                    while task.status != JobSchedule.STATUS_DONE:
-                        res = task_runner(task)
-                        if res:
-                            tasks_events.task_updated(task)
-                        else:
-                            time.sleep(delay_seconds)
-                            delay_seconds *= 2