sbcli-dev 6.2.3__tar.gz → 6.2.4__tar.gz

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 6.2.3
+Version: 6.2.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=6.2.3
+SIMPLY_BLOCK_VERSION=6.2.4
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main
 DOCKER_USER=hamdysimplyblock

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 6.2.3
+Version: 6.2.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/sbcli_dev.egg-info/SOURCES.txt

@@ -106,6 +106,7 @@ simplyblock_core/services/remove_service.sh
 simplyblock_core/services/service_template.service
 simplyblock_core/services/spdk_http_proxy_server.py
 simplyblock_core/services/storage_node_monitor.py
+simplyblock_core/services/storage_node_self_lock.py
 simplyblock_core/services/tasks_runner_failed_migration.py
 simplyblock_core/services/tasks_runner_migration.py
 simplyblock_core/services/tasks_runner_new_dev_migration.py

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/simplyblock_core/constants.py

@@ -56,7 +56,7 @@ FDB_CHECK_INTERVAL_SEC = 60
 
 SIMPLY_BLOCK_DOCKER_IMAGE = "simplyblock/simplyblock:main"
 SIMPLY_BLOCK_CLI_NAME = "sbcli-dev"
-TASK_EXEC_INTERVAL_SEC = 30
+TASK_EXEC_INTERVAL_SEC = 10
 TASK_EXEC_RETRY_COUNT = 8
 
 

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/simplyblock_core/distr_controller.py

@@ -14,6 +14,8 @@ logger = logging.getLogger()
 def send_node_status_event(node, node_status, target_node=None):
     db_controller = DBController()
     node_id = node.get_id()
+    if node_status == StorageNode.STATUS_SCHEDULABLE:
+        node_status = StorageNode.STATUS_UNREACHABLE
     logging.info(f"Sending event updates, node: {node_id}, status: {node_status}")
     node_status_event = {
         "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -289,6 +289,20 @@ services:
     environment:
       SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
+  StorageNodeSelfLock:
+    image: $SIMPLYBLOCK_DOCKER_IMAGE
+    command: "python simplyblock_core/services/storage_node_self_lock.py"
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == worker]
+    volumes:
+      - "/etc/foundationdb:/etc/foundationdb"
+    networks:
+      - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
+
 
 volumes:
   os_data:

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/simplyblock_core/services/storage_node_monitor.py

@@ -165,12 +165,11 @@ def get_current_cluster_status(cluster_id):
 
 def update_cluster_status(cluster_id):
     cluster = db_controller.get_cluster_by_id(cluster_id)
-
-    if cluster.status == Cluster.STATUS_READONLY or cluster.status == Cluster.STATUS_UNREADY \
-            or Cluster.STATUS_IN_ACTIVATION:
+    logger.info("cluster_status: %s", cluster.status)
+    if cluster.status in [Cluster.STATUS_READONLY, Cluster.STATUS_UNREADY, Cluster.STATUS_IN_ACTIVATION]:
         return
+
     cluster_current_status = get_current_cluster_status(cluster_id)
-    logger.info("cluster_status: %s", cluster.status)
     logger.info("cluster_new_status: %s", cluster_current_status)
     if cluster.status not in [Cluster.STATUS_ACTIVE, Cluster.STATUS_UNREADY] and cluster_current_status == Cluster.STATUS_ACTIVE:
         # cluster_ops.cluster_activate(cluster_id, True)
@@ -274,11 +273,11 @@ while True:
             else:
                 set_node_offline(snode)
 
-                if not ping_check and not node_api_check and not spdk_process:
-                    # restart on new node
-                    storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_SCHEDULABLE)
+                # if not ping_check and not node_api_check and not spdk_process:
+                #     # restart on new node
+                #     storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_SCHEDULABLE)
 
-                elif ping_check and node_api_check and (not spdk_process or not node_rpc_check):
+                if ping_check and node_api_check and (not spdk_process or not node_rpc_check):
                     # add node to auto restart
                     if cluster.status == Cluster.STATUS_ACTIVE:
                         tasks_controller.add_node_to_auto_restart(snode)

sbcli_dev-6.2.4/simplyblock_core/services/storage_node_self_lock.py

@@ -0,0 +1,41 @@
+# coding=utf-8
+import time
+
+import fdb
+
+from simplyblock_core import constants, utils
+from simplyblock_core.snode_client import SNodeClient
+
+logger = utils.get_logger(__name__)
+
+logger.info("Starting node monitor")
+MAX_RETRY = 6
+retry = MAX_RETRY
+error_open = False
+snode_api = SNodeClient("0.0.0.0:5000")
+
+
+while True:
+
+    is_up, _ = snode_api.spdk_process_is_up()
+    if is_up:
+        if error_open is True and retry <= 0:
+            snode_api.spdk_process_kill()
+            error_open = False
+            retry = MAX_RETRY
+        else:
+            try:
+                fdb.api_version(constants.KVD_DB_VERSION)
+                db = fdb.open(constants.KVD_DB_FILE_PATH)
+                db.options.set_transaction_timeout(5)
+                d =  db.get_range_startswith(b"/",  limit=1)
+                logger.info("Open db ok")
+                error_open = False
+                retry = MAX_RETRY
+            except Exception as e:
+                logger.error(e)
+                logger.info("Open db error")
+                error_open = True
+                retry -= 1
+
+    time.sleep(10)

{sbcli_dev-6.2.3 → sbcli_dev-6.2.4}/simplyblock_core/utils.py

@@ -17,9 +17,6 @@ from simplyblock_core import constants
 from simplyblock_core import shell_utils
 
 
-logger = logging.getLogger()
-
-
 def get_env_var(name, default=None, is_required=False):
     if not name:
         logger.warning("Invalid env var name %s", name)
@@ -131,7 +128,7 @@ def get_docker_client(cluster_id=None):
     nodes = db_controller.get_mgmt_nodes(cluster_id)
     if not nodes:
         logger.error("No mgmt nodes was found in the cluster!")
-        exit(1)
+        return False
 
     docker_ips = [node.docker_ip_port for node in nodes]
 
@@ -139,9 +136,10 @@ def get_docker_client(cluster_id=None):
         try:
             c = docker.DockerClient(base_url=f"tcp://{ip}", version="auto")
             return c
-        except docker.errors.DockerException as e:
+        except Exception as e:
             print(e)
-    raise e
+            raise e
+    return False
 
 
 def dict_agg(data, mean=False):
@@ -580,23 +578,23 @@ def decimal_to_hex_power_of_2(decimal_number):
 
 def get_logger(name):
     # first configure a root logger
-    logger = logging.getLogger()
+    logg = logging.getLogger(name)
     log_level = os.getenv("SIMPLYBLOCK_LOG_LEVEL")
     log_level = log_level.upper() if log_level else constants.LOG_LEVEL
 
     try:
-        logger.setLevel(log_level)
+        logg.setLevel(log_level)
     except ValueError as e:
-        logger.warning(f'Invalid SIMPLYBLOCK_LOG_LEVEL: {str(e)}')
-        logger.setLevel(constants.LOG_LEVEL)
+        logg.warning(f'Invalid SIMPLYBLOCK_LOG_LEVEL: {str(e)}')
+        logg.setLevel(constants.LOG_LEVEL)
 
     logger_handler = logging.StreamHandler(stream=sys.stdout)
     logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-    logger.addHandler(logger_handler)
+    logg.addHandler(logger_handler)
 
     gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
-    logger.addHandler(gelf_handler)
-    return logging.getLogger(name)
+    logg.addHandler(gelf_handler)
+    return logg
 
 
 def parse_size(size_string: str):
@@ -653,4 +651,7 @@ def strfdelta(tdelta):
             if values[field] > 0:
                 out += f"{values[field]}{field.lower()} "
 
-    return out.strip()
+    return out.strip()
+
+
+logger = get_logger(__name__)