sbcli-dev 3.9.4__zip → 3.9.6__zip

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.4
+Version: 3.9.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.9.4
+SIMPLY_BLOCK_VERSION=3.9.6
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.4
+Version: 3.9.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/cluster_ops.py

@@ -215,8 +215,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     shutil.rmtree(temp_dir)
 
     logger.info("Deploying swarm stack ...")
+    log_level = "DEBUG" if constants.LOG_WEB_DEBUG else "INFO"
     ret = scripts.deploy_stack(cli_pass, DEV_IP, constants.SIMPLY_BLOCK_DOCKER_IMAGE, c.secret, c.uuid,
-                               log_del_interval, metrics_retention_period)
+                               log_del_interval, metrics_retention_period, log_level=log_level)
     logger.info("Deploying swarm stack > Done")
 
     if ret == 0:

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/constants.py

@@ -7,7 +7,7 @@ KVD_DB_TIMEOUT_MS = 10000
 SPK_DIR = '/home/ec2-user/spdk'
 RPC_HTTP_PROXY_PORT = 8080
 LOG_LEVEL = logging.INFO
-LOG_WEB_DEBUG = True
+LOG_WEB_DEBUG = False
 
 INSTALL_DIR = os.path.dirname(os.path.realpath(__file__))
 

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/controllers/device_controller.py

@@ -80,6 +80,19 @@ def device_set_read_only(device_id):
 
 
 def device_set_online(device_id):
+    db_controller = DBController()
+    dev = db_controller.get_storage_devices(device_id)
+    snode = db_controller.get_storage_node_by_id(dev.node_id)
+    logger.info("Make other nodes connect to the node devices")
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    for node in snodes:
+        if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+            continue
+        node.remote_devices = storage_node_ops._connect_to_remote_devs(node)
+        if node.enable_ha_jm:
+            node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
+        node.write_to_db()
+
     ret = device_set_state(device_id, NVMeDevice.STATUS_ONLINE)
     if ret:
         logger.info("Adding task to device data migration")

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/distr_controller.py

@@ -43,10 +43,10 @@ def send_dev_status_event(device, dev_status):
     logger.debug(node_status_event)
     snodes = db_controller.get_storage_nodes_by_cluster_id(device.cluster_id)
     for node in snodes:
-        if node.status != node.STATUS_ONLINE:
+        if node.status != StorageNode.STATUS_ONLINE:
             continue
         logging.debug(f"Sending event updates, device: {storage_ID}, status: {dev_status}, node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=2)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -59,7 +59,7 @@ def disconnect_device(device):
         if node.status != node.STATUS_ONLINE:
             continue
         new_remote_devices = []
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=1)
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=5, retry=2)
         for rem_dev in node.remote_devices:
             if rem_dev.get_id() == device.get_id():
                 ctrl_name = rem_dev.remote_bdev[:-2]
@@ -98,7 +98,8 @@ def get_distr_cluster_map(snodes, target_node):
                         break
             if not name:
                 name = f"remote_{dev.alceml_bdev}n1"
-                dev_status = NVMeDevice.STATUS_UNAVAILABLE
+                if dev_status == NVMeDevice.STATUS_ONLINE:
+                    dev_status = NVMeDevice.STATUS_UNAVAILABLE
             logger.debug(f"Device: {dev.get_id()}, status: {dev_status}, bdev_name: {name}")
             dev_map[dev.cluster_device_order] = {
                 "UUID": dev.get_id(),

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/scripts/__init__.py

@@ -30,10 +30,12 @@ def configure_docker(docker_ip):
     return __run_script(['bash', '-x', os.path.join(DIR_PATH, 'config_docker.sh'), docker_ip])
 
 
-def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, log_del_interval, metrics_retention_period):
+def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id,
+                 log_del_interval, metrics_retention_period, log_level):
     pass_hash = hashlib.sha256(graylog_password.encode('utf-8')).hexdigest()
     return __run_script(
-        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, graylog_password, cluster_id, log_del_interval, metrics_retention_period])
+        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash,
+         graylog_password, cluster_id, log_del_interval, metrics_retention_period, log_level])
 
 
 def deploy_cleaner():

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/scripts/deploy_stack.sh

@@ -11,6 +11,7 @@ export CLUSTER_SECRET=$5
 export CLUSTER_ID=$6
 export LOG_DELETION_INTERVAL=$7
 export RETENTION_PERIOD=$8
+export LOG_LEVEL=$9
 export DIR="$(dirname "$(realpath "$0")")"
 
 if [ -s "/etc/foundationdb/fdb.cluster" ]

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -6,6 +6,7 @@ services:
     networks:
       - hostnet
     environment:
+      SIMPLYBLOCK_LOG_LEVEL: '$LOG_LEVEL'
       FDB_CLUSTER_FILE_CONTENTS: '$FDB_CLUSTER_FILE_CONTENTS'
       FDB_NETWORKING_MODE: 'container'
       FDB_CLUSTER_FILE: '/etc/foundationdb/fdb.cluster'
@@ -32,8 +33,9 @@ services:
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
     environment:
-      - FLASK_DEBUG=False
-      - FLASK_ENV=production
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
+      FLASK_DEBUG: "False"
+      FLASK_ENV: "production"
 
   StorageNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -45,6 +47,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   MgmtNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -56,6 +60,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CachingNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -67,6 +73,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   LVolStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -78,6 +86,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CachedLVolStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -89,18 +99,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
-
-  PortStatsCollector:
-    image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/port_stat_collector.py"
-    networks:
-      - hostnet
-    deploy:
-      mode: global
-      placement:
-        constraints: [node.role == worker]
-    volumes:
-      - "/etc/foundationdb:/etc/foundationdb"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   MainDistrEventCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -112,6 +112,8 @@ services:
         constraints: [node.role == manager]
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   HAProxy:
     image: haproxytech/haproxy-debian:latest
@@ -127,6 +129,8 @@ services:
       - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CapacityAndStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -138,6 +142,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CapacityMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -149,6 +155,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   HealthCheck:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -160,6 +168,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   DeviceMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -171,6 +181,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   LVolMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -182,10 +194,13 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CleanupFDB:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
     environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
       LOG_DELETION_INTERVAL: "${LOG_DELETION_INTERVAL}"
     command: "python simplyblock_core/workers/cleanup_foundationdb.py"
     deploy:
@@ -206,6 +221,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -217,6 +234,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerFailedMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -228,6 +247,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerNewDeviceMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -239,6 +260,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   NewDeviceDiscovery:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -250,6 +273,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksNodeAddRunner:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -261,6 +286,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
 
 volumes:

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/services/health_check_service.py

@@ -4,6 +4,7 @@ from datetime import datetime
 
 
 from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 from simplyblock_core import constants, kv_store, utils, distr_controller
@@ -107,12 +108,13 @@ while True:
                     snode.rpc_username, snode.rpc_password,
                     timeout=10, retry=1)
                 for remote_device in snode.remote_devices:
-                    ret = rpc_client.get_bdevs(remote_device.remote_bdev)
-                    if ret:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
-                    else:
-                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    node_remote_devices_check &= bool(ret)
+                    if db_controller.get_storage_device_by_id(remote_device.get_id()).status == NVMeDevice.STATUS_ONLINE:
+                        ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                        if ret:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                        else:
+                            logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                        node_remote_devices_check &= bool(ret)
 
                 if snode.jm_device:
                     jm_device = snode.jm_device

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/services/main_distr_event_collector.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+import datetime
 import threading
 import time
 
@@ -9,6 +10,7 @@ from simplyblock_core.models.lvol_model import LVol
 
 
 from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 
 
@@ -31,6 +33,17 @@ def process_device_event(event):
                     if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
                         logger.info(f"The storage device is not online, skipping. status: {dev.status}")
                         event.status = 'skipped'
+                        node_status_event = {
+                            "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
+                            "event_type": "device_status",
+                            "storage_ID": storage_id,
+                            "status": dev.status}
+                        events = {"events": [node_status_event]}
+                        snode = db_controller.get_storage_node_by_id(node_id)
+                        rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+                        ret = rpc_client.distr_status_events_update(events)
+                        if not ret:
+                            logger.warning("Failed to send event update")
                         return
 
                     device = dev
@@ -56,9 +69,27 @@ def process_device_event(event):
                     new_remote_devices.append(rem_dev)
             node.remote_devices = new_remote_devices
             node.write_to_db(db_controller.kv_store)
-            device_controller.device_set_unavailable(device_id)
+            node_status_event = {
+                "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
+                "event_type": "device_status",
+                "storage_ID": storage_id,
+                "status": NVMeDevice.STATUS_UNAVAILABLE}
+            events = {"events": [node_status_event]}
+            ret = rpc_client.distr_status_events_update(events)
+            if not ret:
+                logger.warning("Failed to send event update")
+
+            dev_node = db_controller.get_storage_node_by_id(device.node_id)
+            if dev_node.status == StorageNode.STATUS_ONLINE:
+                device_controller.device_set_io_error(device_id, True)
+                device_controller.device_set_unavailable(device_id)
 
         else:
+            if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]:
+                logger.info(f"Node is not online, skipping. status: {node.status}")
+                event.status = 'skipped'
+                return
+
             if event.message == 'SPDK_BDEV_EVENT_REMOVE':
                 if device.node_id == node_id:
                     logger.info(f"Removing storage id: {storage_id} from node: {node_id}")

{sbcli_dev-3.9.4 → sbcli_dev-3.9.6}/simplyblock_core/storage_node_ops.py

@@ -752,7 +752,7 @@ def _connect_to_remote_devs(this_node):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
 
     remote_devices = []
     # connect to remote devs
@@ -786,7 +786,7 @@ def _connect_to_remote_jm_devs(this_node, jm_ids=[]):
 
     rpc_client = RPCClient(
         this_node.mgmt_ip, this_node.rpc_port,
-        this_node.rpc_username, this_node.rpc_password, timeout=5, retry=2)
+        this_node.rpc_username, this_node.rpc_password, timeout=10, retry=2)
 
     node_bdevs = rpc_client.get_bdevs()
     node_bdev_names = [b['name'] for b in node_bdevs]
@@ -1640,7 +1640,7 @@ def restart_storage_node(
             logger.info(f"Device not found: {db_dev.get_id()}")
             db_dev.status = NVMeDevice.STATUS_REMOVED
             removed_devices.append(db_dev)
-            # distr_controller.send_dev_status_event(db_dev, db_dev.status)
+            distr_controller.send_dev_status_event(db_dev, db_dev.status)
 
     if snode.jm_device and "serial_number" in snode.jm_device.device_data_dict:
         known_devices_sn.append(snode.jm_device.device_data_dict['serial_number'])