sbcli-dev 17.2.4__tar.gz → 17.2.6__tar.gz

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.2.4
+Version: 17.2.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.2.4
+Version: 17.2.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/sbcli_dev.egg-info/SOURCES.txt

@@ -138,6 +138,7 @@ simplyblock_web/blueprints/web_api_cluster.py
 simplyblock_web/blueprints/web_api_deployer.py
 simplyblock_web/blueprints/web_api_device.py
 simplyblock_web/blueprints/web_api_lvol.py
+simplyblock_web/blueprints/web_api_metrics.py
 simplyblock_web/blueprints/web_api_mgmt_node.py
 simplyblock_web/blueprints/web_api_pool.py
 simplyblock_web/blueprints/web_api_snapshot.py

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/constants.py

@@ -132,3 +132,4 @@ HA_JM_COUNT=3
 SENTRY_SDK_DNS = "https://745047b017ac424b4173550e19910fb7@o4508953941311488.ingest.de.sentry.io/4508996361584720"
 ONE_KB = 1024
 TEMP_CORES_FILE = "/etc/simplyblock/tmp_cores_config"
+PROMETHEUS_MULTIPROC_DIR = "/etc/simplyblock/metrics"

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/controllers/lvol_controller.py

@@ -806,7 +806,6 @@ def _remove_bdev_stack(bdev_stack, rpc_client):
             logger.error(f"Failed to delete BDev {name}")
 
         bdev['status'] = 'deleted'
-        time.sleep(1)
     return True
 
 
@@ -818,7 +817,7 @@ def delete_lvol_from_node(lvol_id, node_id, clear_data=True):
         return True
 
     logger.info(f"Deleting LVol:{lvol.get_id()} from node:{snode.get_id()}")
-    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+    rpc_client = RPCClient(snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, timeout=5, retry=2)
 
     # 1- remove subsystem
     if rpc_client.subsystem_list(lvol.nqn):
@@ -845,6 +844,11 @@ def delete_lvol(id_or_name, force_delete=False):
             logger.error(f"lvol not found: {id_or_name}")
             return False
 
+    if lvol.status == LVol.STATUS_IN_DELETION:
+        logger.info(f"lvol:{lvol.get_id()} status is in deletion")
+        if not force_delete:
+            return False
+
     pool = db_controller.get_pool_by_id(lvol.pool_uuid)
     if pool.status == Pool.STATUS_INACTIVE:
         logger.error(f"Pool is disabled")

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/db_controller.py

@@ -136,7 +136,7 @@ class DBController(metaclass=Singleton):
     def get_lvols(self, cluster_id=None) -> List[LVol]:
         lvols = self.get_all_lvols()
         if not cluster_id:
-            return sorted(lvols, key=lambda x: x.create_dt)
+            return lvols
 
         node_ids=[]
         cluster_lvols = []
@@ -147,7 +147,7 @@ class DBController(metaclass=Singleton):
             if lvol.node_id in node_ids:
                 cluster_lvols.append(lvol)
 
-        return sorted(cluster_lvols, key=lambda x: x.create_dt)
+        return cluster_lvols
 
     def get_all_lvols(self) -> List[LVol]:
         lvols = LVol().read_from_db(self.kv_store)
@@ -177,9 +177,9 @@ class DBController(metaclass=Singleton):
             return ret[0]
 
     def get_lvol_by_id(self, id) -> LVol:
-        for lvol in self.get_lvols():
-            if lvol.get_id() == id:
-                return lvol
+        lvols = LVol().read_from_db(self.kv_store, id=id)
+        if lvols:
+            return lvols[0]
 
     def get_lvol_by_name(self, lvol_name) -> LVol:
         for lvol in self.get_lvols():

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=17.2.4
+SIMPLY_BLOCK_VERSION=17.2.6
 
 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -60,6 +60,7 @@ services:
         constraints: [node.role == manager]
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
+      - "/etc/simplyblock:/etc/simplyblock"
     environment:
       SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
       FLASK_DEBUG: "False"
@@ -112,6 +113,7 @@ services:
         constraints: [node.role == manager]
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
+      - "/etc/simplyblock:/etc/simplyblock"
     networks:
       - hostnet
     environment:
@@ -169,6 +171,7 @@ services:
         constraints: [node.role == manager]
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
+      - "/etc/simplyblock:/etc/simplyblock"
     networks:
       - hostnet
     environment:

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -1,12 +1,16 @@
 # coding=utf-8
+import os
 import time
 
-from simplyblock_core import constants, db_controller, utils
+from simplyblock_core import constants, db_controller, utils, shell_utils
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
 from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject
 
+PROMETHEUS_MULTIPROC_DIR = constants.PROMETHEUS_MULTIPROC_DIR
+os.environ["PROMETHEUS_MULTIPROC_DIR"] = PROMETHEUS_MULTIPROC_DIR
+from prometheus_client import Gauge, CollectorRegistry, multiprocess
 
 logger = utils.get_logger(__name__)
 
@@ -83,6 +87,18 @@ def add_device_stats(cl, device, capacity_dict, stats_dict):
     stat_obj = DeviceStatObject(data=data)
     stat_obj.write_to_db(db_controller.kv_store)
     last_object_record[device.get_id()] = stat_obj
+
+    ng = get_device_metrics()
+    for g in ng:
+        v = g.replace("device_", "")
+        if v in data:
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(data[v])
+        elif v == "status_code":
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(device.get_status_code())
+        elif v == "health_check":
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(device.health_check)
+
+
     return stat_obj
 
 
@@ -116,6 +132,17 @@ def add_node_stats(node, records):
     })
     stat_obj = NodeStatObject(data=data)
     stat_obj.write_to_db(db_controller.kv_store)
+
+    ng = get_snode_metrics()
+    for g in ng:
+        v = g.replace("snode_", "")
+        if v in data:
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(data[v])
+        elif v == "status_code":
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(node.get_status_code())
+        elif v == "health_check":
+            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(node.health_check)
+
     return stat_obj
 
 
@@ -144,9 +171,103 @@ def add_cluster_stats(cl, records):
 
     stat_obj = ClusterStatObject(data=data)
     stat_obj.write_to_db(db_controller.kv_store)
+
+    ng = get_cluster_metrics()
+    for g in ng:
+        v = g.replace("cluster_", "")
+        if v in data:
+            ng[g].labels(cluster=cl.get_id()).set(data[v])
+        elif v == "status_code":
+            ng[g].labels(cluster=cl.get_id()).set(cl.get_status_code())
+
     return stat_obj
 
 
+if os.path.exists(PROMETHEUS_MULTIPROC_DIR):
+    shell_utils.run_command(f"rm -rf {PROMETHEUS_MULTIPROC_DIR}")
+os.makedirs(PROMETHEUS_MULTIPROC_DIR, exist_ok=True)
+
+registry = CollectorRegistry()
+multiprocess.MultiProcessCollector(registry)
+
+io_stats_keys = [
+    "date",
+    "read_bytes",
+    "read_bytes_ps",
+    "read_io_ps",
+    "read_io",
+    "read_latency_ps",
+    "write_bytes",
+    "write_bytes_ps",
+    "write_io",
+    "write_io_ps",
+    "write_latency_ps",
+    "size_total",
+    "size_prov",
+    "size_used",
+    "size_free",
+    "size_util",
+    "size_prov_util",
+    "read_latency_ticks",
+    "record_duration",
+    "record_end_time",
+    "record_start_time",
+    "unmap_bytes",
+    "unmap_bytes_ps",
+    "unmap_io",
+    "unmap_io_ps",
+    "unmap_latency_ps",
+    "unmap_latency_ticks",
+    "write_latency_ticks",
+]
+
+ng = {}
+cg = {}
+dg = {}
+lg = {}
+pg = {}
+
+def get_device_metrics():
+    global dg
+    if not dg:
+        labels = ['cluster', "node_ip", "device_id"]
+        for k in io_stats_keys + ["status_code", "health_check"]:
+            dg["device_" + k] = Gauge("device_" + k, "device_" + k, labelnames=labels, registry=registry)
+    return dg
+
+def get_lvol_metrics():
+    global lg
+    if not lg:
+        labels = ['cluster', "pool", "lvol"]
+        for k in io_stats_keys + ["status_code", "health_check"]:
+            lg["lvol_" + k] = Gauge("lvol_" + k, "lvol_" + k, labelnames=labels, registry=registry)
+    return lg
+
+def get_pool_metrics():
+    global pg
+    if not pg:
+        labels = ['cluster', "pool", "name"]
+        for k in io_stats_keys + ["status_code", "health_check"]:
+            pg["pool_" + k] = Gauge("pool_" + k, "pool_" + k, labelnames=labels, registry=registry)
+    return pg
+
+def get_snode_metrics():
+    global ng
+    if not ng:
+        labels = ['cluster', "node_ip"]
+        for k in io_stats_keys + ["status_code", "health_check"]:
+            ng["snode_" + k] = Gauge("snode_" + k, "snode_" + k, labelnames=labels, registry=registry)
+    return ng
+
+def get_cluster_metrics():
+    global cg
+    if not cg:
+        labels = ['cluster']
+        for k in io_stats_keys + ["status_code"]:
+            cg["cluster_" + k] = Gauge("cluster_" + k, "cluster_" + k, labelnames=labels, registry=registry)
+    return cg
+
+
 
 # get DB controller
 db_controller = db_controller.DBController()

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/services/lvol_stat_collector.py

@@ -8,7 +8,7 @@ from simplyblock_core.models.lvol_model import LVol
 from simplyblock_core.models.stats import LVolStatObject, PoolStatObject
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
-
+from simplyblock_core.services.capacity_and_stats_collector import get_lvol_metrics, get_pool_metrics
 
 logger = utils.get_logger(__name__)
 
@@ -48,7 +48,7 @@ def add_lvol_stats(cluster, lvol, stats_list, capacity_dict=None):
         size_used = 0
         lvol_dict = capacity_dict
         size_total = int(lvol_dict['num_blocks']*lvol_dict['block_size'])
-        cluster_size = cluster.distr_ndcs * cluster.page_size_in_blocks
+        cluster_size = cluster.page_size_in_blocks
         if "driver_specific" in lvol_dict and "lvol" in lvol_dict["driver_specific"]:
             num_allocated_clusters = lvol_dict["driver_specific"]["lvol"]["num_allocated_clusters"]
             size_used = int(num_allocated_clusters*cluster_size)
@@ -155,6 +155,17 @@ def add_lvol_stats(cluster, lvol, stats_list, capacity_dict=None):
     stat_obj = LVolStatObject(data=data)
     stat_obj.write_to_db(db_controller.kv_store)
     last_object_record[lvol.get_id()] = stat_obj
+
+    ng = get_lvol_metrics()
+    for g in ng:
+        v = g.replace("lvol_", "")
+        if v in data:
+            ng[g].labels(cluster=cluster.get_id(), lvol=lvol.get_id(), pool=lvol.pool_name).set(data[v])
+        elif v == "status_code":
+            ng[g].labels(cluster=cluster.get_id(), lvol=lvol.get_id(), pool=lvol.pool_name).set(lvol.get_status_code())
+        elif v == "health_check":
+            ng[g].labels(cluster=cluster.get_id(), lvol=lvol.get_id(), pool=lvol.pool_name).set(lvol.health_check)
+
     return stat_obj
 
 
@@ -174,6 +185,18 @@ def add_pool_stats(pool, records):
 
     stat_obj = PoolStatObject(data=data)
     stat_obj.write_to_db(db_controller.kv_store)
+
+    ng = get_pool_metrics()
+    for g in ng:
+        v = g.replace("pool_", "")
+        if v in data:
+            ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(data[v])
+        elif v == "status_code":
+            ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(pool.get_status_code())
+        elif v == "health_check":
+            ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(pool.health_check)
+
+
     return stat_obj
 
 

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/services/storage_node_monitor.py

@@ -261,7 +261,7 @@ while True:
             if snode.status == StorageNode.STATUS_SCHEDULABLE and not ping_check and not node_api_check:
                 continue
 
-            spdk_process = True
+            spdk_process = False
             if node_api_check:
                 # 3- check spdk_process
                 spdk_process = health_controller._check_spdk_process_up(snode.mgmt_ip, snode.rpc_port)

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_core/utils.py

@@ -115,7 +115,7 @@ _humanbytes_parameter = {
 
 def humanbytes(size: int, mode: str = 'si') -> str:
     """Return the given bytes as a human friendly including the appropriate unit."""
-    if not size:
+    if not size or size < 0:
         return '0 B'
 
     base, exponent, log, infix = _humanbytes_parameter[mode]

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_web/app.py

@@ -7,11 +7,9 @@ from flask import Flask
 import utils
 from blueprints import web_api_cluster, web_api_mgmt_node, web_api_device, \
     web_api_lvol, web_api_storage_node, web_api_pool, web_api_caching_node, \
-    web_api_snapshot, web_api_deployer, swagger_ui_blueprint
+    web_api_snapshot, web_api_deployer, swagger_ui_blueprint, web_api_metrics
 from auth_middleware import token_required
 from simplyblock_core import constants, utils as core_utils
-from werkzeug.middleware.dispatcher import DispatcherMiddleware
-from prometheus_client import make_wsgi_app
 
 logger = core_utils.get_logger(__name__)
 
@@ -35,6 +33,7 @@ app.register_blueprint(web_api_pool.bp)
 app.register_blueprint(web_api_caching_node.bp)
 app.register_blueprint(web_api_deployer.bp)
 app.register_blueprint(swagger_ui_blueprint.bp, url_prefix=swagger_ui_blueprint.SWAGGER_URL)
+app.register_blueprint(web_api_metrics.bp)
 
 
 @app.before_request
@@ -48,10 +47,6 @@ def status():
     return utils.get_response("Live")
 
 
-app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {
-    '/cluster/metrics': make_wsgi_app()
-})
-
 app.config['JSONIFY_PRETTYPRINT_REGULAR'] = True
 if __name__ == '__main__':
     logging.getLogger('werkzeug').setLevel(constants.LOG_WEB_LEVEL)

{sbcli_dev-17.2.4 → sbcli_dev-17.2.6}/simplyblock_web/auth_middleware.py

@@ -12,7 +12,8 @@ def token_required(f):
     @wraps(f)
     def decorated(*args, **kwargs):
 
-        if request.method == "GET" and request.path.startswith("/swagger"):
+        if request.method == "GET" and [request.path.startswith("/swagger") or
+                                        request.path.startswith("/cluster/metrics")]:
             return f(*args, **kwargs)
 
         cluster_id = None

sbcli_dev-17.2.6/simplyblock_web/blueprints/web_api_metrics.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# encoding: utf-8
+import json
+import logging
+import os
+
+from flask import Blueprint
+from simplyblock_core import constants
+
+PROMETHEUS_MULTIPROC_DIR = constants.PROMETHEUS_MULTIPROC_DIR
+os.environ["PROMETHEUS_MULTIPROC_DIR"] = PROMETHEUS_MULTIPROC_DIR
+
+from prometheus_client import generate_latest, multiprocess
+from flask import Response
+from prometheus_client import CollectorRegistry
+
+
+logger = logging.getLogger(__name__)
+
+bp = Blueprint("metrics", __name__)
+
+registry = CollectorRegistry()
+multiprocess.MultiProcessCollector(registry)
+
+@bp.route('/cluster/metrics', methods=['GET'])
+def get_data():
+    return Response(generate_latest(registry), mimetype=str('text/plain; version=0.0.4; charset=utf-8'))