sbcli-dev 17.2.6__tar.gz → 17.2.8__tar.gz

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.2.6
+Version: 17.2.8
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbcli-dev
-Version: 17.2.6
+Version: 17.2.8
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=17.2.6
+SIMPLY_BLOCK_VERSION=17.2.8
 
 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_core/scripts/prometheus.yml

@@ -4,10 +4,12 @@ global:
     monitor: 'codelab-monitor'
 
 scrape_configs:
-  - job_name: 'metricsgateway'
+  - job_name: 'cluster_metrics'
     static_configs:
-      - targets: ['pushgateway:9091']
-      
+      - targets: ['HAProxy:80']
+        labels:
+          metrics_path: '/cluster/metrics'
+
   - job_name: 'node'
     dns_sd_configs:
     - names:

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -92,11 +92,11 @@ def add_device_stats(cl, device, capacity_dict, stats_dict):
     for g in ng:
         v = g.replace("device_", "")
         if v in data:
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(data[v])
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), device_id=device.get_id()).set(data[v])
         elif v == "status_code":
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(device.get_status_code())
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), device_id=device.get_id()).set(device.get_status_code())
         elif v == "health_check":
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id(), device_id=device.get_id()).set(device.health_check)
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id(), device_id=device.get_id()).set(device.health_check)
 
 
     return stat_obj
@@ -137,11 +137,11 @@ def add_node_stats(node, records):
     for g in ng:
         v = g.replace("snode_", "")
         if v in data:
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(data[v])
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id()).set(data[v])
         elif v == "status_code":
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(node.get_status_code())
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id()).set(node.get_status_code())
         elif v == "health_check":
-            ng[g].labels(cluster=cl.get_id(), node_ip=node.get_id()).set(node.health_check)
+            ng[g].labels(cluster=cl.get_id(), snode=node.get_id()).set(node.health_check)
 
     return stat_obj
 
@@ -183,9 +183,9 @@ def add_cluster_stats(cl, records):
     return stat_obj
 
 
-if os.path.exists(PROMETHEUS_MULTIPROC_DIR):
-    shell_utils.run_command(f"rm -rf {PROMETHEUS_MULTIPROC_DIR}")
-os.makedirs(PROMETHEUS_MULTIPROC_DIR, exist_ok=True)
+if not os.path.exists(PROMETHEUS_MULTIPROC_DIR):
+    shell_utils.run_command(f"mkdir -p {PROMETHEUS_MULTIPROC_DIR}")
+
 
 registry = CollectorRegistry()
 multiprocess.MultiProcessCollector(registry)
@@ -224,37 +224,19 @@ io_stats_keys = [
 ng = {}
 cg = {}
 dg = {}
-lg = {}
-pg = {}
 
 def get_device_metrics():
     global dg
     if not dg:
-        labels = ['cluster', "node_ip", "device_id"]
+        labels = ['cluster', "snode", "device_id"]
         for k in io_stats_keys + ["status_code", "health_check"]:
             dg["device_" + k] = Gauge("device_" + k, "device_" + k, labelnames=labels, registry=registry)
     return dg
 
-def get_lvol_metrics():
-    global lg
-    if not lg:
-        labels = ['cluster', "pool", "lvol"]
-        for k in io_stats_keys + ["status_code", "health_check"]:
-            lg["lvol_" + k] = Gauge("lvol_" + k, "lvol_" + k, labelnames=labels, registry=registry)
-    return lg
-
-def get_pool_metrics():
-    global pg
-    if not pg:
-        labels = ['cluster', "pool", "name"]
-        for k in io_stats_keys + ["status_code", "health_check"]:
-            pg["pool_" + k] = Gauge("pool_" + k, "pool_" + k, labelnames=labels, registry=registry)
-    return pg
-
 def get_snode_metrics():
     global ng
     if not ng:
-        labels = ['cluster', "node_ip"]
+        labels = ['cluster', "snode"]
         for k in io_stats_keys + ["status_code", "health_check"]:
             ng["snode_" + k] = Gauge("snode_" + k, "snode_" + k, labelnames=labels, registry=registry)
     return ng

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_core/services/lvol_stat_collector.py

@@ -1,20 +1,80 @@
 # coding=utf-8
 import time
+import os
 
-from simplyblock_core import constants, db_controller, utils
+from simplyblock_core import constants, db_controller, utils, shell_utils
 from simplyblock_core.controllers import lvol_events
 from simplyblock_core.models.cluster import Cluster
 from simplyblock_core.models.lvol_model import LVol
 from simplyblock_core.models.stats import LVolStatObject, PoolStatObject
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
-from simplyblock_core.services.capacity_and_stats_collector import get_lvol_metrics, get_pool_metrics
+PROMETHEUS_MULTIPROC_DIR = constants.PROMETHEUS_MULTIPROC_DIR
+os.environ["PROMETHEUS_MULTIPROC_DIR"] = PROMETHEUS_MULTIPROC_DIR
+from prometheus_client import Gauge, CollectorRegistry, multiprocess
+
+if not os.path.exists(PROMETHEUS_MULTIPROC_DIR):
+    shell_utils.run_command(f"mkdir -p {PROMETHEUS_MULTIPROC_DIR}")
 
-logger = utils.get_logger(__name__)
 
+logger = utils.get_logger(__name__)
 
 last_object_record = {}
 
+registry = CollectorRegistry()
+multiprocess.MultiProcessCollector(registry)
+
+io_stats_keys = [
+    "date",
+    "read_bytes",
+    "read_bytes_ps",
+    "read_io_ps",
+    "read_io",
+    "read_latency_ps",
+    "write_bytes",
+    "write_bytes_ps",
+    "write_io",
+    "write_io_ps",
+    "write_latency_ps",
+    "size_total",
+    "size_prov",
+    "size_used",
+    "size_free",
+    "size_util",
+    "size_prov_util",
+    "read_latency_ticks",
+    "record_duration",
+    "record_end_time",
+    "record_start_time",
+    "unmap_bytes",
+    "unmap_bytes_ps",
+    "unmap_io",
+    "unmap_io_ps",
+    "unmap_latency_ps",
+    "unmap_latency_ticks",
+    "write_latency_ticks",
+]
+
+lg = {}
+pg = {}
+
+
+def get_lvol_metrics():
+    global lg
+    if not lg:
+        labels = ['cluster', "pool", "lvol"]
+        for k in io_stats_keys + ["status_code", "health_check"]:
+            lg["lvol_" + k] = Gauge("lvol_" + k, "lvol_" + k, labelnames=labels, registry=registry)
+    return lg
+
+def get_pool_metrics():
+    global pg
+    if not pg:
+        labels = ['cluster', "pool", "name"]
+        for k in io_stats_keys + ["status_code"]:
+            pg["pool_" + k] = Gauge("pool_" + k, "pool_" + k, labelnames=labels, registry=registry)
+    return pg
+
 
 def sum_stats(stats_list):
     if not stats_list or len(stats_list) == 0:
@@ -193,9 +253,6 @@ def add_pool_stats(pool, records):
             ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(data[v])
         elif v == "status_code":
             ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(pool.get_status_code())
-        elif v == "health_check":
-            ng[g].labels(cluster=cluster.get_id(), name=pool.pool_name, pool=pool.get_id()).set(pool.health_check)
-
 
     return stat_obj
 

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_core/services/spdk_http_proxy_server.py

@@ -50,7 +50,7 @@ def rpc_call(req):
     response = None
 
     while not closed:
-        newdata = sock.recv(1024)
+        newdata = sock.recv(1024*1024*1024)
         if newdata == b'':
             closed = True
         buf += newdata.decode('ascii')
@@ -151,7 +151,7 @@ def run_server(host, port, user, password, is_threading_enabled=False):
 
 TIMEOUT = int(get_env_var("TIMEOUT", is_required=False, default=60*5))
 is_threading_enabled = get_env_var("MULTI_THREADING_ENABLED", is_required=False, default=False)
-server_ip = get_env_var("SERVER_IP", is_required=True)
+server_ip = get_env_var("SERVER_IP", is_required=True, default="")
 rpc_port = get_env_var("RPC_PORT", is_required=True)
 rpc_username = get_env_var("RPC_USERNAME", is_required=True)
 rpc_password = get_env_var("RPC_PASSWORD", is_required=True)

{sbcli_dev-17.2.6 → sbcli_dev-17.2.8}/simplyblock_web/blueprints/web_api_metrics.py

@@ -5,7 +5,7 @@ import logging
 import os
 
 from flask import Blueprint
-from simplyblock_core import constants
+from simplyblock_core import constants, shell_utils
 
 PROMETHEUS_MULTIPROC_DIR = constants.PROMETHEUS_MULTIPROC_DIR
 os.environ["PROMETHEUS_MULTIPROC_DIR"] = PROMETHEUS_MULTIPROC_DIR
@@ -15,6 +15,10 @@ from flask import Response
 from prometheus_client import CollectorRegistry
 
 
+if not os.path.exists(PROMETHEUS_MULTIPROC_DIR):
+    shell_utils.run_command(f"mkdir -p {PROMETHEUS_MULTIPROC_DIR}")
+
+
 logger = logging.getLogger(__name__)
 
 bp = Blueprint("metrics", __name__)