sbcli-pre 1.1.3__zip → 1.1.5__zip

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.1.3
+Version: 1.1.5
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=1.1.3
+SIMPLY_BLOCK_VERSION=1.1.5
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
 

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.1.3
+Version: 1.1.5
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/sbcli_pre.egg-info/SOURCES.txt

@@ -73,7 +73,7 @@ simplyblock_core/scripts/prometheus.yml
 simplyblock_core/scripts/run_ssh.sh
 simplyblock_core/scripts/set_db_config.sh
 simplyblock_core/scripts/stack_deploy_wait.sh
-simplyblock_core/scripts/alerting/alert_resources.yaml
+simplyblock_core/scripts/alerting/alert_resources.yaml.j2
 simplyblock_core/scripts/alerting/alert_rules.yaml
 simplyblock_core/scripts/dashboards/cluster.json
 simplyblock_core/scripts/dashboards/devices.json

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_cli/cli.py

@@ -235,6 +235,10 @@ class CLIWrapper:
                                  dest='log_del_interval', default='7d')
         sub_command.add_argument("--metrics-retention-period", help='retention period for prometheus metrics, default: 7d',
                                  dest='metrics_retention_period', default='7d')
+        sub_command.add_argument("--contact-point", help='the email or slack webhook url to be used for alerting',
+                                 dest='contact_point', default='')
+        sub_command.add_argument("--grafana-endpoint", help='the endpoint url for grafana',
+                                 dest='grafana_endpoint', default='')
 
         # add cluster
         sub_command = self.add_sub_command(subparser, 'add', 'Add new cluster')
@@ -626,9 +630,6 @@ class CLIWrapper:
             elif sub_command == "deploy-cleaner":
                 ret = storage_ops.deploy_cleaner()
 
-            elif sub_command == "add":
-                ret = self.storage_node_add(args)
-
             elif sub_command == "add-node":
                 cluster_id = args.cluster_id
                 node_ip = args.node_ip
@@ -1098,14 +1099,6 @@ class CLIWrapper:
         out = storage_ops.list_storage_nodes(self.db_store, args.json)
         return out
 
-    def storage_node_add(self, args):
-        cluster_id = args.cluster_id
-        ifname = args.ifname
-        data_nics = args.data_nics
-        # TODO: Validate the inputs
-        out = storage_ops.add_storage_node(cluster_id, ifname, data_nics)
-        return out
-
     def storage_node_list_devices(self, args):
         node_id = args.node_id
         sort = args.sort
@@ -1137,11 +1130,13 @@ class CLIWrapper:
         ifname = args.ifname
         log_del_interval = args.log_del_interval
         metrics_retention_period = args.metrics_retention_period
+        contact_point = args.contact_point
+        grafana_endpoint = args.grafana_endpoint
 
         return cluster_ops.create_cluster(
             blk_size, page_size_in_blocks,
             CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
-            ifname, log_del_interval, metrics_retention_period)
+            ifname, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint)
 
     def query_yes_no(self, question, default="yes"):
         """Ask a yes/no question via raw_input() and return their answer.

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/cluster_ops.py

@@ -2,11 +2,16 @@
 import json
 import logging
 import os
+import re
+import tempfile
+import shutil
+import subprocess
 import time
 import uuid
 
 import docker
 import requests
+from jinja2 import Environment, FileSystemLoader
 
 from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops
 from simplyblock_core.controllers import cluster_events, device_controller
@@ -16,7 +21,7 @@ from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 logger = logging.getLogger()
-
+TOP_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 def _add_grafana_dashboards(username, password, cluster_ip):
     url = f"http://{username}:{password}@{cluster_ip}/grafana/api/dashboards/import"
@@ -69,7 +74,8 @@ def _add_graylog_input(cluster_ip, password):
 
 
 def create_cluster(blk_size, page_size_in_blocks, cli_pass,
-                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period):
+                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period,
+                    contact_point, grafana_endpoint):
     logger.info("Installing dependencies...")
     ret = scripts.install_deps()
     logger.info("Installing dependencies > Done")
@@ -124,6 +130,43 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     if prov_cap_crit and prov_cap_crit > 0:
         c.prov_cap_crit = prov_cap_crit
 
+    alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")
+    alert_resources_file = "alert_resources.yaml"
+
+    env = Environment(loader=FileSystemLoader(alerts_template_folder), trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template(f'{alert_resources_file}.j2')
+
+    slack_pattern = re.compile(r"https://hooks\.slack\.com/services/\S+")
+    email_pattern = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+
+    if slack_pattern.match(contact_point):
+        ALERT_TYPE = "slack"
+    elif email_pattern.match(contact_point):
+        ALERT_TYPE = "email"
+    else:
+        ALERT_TYPE = "slack" 
+
+    values = {
+        'CONTACT_POINT': contact_point,
+        'GRAFANA_ENDPOINT': grafana_endpoint,
+        'ALERT_TYPE': ALERT_TYPE,
+    }
+
+    temp_dir = tempfile.mkdtemp()
+
+    temp_file_path = os.path.join(temp_dir, alert_resources_file)
+    with open(temp_file_path, 'w') as file:
+        file.write(template.render(values))
+
+    destination_file_path = os.path.join(alerts_template_folder, alert_resources_file)
+    try:
+        subprocess.run(['sudo', '-v'], check=True) # sudo -v checks if the current user has sudo permissions
+        subprocess.run(['sudo', 'mv', temp_file_path, destination_file_path], check=True)
+        print(f"File moved to {destination_file_path} successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred: {e}")
+    shutil.rmtree(temp_dir)
+
     logger.info("Deploying swarm stack ...")
     ret = scripts.deploy_stack(cli_pass, DEV_IP, constants.SIMPLY_BLOCK_DOCKER_IMAGE, c.secret, c.uuid, log_del_interval, metrics_retention_period)
     logger.info("Deploying swarm stack > Done")

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/constants.py

@@ -20,7 +20,7 @@ DEV_MONITOR_INTERVAL_SEC = 10
 DEV_STAT_COLLECTOR_INTERVAL_SEC = 2
 PROT_STAT_COLLECTOR_INTERVAL_SEC = 2
 DISTR_EVENT_COLLECTOR_INTERVAL_SEC = 2
-DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS = 20
+DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS = 10
 CAP_MONITOR_INTERVAL_SEC = 30
 SSD_VENDOR_WHITE_LIST = ["1d0f:cd01", "1d0f:cd00"]
 

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/controllers/health_controller.py

@@ -69,7 +69,7 @@ def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password):
     try:
         rpc_client = RPCClient(
             rpc_ip, rpc_port, rpc_username, rpc_password,
-            timeout=3, retry=1)
+            timeout=10, retry=1)
         ret = rpc_client.get_version()
         if ret:
             logger.debug(f"SPDK version: {ret['version']}")

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/rpc_client.py

@@ -434,8 +434,9 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 0,
+            "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
+            "ctrlr_loss_timeout_sec": 2,
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -483,9 +484,9 @@ class RPCClient:
         params = {
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
-            "ctrlr_loss_timeout_sec": -1,
-            "fast_io_fail_timeout_sec": 5,
-            "reconnect_delay_sec": 5,
+            "ctrlr_loss_timeout_sec": 2,
+            "fast_io_fail_timeout_sec": 1,
+            "reconnect_delay_sec": 1,
             "keep_alive_timeout_ms": 200,
             "transport_ack_timeout": 7,
             "timeout_us": 100000

sbcli_pre-1.1.3/simplyblock_core/scripts/alerting/alert_resources.yaml → sbcli_pre-1.1.5/simplyblock_core/scripts/alerting/alert_resources.yaml.j2

@@ -12,15 +12,26 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: slack
+        type: {{ ALERT_TYPE }}
+        {% if ALERT_TYPE == 'slack' %}
         settings:
           username: grafana_bot
-          url: 'https://hooks.slack.com/services/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'
+          url: '{{ CONTACT_POINT }}'
           title: |
-            {{ template "slack.title" . }}
+            {{ '{{' }} template "slack.title" . {{ '}}' }}
           text: |
-            {{ template "slack.message" . }}
+            {{ '{{' }} template "slack.message" . {{ '}}' }}
+        {% else %}
+        settings:
+          addresses: '{{ CONTACT_POINT }}'
+          subject: |
+            {{ '{{' }} template "email.subject" . {{ '}}' }}
+          body: |
+            {{ '{{' }} template "email.body" . {{ '}}' }}
+        {% endif %}
 
+{% if ALERT_TYPE == 'slack' %}
+{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -38,7 +49,9 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      *Explore logs:* https://grafanaURL.com/explore?orgId=1
+      {% endraw %}
+      *Explore logs:* {{ GRAFANA_ENDPOINT }}
+      {% raw %}
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -65,3 +78,39 @@ templates:
       {{ end }}
 
       {{- end }}
+{% endraw %}
+{% else %}
+{% raw %}
+  - orgId: 1
+    name: email.subject
+    template: |-
+      {{ define "email.subject" -}}
+      [{{ .Status | toUpper }}] Grafana Alert
+      {{- end -}}
+  - orgId: 1
+    name: email.body
+    template: |-
+      {{ define "email.body" -}}
+      Alert: {{ .Labels.alertname }}
+      {{ if .Annotations -}}
+      Summary: {{ .Annotations.summary}}
+      Description: {{ .Annotations.description }}
+      {{ end -}}
+      Log message: {{ index .Labels "message" }}
+      Explore logs: {{ GRAFANA_ENDPOINT }}
+      {{ if .DashboardURL -}}
+      Go to dashboard: {{ .DashboardURL }}
+      {{- end }}
+      {{ if .PanelURL -}}
+      Go to panel: {{ .PanelURL }}
+      {{- end }}
+      Details:
+      {{ range .Labels.SortedPairs -}}
+      - {{ .Name }}: `{{ .Value }}`
+      {{ end -}}
+      {{ if .SilenceURL -}}
+      Silence this alert: {{ .SilenceURL }}
+      {{- end }}
+      {{- end }}
+{% endraw %}
+{% endif %}

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/services/distr_event_collector.py

@@ -140,14 +140,13 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=3, retry=2
-    )
-    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
+        timeout=10, retry=2)
+
     try:
-        # events = client.distr_status_events_get()
-        events = client.distr_status_events_discard_then_get(0, num_of_events)
+        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
+
         if not events:
-            logger.error("Distr events empty")
+            logger.debug("no events found")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -161,8 +160,8 @@ while True:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
 
-        logger.info(f"Discarding events: {num_of_events}")
-        events = client.distr_status_events_discard_then_get(num_of_events, 0)
+            logger.info(f"Discarding events: {len(events)}")
+            client.distr_status_events_discard_then_get(len(events), 0)
 
     except Exception as e:
         logger.error("Failed to process distr events")

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/services/health_check_service.py

@@ -118,7 +118,7 @@ while True:
                 rpc_client = RPCClient(
                     snode.mgmt_ip, snode.rpc_port,
                     snode.rpc_username, snode.rpc_password,
-                    timeout=5, retry=3)
+                    timeout=10, retry=1)
                 for remote_device in snode.remote_devices:
                     ret = rpc_client.get_bdevs(remote_device.remote_bdev)
                     if ret:

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/services/storage_node_monitor.py

@@ -66,8 +66,8 @@ def get_cluster_target_status(cluster_id):
     logger.debug(f"online_devices: {online_devices}")
     logger.debug(f"offline_devices: {offline_devices}")
 
-    # if more than two affected modes then cluster is suspended
-    if affected_nodes > 2:
+    # if more than two affected nodes then cluster is suspended
+    if affected_nodes > 2 or offline_nodes > 2:
         return Cluster.STATUS_SUSPENDED
 
     # if any device goes offline then cluster is degraded
@@ -105,7 +105,7 @@ def update_cluster_status(cluster_id):
 
 def set_node_online(node):
     if node.status != StorageNode.STATUS_ONLINE:
-        storage_node_ops.set_node_status(snode, StorageNode.STATUS_ONLINE)
+        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE)
 
 
 def set_node_offline(node):

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/services/tasks_runner_restart.py

@@ -175,6 +175,7 @@ def task_runner_node(task):
     if ret:
         logger.info(f"Node restart succeeded")
 
+    time.sleep(5)
     if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
         logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "done"

{sbcli_pre-1.1.3 → sbcli_pre-1.1.5}/simplyblock_core/storage_node_ops.py

@@ -799,140 +799,6 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     return "Success"
 
 
-# Deprecated
-def add_storage_node(cluster_id, iface_name, data_nics):
-    db_controller = DBController()
-    kv_store = db_controller.kv_store
-
-    cluster = db_controller.get_cluster_by_id(cluster_id)
-    if not cluster:
-        logger.error("Cluster not found: %s", cluster_id)
-        return False
-
-    logger.info("Add Storage node")
-
-    hostname = utils.get_hostname()
-    snode = db_controller.get_storage_node_by_hostname(hostname)
-    if snode:
-        logger.error("Node already exists, try remove it first.")
-        exit(1)
-    else:
-        snode = StorageNode()
-        snode.uuid = str(uuid.uuid4())
-
-    mgmt_ip = _get_if_ip_address(iface_name)
-    system_id = utils.get_system_id()
-
-    BASE_NQN = cluster.nqn.split(":")[0]
-    subsystem_nqn = f"{BASE_NQN}:{hostname}"
-
-    if data_nics:
-        data_nics = _get_data_nics(data_nics)
-    else:
-        data_nics = _get_data_nics([iface_name])
-
-    rpc_user, rpc_pass = utils.generate_rpc_user_and_pass()
-
-    # creating storage node object
-    snode.status = StorageNode.STATUS_IN_CREATION
-    snode.baseboard_sn = utils.get_baseboard_sn()
-    snode.system_uuid = system_id
-    snode.hostname = hostname
-    snode.host_nqn = subsystem_nqn
-    snode.subsystem = subsystem_nqn
-    snode.data_nics = data_nics
-    snode.mgmt_ip = mgmt_ip
-    snode.rpc_port = constants.RPC_HTTP_PROXY_PORT
-    snode.rpc_username = rpc_user
-    snode.rpc_password = rpc_pass
-    snode.cluster_id = cluster_id
-    snode.write_to_db(kv_store)
-
-    # creating RPCClient instance
-    rpc_client = RPCClient(
-        snode.mgmt_ip,
-        snode.rpc_port,
-        snode.rpc_username,
-        snode.rpc_password)
-
-    logger.info("Getting nvme devices")
-    devs = get_nvme_devices()
-    logger.debug(devs)
-    pcies = [d[0] for d in devs]
-    nvme_devs = addNvmeDevices(cluster, rpc_client, pcies, snode)
-    if not nvme_devs:
-        logger.error("No NVMe devices was found!")
-
-    logger.debug(nvme_devs)
-    snode.nvme_devices = nvme_devs
-
-    # Set device cluster order
-    dev_order = get_next_cluster_device_order(db_controller, cluster_id)
-    for index, nvme in enumerate(snode.nvme_devices):
-        nvme.cluster_device_order = dev_order
-        dev_order += 1
-    snode.write_to_db(db_controller.kv_store)
-
-    # prepare devices
-    # _prepare_cluster_devices(snode)
-
-    logger.info("Connecting to remote devices")
-    remote_devices = _connect_to_remote_devs(snode)
-    snode.remote_devices = remote_devices
-
-    logger.info("Setting node status to Active")
-    snode.status = StorageNode.STATUS_ONLINE
-    snode.write_to_db(kv_store)
-
-    # make other nodes connect to the new devices
-    logger.info("Make other nodes connect to the new devices")
-    snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
-    for node_index, node in enumerate(snodes):
-        if node.get_id() == snode.get_id():
-            continue
-        logger.info(f"Connecting to node: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password)
-        count = 0
-        for dev in snode.nvme_devices:
-            name = f"remote_{dev.alceml_bdev}"
-            ret = rpc_client.bdev_nvme_attach_controller_tcp(name, dev.nvmf_nqn, dev.nvmf_ip, dev.nvmf_port)
-            if not ret:
-                logger.error(f"Failed to connect to device: {name}")
-                continue
-
-            dev.remote_bdev = f"{name}n1"
-            idx = -1
-            for i, d in enumerate(node.remote_devices):
-                if d.get_id() == dev.get_id():
-                    idx = i
-                    break
-            if idx >= 0:
-                node.remote_devices[idx] = dev
-            else:
-                node.remote_devices.append(dev)
-            count += 1
-        node.write_to_db(kv_store)
-        logger.info(f"connected to devices count: {count}")
-
-    logger.info("Sending cluster map")
-    ret = distr_controller.send_cluster_map_to_node(snode)
-    if not ret:
-        return False, "Failed to send cluster map"
-    ret = distr_controller.send_cluster_map_add_node(snode)
-    if not ret:
-        return False, "Failed to send cluster map add node"
-    time.sleep(3)
-
-    logger.info("Sending cluster event updates")
-    distr_controller.send_node_status_event(snode, StorageNode.STATUS_ONLINE)
-
-    for dev in snode.nvme_devices:
-        distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE)
-
-    logger.info("Done")
-    return "Success"
-
-
 def delete_storage_node(node_id):
     db_controller = DBController()
     snode = db_controller.get_storage_node_by_id(node_id)