sbcli-pre 1.1.2__zip → 1.1.4__zip

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.1.2
+Version: 1.1.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=1.1.2
+SIMPLY_BLOCK_VERSION=1.1.4
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
 

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.1.2
+Version: 1.1.4
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/sbcli_pre.egg-info/SOURCES.txt

@@ -73,7 +73,7 @@ simplyblock_core/scripts/prometheus.yml
 simplyblock_core/scripts/run_ssh.sh
 simplyblock_core/scripts/set_db_config.sh
 simplyblock_core/scripts/stack_deploy_wait.sh
-simplyblock_core/scripts/alerting/alert_resources.yaml
+simplyblock_core/scripts/alerting/alert_resources.yaml.j2
 simplyblock_core/scripts/alerting/alert_rules.yaml
 simplyblock_core/scripts/dashboards/cluster.json
 simplyblock_core/scripts/dashboards/devices.json

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_cli/cli.py

@@ -235,6 +235,10 @@ class CLIWrapper:
                                  dest='log_del_interval', default='7d')
         sub_command.add_argument("--metrics-retention-period", help='retention period for prometheus metrics, default: 7d',
                                  dest='metrics_retention_period', default='7d')
+        sub_command.add_argument("--contact-point", help='the email or slack webhook url to be used for alerting',
+                                 dest='contact_point', default='')
+        sub_command.add_argument("--grafana-endpoint", help='the endpoint url for grafana',
+                                 dest='grafana_endpoint', default='')
 
         # add cluster
         sub_command = self.add_sub_command(subparser, 'add', 'Add new cluster')
@@ -1137,11 +1141,13 @@ class CLIWrapper:
         ifname = args.ifname
         log_del_interval = args.log_del_interval
         metrics_retention_period = args.metrics_retention_period
+        contact_point = args.contact_point
+        grafana_endpoint = args.grafana_endpoint
 
         return cluster_ops.create_cluster(
             blk_size, page_size_in_blocks,
             CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
-            ifname, log_del_interval, metrics_retention_period)
+            ifname, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint)
 
     def query_yes_no(self, question, default="yes"):
         """Ask a yes/no question via raw_input() and return their answer.

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/cluster_ops.py

@@ -2,11 +2,16 @@
 import json
 import logging
 import os
+import re
+import tempfile
+import shutil
+import subprocess
 import time
 import uuid
 
 import docker
 import requests
+from jinja2 import Environment, FileSystemLoader
 
 from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops
 from simplyblock_core.controllers import cluster_events, device_controller
@@ -16,7 +21,7 @@ from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 logger = logging.getLogger()
-
+TOP_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 def _add_grafana_dashboards(username, password, cluster_ip):
     url = f"http://{username}:{password}@{cluster_ip}/grafana/api/dashboards/import"
@@ -69,7 +74,8 @@ def _add_graylog_input(cluster_ip, password):
 
 
 def create_cluster(blk_size, page_size_in_blocks, cli_pass,
-                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period):
+                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period,
+                    contact_point, grafana_endpoint):
     logger.info("Installing dependencies...")
     ret = scripts.install_deps()
     logger.info("Installing dependencies > Done")
@@ -124,6 +130,43 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     if prov_cap_crit and prov_cap_crit > 0:
         c.prov_cap_crit = prov_cap_crit
 
+    alerts_template_folder = os.path.join(TOP_DIR, "simplyblock_core/scripts/alerting/")
+    alert_resources_file = "alert_resources.yaml"
+
+    env = Environment(loader=FileSystemLoader(alerts_template_folder), trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template(f'{alert_resources_file}.j2')
+
+    slack_pattern = re.compile(r"https://hooks\.slack\.com/services/\S+")
+    email_pattern = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+
+    if slack_pattern.match(contact_point):
+        ALERT_TYPE = "slack"
+    elif email_pattern.match(contact_point):
+        ALERT_TYPE = "email"
+    else:
+        ALERT_TYPE = "slack" 
+
+    values = {
+        'CONTACT_POINT': contact_point,
+        'GRAFANA_ENDPOINT': grafana_endpoint,
+        'ALERT_TYPE': ALERT_TYPE,
+    }
+
+    temp_dir = tempfile.mkdtemp()
+
+    temp_file_path = os.path.join(temp_dir, alert_resources_file)
+    with open(temp_file_path, 'w') as file:
+        file.write(template.render(values))
+
+    destination_file_path = os.path.join(alerts_template_folder, alert_resources_file)
+    try:
+        subprocess.run(['sudo', '-v'], check=True) # sudo -v checks if the current user has sudo permissions
+        subprocess.run(['sudo', 'mv', temp_file_path, destination_file_path], check=True)
+        print(f"File moved to {destination_file_path} successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred: {e}")
+    shutil.rmtree(temp_dir)
+
     logger.info("Deploying swarm stack ...")
     ret = scripts.deploy_stack(cli_pass, DEV_IP, constants.SIMPLY_BLOCK_DOCKER_IMAGE, c.secret, c.uuid, log_del_interval, metrics_retention_period)
     logger.info("Deploying swarm stack > Done")

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/constants.py

@@ -20,7 +20,7 @@ DEV_MONITOR_INTERVAL_SEC = 10
 DEV_STAT_COLLECTOR_INTERVAL_SEC = 2
 PROT_STAT_COLLECTOR_INTERVAL_SEC = 2
 DISTR_EVENT_COLLECTOR_INTERVAL_SEC = 2
-DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS = 20
+DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS = 10
 CAP_MONITOR_INTERVAL_SEC = 30
 SSD_VENDOR_WHITE_LIST = ["1d0f:cd01", "1d0f:cd00"]
 

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/controllers/health_controller.py

@@ -69,7 +69,7 @@ def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password):
     try:
         rpc_client = RPCClient(
             rpc_ip, rpc_port, rpc_username, rpc_password,
-            timeout=3, retry=1)
+            timeout=10, retry=1)
         ret = rpc_client.get_version()
         if ret:
             logger.debug(f"SPDK version: {ret['version']}")

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/rpc_client.py

@@ -434,8 +434,9 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 0,
+            "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
+            "ctrlr_loss_timeout_sec": 2,
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -483,9 +484,9 @@ class RPCClient:
         params = {
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
-            "ctrlr_loss_timeout_sec": -1,
-            "fast_io_fail_timeout_sec": 5,
-            "reconnect_delay_sec": 5,
+            "ctrlr_loss_timeout_sec": 2,
+            "fast_io_fail_timeout_sec": 1,
+            "reconnect_delay_sec": 1,
             "keep_alive_timeout_ms": 200,
             "transport_ack_timeout": 7,
             "timeout_us": 100000

sbcli_pre-1.1.2/simplyblock_core/scripts/alerting/alert_resources.yaml → sbcli_pre-1.1.4/simplyblock_core/scripts/alerting/alert_resources.yaml.j2

@@ -12,15 +12,26 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: slack
+        type: {{ ALERT_TYPE }}
+        {% if ALERT_TYPE == 'slack' %}
         settings:
           username: grafana_bot
-          url: 'https://hooks.slack.com/services/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'
+          url: '{{ CONTACT_POINT }}'
           title: |
-            {{ template "slack.title" . }}
+            {{ '{{' }} template "slack.title" . {{ '}}' }}
           text: |
-            {{ template "slack.message" . }}
+            {{ '{{' }} template "slack.message" . {{ '}}' }}
+        {% else %}
+        settings:
+          addresses: '{{ CONTACT_POINT }}'
+          subject: |
+            {{ '{{' }} template "email.subject" . {{ '}}' }}
+          body: |
+            {{ '{{' }} template "email.body" . {{ '}}' }}
+        {% endif %}
 
+{% if ALERT_TYPE == 'slack' %}
+{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -38,7 +49,9 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      *Explore logs:* https://grafanaURL.com/explore?orgId=1
+      {% endraw %}
+      *Explore logs:* {{ GRAFANA_ENDPOINT }}
+      {% raw %}
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -65,3 +78,39 @@ templates:
       {{ end }}
 
       {{- end }}
+{% endraw %}
+{% else %}
+{% raw %}
+  - orgId: 1
+    name: email.subject
+    template: |-
+      {{ define "email.subject" -}}
+      [{{ .Status | toUpper }}] Grafana Alert
+      {{- end -}}
+  - orgId: 1
+    name: email.body
+    template: |-
+      {{ define "email.body" -}}
+      Alert: {{ .Labels.alertname }}
+      {{ if .Annotations -}}
+      Summary: {{ .Annotations.summary}}
+      Description: {{ .Annotations.description }}
+      {{ end -}}
+      Log message: {{ index .Labels "message" }}
+      Explore logs: {{ GRAFANA_ENDPOINT }}
+      {{ if .DashboardURL -}}
+      Go to dashboard: {{ .DashboardURL }}
+      {{- end }}
+      {{ if .PanelURL -}}
+      Go to panel: {{ .PanelURL }}
+      {{- end }}
+      Details:
+      {{ range .Labels.SortedPairs -}}
+      - {{ .Name }}: `{{ .Value }}`
+      {{ end -}}
+      {{ if .SilenceURL -}}
+      Silence this alert: {{ .SilenceURL }}
+      {{- end }}
+      {{- end }}
+{% endraw %}
+{% endif %}

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/services/distr_event_collector.py

@@ -140,14 +140,13 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=3, retry=2
-    )
-    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
+        timeout=10, retry=2)
+
     try:
-        # events = client.distr_status_events_get()
-        events = client.distr_status_events_discard_then_get(0, num_of_events)
+        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
+
         if not events:
-            logger.error("Distr events empty")
+            logger.debug("no events found")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -161,8 +160,8 @@ while True:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
 
-        logger.info(f"Discarding events: {num_of_events}")
-        events = client.distr_status_events_discard_then_get(num_of_events, 0)
+            logger.info(f"Discarding events: {len(events)}")
+            client.distr_status_events_discard_then_get(len(events), 0)
 
     except Exception as e:
         logger.error("Failed to process distr events")

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/services/health_check_service.py

@@ -118,7 +118,7 @@ while True:
                 rpc_client = RPCClient(
                     snode.mgmt_ip, snode.rpc_port,
                     snode.rpc_username, snode.rpc_password,
-                    timeout=5, retry=3)
+                    timeout=10, retry=1)
                 for remote_device in snode.remote_devices:
                     ret = rpc_client.get_bdevs(remote_device.remote_bdev)
                     if ret:

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/services/storage_node_monitor.py

@@ -66,8 +66,8 @@ def get_cluster_target_status(cluster_id):
     logger.debug(f"online_devices: {online_devices}")
     logger.debug(f"offline_devices: {offline_devices}")
 
-    # if more than two affected modes then cluster is suspended
-    if affected_nodes > 2:
+    # if more than two affected nodes then cluster is suspended
+    if affected_nodes > 2 or offline_nodes > 2:
         return Cluster.STATUS_SUSPENDED
 
     # if any device goes offline then cluster is degraded
@@ -105,7 +105,7 @@ def update_cluster_status(cluster_id):
 
 def set_node_online(node):
     if node.status != StorageNode.STATUS_ONLINE:
-        storage_node_ops.set_node_status(snode, StorageNode.STATUS_ONLINE)
+        storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE)
 
 
 def set_node_offline(node):

{sbcli_pre-1.1.2 → sbcli_pre-1.1.4}/simplyblock_core/services/tasks_runner_restart.py

@@ -150,9 +150,8 @@ def task_runner_node(task):
         task.write_to_db(db_controller.kv_store)
         return True
 
-    # if _get_node_unavailable_devices_count(node.get_id()) == 0:
-    if node.status == StorageNode.STATUS_ONLINE:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "Node is online"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)
@@ -168,7 +167,7 @@ def task_runner_node(task):
     ret = storage_node_ops.shutdown_storage_node(node.get_id(), force=True)
     if ret:
         logger.info(f"Node shutdown succeeded")
-    time.sleep(5)
+    time.sleep(3)
 
     # resetting node
     logger.info(f"Restart node {node.get_id()}")
@@ -176,9 +175,9 @@ def task_runner_node(task):
     if ret:
         logger.info(f"Node restart succeeded")
 
-    # if _get_node_unavailable_devices_count(node.get_id()) == 0:
-    if node.status == StorageNode.STATUS_ONLINE:
-        logger.info(f"Node is online: {node.get_id()}, no restart needed")
+    time.sleep(5)
+    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+        logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE
         task.write_to_db(db_controller.kv_store)