sbcli-pre 1.2.3__zip → 1.2.5__zip

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/__init__.py

@@ -35,6 +35,10 @@ def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, log
     return __run_script(
         ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, graylog_password, cluster_id, log_del_interval, metrics_retention_period])
 
+def apply_dashboard(grafanaPassword):
+    return __run_script(
+        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'apply_dashboard.sh'), grafanaPassword])
+
 
 def deploy_cleaner():
     return __run_script(['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'clean_local_storage_deploy.sh')])

sbcli_pre-1.2.3/simplyblock_core/scripts/alerting/alert_resources.yaml.j2 → sbcli_pre-1.2.5/simplyblock_core/scripts/alerting/alert_resources.yaml

@@ -12,26 +12,15 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: {{ ALERT_TYPE }}
-        {% if ALERT_TYPE == 'slack' %}
+        type: slack
         settings:
           username: grafana_bot
-          url: '{{ CONTACT_POINT }}'
+          url: 'https://hooks.slack.com/services/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'
           title: |
-            {{ '{{' }} template "slack.title" . {{ '}}' }}
+            {{ template "slack.title" . }}
           text: |
-            {{ '{{' }} template "slack.message" . {{ '}}' }}
-        {% else %}
-        settings:
-          addresses: '{{ CONTACT_POINT }}'
-          subject: |
-            {{ '{{' }} template "email.subject" . {{ '}}' }}
-          body: |
-            {{ '{{' }} template "email.body" . {{ '}}' }}
-        {% endif %}
+            {{ template "slack.message" . }}
 
-{% if ALERT_TYPE == 'slack' %}
-{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -49,9 +38,7 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      {% endraw %}
-      *Explore logs:* {{ GRAFANA_ENDPOINT }}
-      {% raw %}
+      *Explore logs:* https://grafanaURL.com/explore?orgId=1
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -78,39 +65,3 @@ templates:
       {{ end }}
 
       {{- end }}
-{% endraw %}
-{% else %}
-{% raw %}
-  - orgId: 1
-    name: email.subject
-    template: |-
-      {{ define "email.subject" -}}
-      [{{ .Status | toUpper }}] Grafana Alert
-      {{- end -}}
-  - orgId: 1
-    name: email.body
-    template: |-
-      {{ define "email.body" -}}
-      Alert: {{ .Labels.alertname }}
-      {{ if .Annotations -}}
-      Summary: {{ .Annotations.summary}}
-      Description: {{ .Annotations.description }}
-      {{ end -}}
-      Log message: {{ index .Labels "message" }}
-      Explore logs: {{ GRAFANA_ENDPOINT }}
-      {{ if .DashboardURL -}}
-      Go to dashboard: {{ .DashboardURL }}
-      {{- end }}
-      {{ if .PanelURL -}}
-      Go to panel: {{ .PanelURL }}
-      {{- end }}
-      Details:
-      {{ range .Labels.SortedPairs -}}
-      - {{ .Name }}: `{{ .Value }}`
-      {{ end -}}
-      {{ if .SilenceURL -}}
-      Silence this alert: {{ .SilenceURL }}
-      {{- end }}
-      {{- end }}
-{% endraw %}
-{% endif %}

sbcli_pre-1.2.5/simplyblock_core/scripts/apply_dashboard.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+TD=$(dirname -- "$(readlink -f -- "$0")")
+
+# Grafana Password
+export grafanaPassword=$1
+
+# Grafana username
+GF_ADMIN_USER=admin
+
+HOST=0.0.0.0:3000
+
+DASHBOARDS="${TD}/dashboards"
+for dashboard in "${DASHBOARDS}/cluster.json" "${DASHBOARDS}/devices.json" "${DASHBOARDS}/nodes.json" "${DASHBOARDS}/lvols.json" "${DASHBOARDS}/pools.json" "${DASHBOARDS}/node-exporter.json"; do
+    echo -e "\nUploading dashboard: ${dashboard}"
+    curl -X POST -H "Content-Type: application/json" \
+        -d "@${dashboard}" \
+        "http://${GF_ADMIN_USER}:${grafanaPassword}@${HOST}/api/dashboards/import"
+    echo ""
+done
+
+echo "Cluster deployment complete."

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/cluster.json

@@ -2354,4 +2354,4 @@
         "version": 5,
         "weekStart": ""
     }
-}
+}

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/deploy_stack.sh

@@ -19,8 +19,6 @@ then
    export FDB_CLUSTER_FILE_CONTENTS=$FDB_CLUSTER_FILE_CONTENTS
 fi
 
-docker network create monitoring-net -d overlay --attachable
-
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -9,8 +9,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   opensearch:
     image: "opensearchproject/opensearch:2.4.0"
@@ -26,8 +24,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   graylog:
     hostname: "server"
@@ -38,7 +34,7 @@ services:
       GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
       GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}"
       GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
-      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost/graylog/"
+      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
       GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
       GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
     ports:
@@ -47,6 +43,7 @@ services:
     - "5140:5140/tcp"   # Syslog
     - "5555:5555/tcp"   # RAW TCP
     - "5555:5555/udp"   # RAW TCP
+    - "9000:9000/tcp"   # Server API
     - "12201:12201/tcp" # GELF TCP
     - "12201:12201/udp" # GELF UDP
     - "13301:13301/tcp" # Forwarder data
@@ -57,8 +54,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   promagent:
     image: simplyblock/promagent
@@ -69,16 +64,12 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   pushgateway:
     image: prom/pushgateway
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   prometheus:
     image: prom/prometheus:v2.44.0
@@ -94,8 +85,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   node-exporter:
     image: prom/node-exporter:v1.7.0
@@ -116,9 +105,7 @@ services:
       mode: global
       placement:
         constraints: [node.role == worker]
-    networks:
-      - monitoring-net
-
+        
   grafana:
     image: grafana/grafana:10.0.12
     environment:
@@ -127,16 +114,19 @@ services:
       GF_ALERTING_ENABLED: "true"
       GF_PATHS_PROVISIONING: "/etc/grafana/provisioning"
       GF_INSTALL_PLUGINS: "grafana-opensearch-datasource"
-      GF_SERVER_ROOT_URL: "http://localhost/grafana/"
     volumes:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
+    restart: "always"
+    ports:
+      - target: 3000
+        published: 3000
+        protocol: tcp
+        mode: host
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   CleanupGraylog:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -146,8 +136,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
+
+  ### monitoring ###
 
 volumes:
   mongodb_data:
@@ -159,5 +149,6 @@ volumes:
   alertmanager_data:
 
 networks:
-  monitoring-net:
+  hostnet:
     external: true
+    name: host

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -114,7 +114,6 @@ services:
       - 8404:8404
     networks:
       - localnet
-      - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
 
@@ -186,20 +185,9 @@ services:
     networks:
       - hostnet
 
-  TasksRunnerRestart:
+  TasksRunner:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/tasks_runner_restart.py"
-    deploy:
-      placement:
-        constraints: [node.role == manager]
-    volumes:
-      - "/etc/foundationdb:/etc/foundationdb"
-    networks:
-      - hostnet
-
-  TasksRunnerMigration:
-    image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/tasks_runner_migration.py"
+    command: "python simplyblock_core/services/job_tasks.py"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -212,9 +200,6 @@ volumes:
   os_data:
 
 networks:
-  monitoring-net:
-    external: true
-
   hostnet:
     external: true
     name: host

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/haproxy.cfg

@@ -42,16 +42,6 @@ backend wep_api_services
     balance roundrobin
     server-template webapi- 3 WebAppAPI:5000 check resolvers docker init-addr libc,none
 
-backend grafana_services
-    balance roundrobin
-    http-request set-path %[path,regsub(^/grafana/?,/)]
-    server-template grafana- 1 grafana:3000 check resolvers docker init-addr libc,none
-
-backend graylog_services
-    balance roundrobin
-    http-request set-path %[path,regsub(^/graylog/?,/)]
-    server-template graylog- 1 graylog:9000 check resolvers docker init-addr libc,none
-
 frontend stats_front
     bind *:8404
     stats enable
@@ -62,9 +52,4 @@ frontend stats_front
 
 frontend web_api_front
     bind *:80
-
-    use_backend grafana_services if { path /grafana } || { path_beg /grafana/ }
-    use_backend graylog_services if { path /graylog } || { path_beg /graylog/ }
-
     default_backend wep_api_services
-

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/scripts/install_deps.sh

@@ -29,7 +29,6 @@ sudo sed -i 's/#X11UseLocalhost yes/X11UseLocalhost no/g' /etc/ssh/sshd_config
 
 sudo service sshd restart
 sudo modprobe nvme-tcp
-sudo modprobe nbd
 
 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
 

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -193,7 +193,7 @@ while True:
                     logger.info(f"Device is skipped: {device.get_id()} status: {device.status}")
                     continue
                 capacity_dict = rpc_client.alceml_get_capacity(device.alceml_bdev)
-                stats_dict = rpc_client.get_device_stats(device.nvme_bdev)
+                stats_dict = rpc_client.get_device_stats(device.alloc_bdev)
                 record = add_device_stats(cl, device, capacity_dict, stats_dict)
                 if record:
                     devices_records.append(record)

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/services/device_monitor.py

@@ -5,7 +5,7 @@ import sys
 import uuid
 
 from simplyblock_core import constants, kv_store
-from simplyblock_core.controllers import tasks_controller
+from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
@@ -27,6 +27,47 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController()
 
 
+def add_device_to_auto_restart(device):
+    tasks = db_controller.get_job_tasks(device.cluster_id)
+    for task in tasks:
+        if task.device_id == device.get_id():
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return
+
+    ds = JobSchedule()
+    ds.uuid = str(uuid.uuid4())
+    ds.cluster_id = device.cluster_id
+    ds.node_id = device.node_id
+    ds.device_id = device.get_id()
+    ds.date = int(time.time())
+    ds.function_name = "device_restart"
+    ds.status = 'new'
+
+    ds.write_to_db(db_store)
+    return ds.get_id()
+
+
+def add_node_to_auto_restart(node):
+    tasks = db_controller.get_job_tasks(node.cluster_id)
+    for task in tasks:
+        if task.node_id == node.get_id():
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return
+
+    ds = JobSchedule()
+    ds.uuid = str(uuid.uuid4())
+    ds.cluster_id = node.cluster_id
+    ds.node_id = node.get_id()
+    ds.date = int(time.time())
+    ds.function_name = "node_restart"
+    ds.status = 'new'
+
+    ds.write_to_db(db_store)
+    return ds.get_id()
+
+
 logger.info("Starting Device monitor...")
 while True:
     nodes = db_controller.get_storage_nodes()
@@ -49,8 +90,8 @@ while True:
                 auto_restart_devices.append(dev)
 
         if len(auto_restart_devices) == 1:
-            tasks_controller.add_device_to_auto_restart(auto_restart_devices[0])
+            add_device_to_auto_restart(auto_restart_devices[0])
         elif len(auto_restart_devices) >= 2 and len(online_devices) == 0:
-            tasks_controller.add_node_to_auto_restart(node)
+            add_node_to_auto_restart(node)
 
     time.sleep(constants.DEV_MONITOR_INTERVAL_SEC)

{sbcli_pre-1.2.3 → sbcli_pre-1.2.5}/simplyblock_core/services/distr_event_collector.py

@@ -88,7 +88,7 @@ def process_lvol_event(event):
     if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
         vuid = event.object_dict['vuid']
         lvol = None
-        for lv in db_controller.get_lvols():  # pass
+        for lv in db_controller.get_lvols():
             if lv.vuid == vuid:
                 lvol = lv
                 break
@@ -127,6 +127,7 @@ def process_event(event_id):
 
 hostname = utils.get_hostname()
 logger.info("Starting Distr event collector...")
+logger.info(f"Node:{hostname}")
 while True:
     time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
 
@@ -140,13 +141,14 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=10, retry=2)
-
+        timeout=3, retry=2
+    )
+    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
     try:
-        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
-
+        # events = client.distr_status_events_get()
+        events = client.distr_status_events_discard_then_get(0, num_of_events)
         if not events:
-            logger.debug("no events found")
+            logger.error("Distr events empty")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -159,11 +161,10 @@ while True:
         for eid in event_ids:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
-
-            logger.info(f"Discarding events: {len(events)}")
-            client.distr_status_events_discard_then_get(len(events), 0)
+        logger.info(f"Discarding events: {num_of_events}")
+        events = client.distr_status_events_discard_then_get(num_of_events, 0)
 
     except Exception as e:
-        logger.error("Failed to process distr events")
+        logger.error("Failed to get distr events")
         logger.exception(e)
         continue

sbcli_pre-1.2.5/simplyblock_core/services/health_check_service.py

@@ -0,0 +1,136 @@
+# coding=utf-8
+import logging
+
+import time
+import sys
+from datetime import datetime
+
+
+from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.storage_node import StorageNode
+from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core import constants, kv_store
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+def set_node_health_check(snode, health_check_status):
+    snode = db_controller.get_storage_node_by_id(snode.get_id())
+    if snode.health_check == health_check_status:
+        return
+    old_status = snode.health_check
+    snode.health_check = health_check_status
+    snode.updated_at = str(datetime.now())
+    snode.write_to_db(db_store)
+    storage_events.snode_health_check_change(snode, snode.health_check, old_status, caused_by="monitor")
+
+
+def set_device_health_check(cluster_id, device, health_check_status):
+    if device.health_check == health_check_status:
+        return
+    nodes = db_controller.get_storage_nodes()
+    for node in nodes:
+        if node.nvme_devices:
+            for dev in node.nvme_devices:
+                if dev.get_id() == device.get_id():
+                    old_status = dev.health_check
+                    dev.health_check = health_check_status
+                    node.write_to_db(db_store)
+                    device_events.device_health_check_change(
+                        dev, dev.health_check, old_status, caused_by="monitor")
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_store = kv_store.KVStore()
+db_controller = kv_store.DBController()
+
+logger.info("Starting health check service")
+while True:
+    cluster_id = ""
+    cl = db_controller.get_clusters()
+    if cl:
+        cluster_id = cl[0].get_id()
+
+    snodes = db_controller.get_storage_nodes()
+    if not snodes:
+        logger.error("storage nodes list is empty")
+
+    for snode in snodes:
+        logger.info("Node: %s, status %s", snode.get_id(), snode.status)
+
+        if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+            logger.info(f"Node status is: {snode.status}, skipping")
+            continue
+
+        # 1- check node ping
+        ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+        logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+        # 2- check node API
+        node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+        logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+        if snode.status == StorageNode.STATUS_OFFLINE:
+            set_node_health_check(snode, ping_check & node_api_check)
+            continue
+
+        # 3- check node RPC
+        node_rpc_check = health_controller._check_node_rpc(
+            snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+        logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+        # 4- docker API
+        node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+        logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+        is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+
+        health_check_status = is_node_online
+        if not node_rpc_check:
+            logger.info("Putting all devices to unavailable state because RPC check failed")
+            for dev in snode.nvme_devices:
+                if dev.io_error:
+                    logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
+                    continue
+                set_device_health_check(cluster_id, dev, False)
+        else:
+            logger.info(f"Node device count: {len(snode.nvme_devices)}")
+            node_devices_check = True
+            node_remote_devices_check = True
+
+            for dev in snode.nvme_devices:
+                if dev.io_error:
+                    logger.debug(f"Skipping Device check because of io_error {dev.get_id()}")
+                    continue
+                ret = health_controller.check_device(dev.get_id())
+                set_device_health_check(cluster_id, dev, ret)
+                if dev.status == dev.STATUS_ONLINE:
+                    node_devices_check &= ret
+
+            logger.info(f"Node remote device: {len(snode.remote_devices)}")
+            rpc_client = RPCClient(
+                snode.mgmt_ip, snode.rpc_port,
+                snode.rpc_username, snode.rpc_password,
+                timeout=5, retry=3)
+            for remote_device in snode.remote_devices:
+                ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                if ret:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                else:
+                    logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                node_remote_devices_check &= bool(ret)
+
+            health_check_status = is_node_online and node_devices_check and node_remote_devices_check
+        set_node_health_check(snode, health_check_status)
+
+    time.sleep(constants.HEALTH_CHECK_INTERVAL_SEC)
+