sbcli-pre 1.2.5__zip → 1.2.7__zip

sbcli_pre-1.2.5/simplyblock_core/scripts/alerting/alert_resources.yaml → sbcli_pre-1.2.7/simplyblock_core/scripts/alerting/alert_resources.yaml.j2

@@ -12,15 +12,26 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: slack
+        type: {{ ALERT_TYPE }}
+        {% if ALERT_TYPE == 'slack' %}
         settings:
           username: grafana_bot
-          url: 'https://hooks.slack.com/services/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'
+          url: '{{ CONTACT_POINT }}'
           title: |
-            {{ template "slack.title" . }}
+            {{ '{{' }} template "slack.title" . {{ '}}' }}
           text: |
-            {{ template "slack.message" . }}
+            {{ '{{' }} template "slack.message" . {{ '}}' }}
+        {% else %}
+        settings:
+          addresses: '{{ CONTACT_POINT }}'
+          subject: |
+            {{ '{{' }} template "email.subject" . {{ '}}' }}
+          body: |
+            {{ '{{' }} template "email.body" . {{ '}}' }}
+        {% endif %}
 
+{% if ALERT_TYPE == 'slack' %}
+{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -38,7 +49,9 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      *Explore logs:* https://grafanaURL.com/explore?orgId=1
+      {% endraw %}
+      *Explore logs:* {{ GRAFANA_ENDPOINT }}
+      {% raw %}
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -65,3 +78,39 @@ templates:
       {{ end }}
 
       {{- end }}
+{% endraw %}
+{% else %}
+{% raw %}
+  - orgId: 1
+    name: email.subject
+    template: |-
+      {{ define "email.subject" -}}
+      [{{ .Status | toUpper }}] Grafana Alert
+      {{- end -}}
+  - orgId: 1
+    name: email.body
+    template: |-
+      {{ define "email.body" -}}
+      Alert: {{ .Labels.alertname }}
+      {{ if .Annotations -}}
+      Summary: {{ .Annotations.summary}}
+      Description: {{ .Annotations.description }}
+      {{ end -}}
+      Log message: {{ index .Labels "message" }}
+      Explore logs: {{ GRAFANA_ENDPOINT }}
+      {{ if .DashboardURL -}}
+      Go to dashboard: {{ .DashboardURL }}
+      {{- end }}
+      {{ if .PanelURL -}}
+      Go to panel: {{ .PanelURL }}
+      {{- end }}
+      Details:
+      {{ range .Labels.SortedPairs -}}
+      - {{ .Name }}: `{{ .Value }}`
+      {{ end -}}
+      {{ if .SilenceURL -}}
+      Silence this alert: {{ .SilenceURL }}
+      {{- end }}
+      {{- end }}
+{% endraw %}
+{% endif %}

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/dashboards/cluster.json

@@ -2354,4 +2354,4 @@
         "version": 5,
         "weekStart": ""
     }
-}
+}

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/deploy_stack.sh

@@ -19,6 +19,15 @@ then
    export FDB_CLUSTER_FILE_CONTENTS=$FDB_CLUSTER_FILE_CONTENTS
 fi
 
+docker network create monitoring-net -d overlay --attachable
+
+INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+
+if [ -n "$INSTANCE_ID" ] 
+then
+  export USE_EFS="rexray/efs"
+fi
+
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -9,6 +9,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   opensearch:
     image: "opensearchproject/opensearch:2.4.0"
@@ -24,6 +26,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   graylog:
     hostname: "server"
@@ -34,16 +38,16 @@ services:
       GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
       GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}"
       GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
-      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
+      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost/graylog/"
       GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
       GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
+      GRAYLOG_SKIP_PREFLIGHT_CHECKS: "true"
     ports:
     - "5044:5044/tcp"   # Beats
     - "5140:5140/udp"   # Syslog
     - "5140:5140/tcp"   # Syslog
     - "5555:5555/tcp"   # RAW TCP
     - "5555:5555/udp"   # RAW TCP
-    - "9000:9000/tcp"   # Server API
     - "12201:12201/tcp" # GELF TCP
     - "12201:12201/udp" # GELF UDP
     - "13301:13301/tcp" # Forwarder data
@@ -54,22 +58,28 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   promagent:
     image: simplyblock/promagent
     environment:
       ClusterID: "${CLUSTER_ID}"
-      ClusterIP: "${CLUSTER_IP}"
+      ClusterIP: "HAProxy"
       ClusterSecret: "${CLUSTER_SECRET}"
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   pushgateway:
     image: prom/pushgateway
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   prometheus:
     image: prom/prometheus:v2.44.0
@@ -85,6 +95,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   node-exporter:
     image: prom/node-exporter:v1.7.0
@@ -105,7 +117,9 @@ services:
       mode: global
       placement:
         constraints: [node.role == worker]
-        
+    networks:
+      - monitoring-net
+
   grafana:
     image: grafana/grafana:10.0.12
     environment:
@@ -114,19 +128,16 @@ services:
       GF_ALERTING_ENABLED: "true"
       GF_PATHS_PROVISIONING: "/etc/grafana/provisioning"
       GF_INSTALL_PLUGINS: "grafana-opensearch-datasource"
+      GF_SERVER_ROOT_URL: "http://localhost/grafana/"
     volumes:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
-    restart: "always"
-    ports:
-      - target: 3000
-        published: 3000
-        protocol: tcp
-        mode: host
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   CleanupGraylog:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -136,19 +147,25 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-
-  ### monitoring ###
+    networks:
+      - monitoring-net
 
 volumes:
   mongodb_data:
+    driver: ${USE_EFS:-local}
   os_data:
+    driver: ${USE_EFS:-local}
   graylog_data:
+    driver: ${USE_EFS:-local}
   graylog_journal:
+    driver: ${USE_EFS:-local}
   grafana_data:
+    driver: ${USE_EFS:-local}
+  graylog_config:
+    driver: ${USE_EFS:-local}  
   prometheus_data:
-  alertmanager_data:
+    driver: ${USE_EFS:-local}
 
 networks:
-  hostnet:
+  monitoring-net:
     external: true
-    name: host

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -114,6 +114,7 @@ services:
       - 8404:8404
     networks:
       - localnet
+      - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
 
@@ -185,9 +186,20 @@ services:
     networks:
       - hostnet
 
-  TasksRunner:
+  TasksRunnerRestart:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/job_tasks.py"
+    command: "python simplyblock_core/services/tasks_runner_restart.py"
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+    volumes:
+      - "/etc/foundationdb:/etc/foundationdb"
+    networks:
+      - hostnet
+
+  TasksRunnerMigration:
+    image: $SIMPLYBLOCK_DOCKER_IMAGE
+    command: "python simplyblock_core/services/tasks_runner_migration.py"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -200,6 +212,9 @@ volumes:
   os_data:
 
 networks:
+  monitoring-net:
+    external: true
+
   hostnet:
     external: true
     name: host

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/haproxy.cfg

@@ -42,6 +42,16 @@ backend wep_api_services
     balance roundrobin
     server-template webapi- 3 WebAppAPI:5000 check resolvers docker init-addr libc,none
 
+backend grafana_services
+    balance roundrobin
+    http-request set-path %[path,regsub(^/grafana/?,/)]
+    server-template grafana- 1 grafana:3000 check resolvers docker init-addr libc,none
+
+backend graylog_services
+    balance roundrobin
+    http-request set-path %[path,regsub(^/graylog/?,/)]
+    server-template graylog- 1 graylog:9000 check resolvers docker init-addr libc,none
+
 frontend stats_front
     bind *:8404
     stats enable
@@ -52,4 +62,9 @@ frontend stats_front
 
 frontend web_api_front
     bind *:80
+
+    use_backend grafana_services if { path /grafana } || { path_beg /grafana/ }
+    use_backend graylog_services if { path /graylog } || { path_beg /graylog/ }
+
     default_backend wep_api_services
+

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/install_deps.sh

@@ -15,6 +15,8 @@ sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce do
 sudo systemctl enable docker
 sudo systemctl start docker
 
+sudo docker plugin install --grant-all-permissions rexray/efs REXRAY_PREEMPT=true
+
 wget https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm -q
 sudo rpm -U foundationdb-clients-7.3.3-1.el7.x86_64.rpm --quiet --reinstall
 rm -f foundationdb-clients-7.3.3-1.el7.x86_64.rpm
@@ -29,6 +31,7 @@ sudo sed -i 's/#X11UseLocalhost yes/X11UseLocalhost no/g' /etc/ssh/sshd_config
 
 sudo service sshd restart
 sudo modprobe nvme-tcp
+sudo modprobe nbd
 
 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
 

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/scripts/stack_deploy_wait.sh

@@ -6,7 +6,7 @@ opt_h=0
 opt_l=""
 opt_r=0
 opt_s=5
-opt_t=600 # 10 min
+opt_t=900 # 15 min
 
 start_epoc=$(date +%s)
 cmd_min_timeout=15

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -193,7 +193,7 @@ while True:
                     logger.info(f"Device is skipped: {device.get_id()} status: {device.status}")
                     continue
                 capacity_dict = rpc_client.alceml_get_capacity(device.alceml_bdev)
-                stats_dict = rpc_client.get_device_stats(device.alloc_bdev)
+                stats_dict = rpc_client.get_device_stats(device.nvme_bdev)
                 record = add_device_stats(cl, device, capacity_dict, stats_dict)
                 if record:
                     devices_records.append(record)

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/device_monitor.py

@@ -5,7 +5,7 @@ import sys
 import uuid
 
 from simplyblock_core import constants, kv_store
-from simplyblock_core.models.job_schedule import JobSchedule
+from simplyblock_core.controllers import tasks_controller
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
@@ -27,47 +27,6 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController()
 
 
-def add_device_to_auto_restart(device):
-    tasks = db_controller.get_job_tasks(device.cluster_id)
-    for task in tasks:
-        if task.device_id == device.get_id():
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return
-
-    ds = JobSchedule()
-    ds.uuid = str(uuid.uuid4())
-    ds.cluster_id = device.cluster_id
-    ds.node_id = device.node_id
-    ds.device_id = device.get_id()
-    ds.date = int(time.time())
-    ds.function_name = "device_restart"
-    ds.status = 'new'
-
-    ds.write_to_db(db_store)
-    return ds.get_id()
-
-
-def add_node_to_auto_restart(node):
-    tasks = db_controller.get_job_tasks(node.cluster_id)
-    for task in tasks:
-        if task.node_id == node.get_id():
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return
-
-    ds = JobSchedule()
-    ds.uuid = str(uuid.uuid4())
-    ds.cluster_id = node.cluster_id
-    ds.node_id = node.get_id()
-    ds.date = int(time.time())
-    ds.function_name = "node_restart"
-    ds.status = 'new'
-
-    ds.write_to_db(db_store)
-    return ds.get_id()
-
-
 logger.info("Starting Device monitor...")
 while True:
     nodes = db_controller.get_storage_nodes()
@@ -89,9 +48,9 @@ while True:
                 logger.info("Adding device to auto restart")
                 auto_restart_devices.append(dev)
 
-        if len(auto_restart_devices) == 1:
-            add_device_to_auto_restart(auto_restart_devices[0])
-        elif len(auto_restart_devices) >= 2 and len(online_devices) == 0:
-            add_node_to_auto_restart(node)
+        if len(auto_restart_devices) >= 2 or len(online_devices) == 0:
+            tasks_controller.add_node_to_auto_restart(node)
+        elif len(auto_restart_devices) == 1:
+            tasks_controller.add_device_to_auto_restart(auto_restart_devices[0])
 
     time.sleep(constants.DEV_MONITOR_INTERVAL_SEC)

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/distr_event_collector.py

@@ -88,7 +88,7 @@ def process_lvol_event(event):
     if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
         vuid = event.object_dict['vuid']
         lvol = None
-        for lv in db_controller.get_lvols():
+        for lv in db_controller.get_lvols():  # pass
             if lv.vuid == vuid:
                 lvol = lv
                 break
@@ -127,7 +127,6 @@ def process_event(event_id):
 
 hostname = utils.get_hostname()
 logger.info("Starting Distr event collector...")
-logger.info(f"Node:{hostname}")
 while True:
     time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
 
@@ -141,14 +140,13 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=3, retry=2
-    )
-    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
+        timeout=10, retry=2)
+
     try:
-        # events = client.distr_status_events_get()
-        events = client.distr_status_events_discard_then_get(0, num_of_events)
+        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
+
         if not events:
-            logger.error("Distr events empty")
+            logger.debug("no events found")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -161,10 +159,11 @@ while True:
         for eid in event_ids:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
-        logger.info(f"Discarding events: {num_of_events}")
-        events = client.distr_status_events_discard_then_get(num_of_events, 0)
+
+            logger.info(f"Discarding events: {len(events)}")
+            client.distr_status_events_discard_then_get(len(events), 0)
 
     except Exception as e:
-        logger.error("Failed to get distr events")
+        logger.error("Failed to process distr events")
         logger.exception(e)
         continue

sbcli_pre-1.2.7/simplyblock_core/services/health_check_service.py

@@ -0,0 +1,134 @@
+# coding=utf-8
+import logging
+
+import time
+import sys
+from datetime import datetime
+
+
+from simplyblock_core.controllers import health_controller, storage_events, device_events
+from simplyblock_core.models.storage_node import StorageNode
+from simplyblock_core.rpc_client import RPCClient
+from simplyblock_core import constants, kv_store
+
+# Import the GELF logger
+from graypy import GELFUDPHandler
+
+def set_node_health_check(snode, health_check_status):
+    snode = db_controller.get_storage_node_by_id(snode.get_id())
+    if snode.health_check == health_check_status:
+        return
+    old_status = snode.health_check
+    snode.health_check = health_check_status
+    snode.updated_at = str(datetime.now())
+    snode.write_to_db(db_store)
+    storage_events.snode_health_check_change(snode, snode.health_check, old_status, caused_by="monitor")
+
+
+def set_device_health_check(cluster_id, device, health_check_status):
+    if device.health_check == health_check_status:
+        return
+    nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+    for node in nodes:
+        if node.nvme_devices:
+            for dev in node.nvme_devices:
+                if dev.get_id() == device.get_id():
+                    old_status = dev.health_check
+                    dev.health_check = health_check_status
+                    node.write_to_db(db_store)
+                    device_events.device_health_check_change(
+                        dev, dev.health_check, old_status, caused_by="monitor")
+
+
+# configure logging
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+logger = logging.getLogger()
+logger.addHandler(gelf_handler)
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+# get DB controller
+db_store = kv_store.KVStore()
+db_controller = kv_store.DBController()
+
+logger.info("Starting health check service")
+while True:
+    clusters = db_controller.get_clusters()
+    for cluster in clusters:
+        cluster_id = cluster.get_id()
+        snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
+        if not snodes:
+            logger.error("storage nodes list is empty")
+
+        for snode in snodes:
+            logger.info("Node: %s, status %s", snode.get_id(), snode.status)
+
+            if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE]:
+                logger.info(f"Node status is: {snode.status}, skipping")
+                continue
+
+            # 1- check node ping
+            ping_check = health_controller._check_node_ping(snode.mgmt_ip)
+            logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
+
+            # 2- check node API
+            node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+            logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
+
+            if snode.status == StorageNode.STATUS_OFFLINE:
+                set_node_health_check(snode, ping_check & node_api_check)
+                continue
+
+            # 3- check node RPC
+            node_rpc_check = health_controller._check_node_rpc(
+                snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)
+            logger.info(f"Check: node RPC {snode.mgmt_ip}:{snode.rpc_port} ... {node_rpc_check}")
+
+            # 4- docker API
+            node_docker_check = health_controller._check_node_docker_api(snode.mgmt_ip)
+            logger.info(f"Check: node docker API {snode.mgmt_ip}:2375 ... {node_docker_check}")
+
+            is_node_online = ping_check and node_api_check and node_rpc_check and node_docker_check
+
+            health_check_status = is_node_online
+            if not node_rpc_check:
+                logger.info("Putting all devices to unavailable state because RPC check failed")
+                for dev in snode.nvme_devices:
+                    if dev.io_error:
+                        logger.debug(f"Skipping Device action because of io_error {dev.get_id()}")
+                        continue
+                    set_device_health_check(cluster_id, dev, False)
+            else:
+                logger.info(f"Node device count: {len(snode.nvme_devices)}")
+                node_devices_check = True
+                node_remote_devices_check = True
+
+                for dev in snode.nvme_devices:
+                    if dev.io_error:
+                        logger.debug(f"Skipping Device check because of io_error {dev.get_id()}")
+                        continue
+                    ret = health_controller.check_device(dev.get_id())
+                    set_device_health_check(cluster_id, dev, ret)
+                    if dev.status == dev.STATUS_ONLINE:
+                        node_devices_check &= ret
+
+                logger.info(f"Node remote device: {len(snode.remote_devices)}")
+                rpc_client = RPCClient(
+                    snode.mgmt_ip, snode.rpc_port,
+                    snode.rpc_username, snode.rpc_password,
+                    timeout=10, retry=1)
+                for remote_device in snode.remote_devices:
+                    ret = rpc_client.get_bdevs(remote_device.remote_bdev)
+                    if ret:
+                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
+                    else:
+                        logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
+                    node_remote_devices_check &= bool(ret)
+
+                health_check_status = is_node_online and node_devices_check and node_remote_devices_check
+            set_node_health_check(snode, health_check_status)
+
+    time.sleep(constants.HEALTH_CHECK_INTERVAL_SEC)
+

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/lvol_monitor.py

@@ -49,7 +49,7 @@ db_controller = kv_store.DBController()
 
 logger.info("Starting LVol monitor...")
 while True:
-    lvols = db_controller.get_lvols()
+    lvols = db_controller.get_lvols()  # pass
     if not lvols:
         logger.error("LVols list is empty")
 

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/lvol_stat_collector.py

@@ -115,7 +115,7 @@ logger.info("Starting stats collector...")
 while True:
 
     pools = db_controller.get_pools()
-    all_lvols = db_controller.get_lvols()
+    all_lvols = db_controller.get_lvols()  # pass
     for pool in pools:
         lvols = []
         for lvol in all_lvols:

{sbcli_pre-1.2.5 → sbcli_pre-1.2.7}/simplyblock_core/services/port_stat_collector.py

@@ -74,4 +74,3 @@ while True:
             update_port_stats(snode, nic, stats)
         else:
             logger.error("Error getting port stats: %s", nic.get_id())
-