sbcli-pre 1.2.4__zip → 1.2.5__zip

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/rpc_client.py

@@ -306,9 +306,9 @@ class RPCClient:
         params = {"name": name}
         return self._request("bdev_compress_delete", params)
 
-    def ultra21_bdev_pass_create(self, base_bdev, vuid, pt_name):
+    def ultra21_bdev_pass_create(self, alloc_bdev, vuid, pt_name):
         params = {
-            "base_bdev": base_bdev,
+            "base_bdev": alloc_bdev,
             "vuid": vuid,
             "pt_bdev": pt_name
         }
@@ -318,8 +318,7 @@ class RPCClient:
         params = {"name": name}
         return self._request2("ultra21_bdev_pass_delete", params)
 
-    def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3,
-                           dev_cpu_mask=""):
+    def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3):
         params = {
             "name": alceml_name,
             "cntr_path": nvme_name,
@@ -335,13 +334,10 @@ class RPCClient:
             "use_optimized": True,
             "pba_nbalign": 4096
         }
-        if dev_cpu_mask:
-            params["bdb_lcpu_mask"] = int(dev_cpu_mask,16)
         return self._request("bdev_alceml_create", params)
 
     def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names,
-                            chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152,
-                            dev_cpu_mask=""):
+                            chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152):
         """"
             // Optional (not specified = no HA)
             // Comma-separated communication addresses, for each node, e.g. "192.168.10.1:45001,192.168.10.1:32768".
@@ -367,8 +363,6 @@ class RPCClient:
         if ha_comm_addrs:
             params['ha_comm_addrs'] = ha_comm_addrs
             params['ha_inode_self'] = ha_inode_self
-        if dev_cpu_mask:
-            params["bdb_lcpu_mask"] = int(dev_cpu_mask, 16)
 
         return self._request("bdev_distrib_create", params)
 
@@ -392,7 +386,7 @@ class RPCClient:
         params = {
             "name": name,
             "raid_level": "0",
-            "strip_size_kb": 4,
+            "strip_size_kb": 4 * len(bdevs_list),
             "base_bdevs": bdevs_list
         }
         return self._request("bdev_raid_create", params)
@@ -434,9 +428,8 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 1,
+            "fast_io_fail_timeout_sec": 0,
             "num_io_queues": 16384,
-            "ctrlr_loss_timeout_sec": 2,
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -484,9 +477,9 @@ class RPCClient:
         params = {
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
-            "ctrlr_loss_timeout_sec": 2,
-            "fast_io_fail_timeout_sec": 1,
-            "reconnect_delay_sec": 1,
+            "ctrlr_loss_timeout_sec": -1,
+            "fast_io_fail_timeout_sec": 5,
+            "reconnect_delay_sec": 5,
             "keep_alive_timeout_ms": 200,
             "transport_ack_timeout": 7,
             "timeout_us": 100000
@@ -590,14 +583,12 @@ class RPCClient:
         }
         return self._request("ultra21_lvol_dismount", params)
 
-    def bdev_jm_create(self, name, name_storage1, block_size=4096, dev_cpu_mask=""):
+    def bdev_jm_create(self, name, name_storage1, block_size=4096):
         params = {
             "name": name,
             "name_storage1": name_storage1,
             "block_size": block_size
         }
-        if dev_cpu_mask:
-            params["bdb_lcpu_mask"] = int(dev_cpu_mask, 16)
         return self._request("bdev_jm_create", params)
 
     def bdev_jm_delete(self, name):
@@ -629,42 +620,6 @@ class RPCClient:
     def framework_start_init(self):
         return self._request("framework_start_init")
 
-    def bdev_examine(self, name):
-        params = {"name": name}
-        return self._request("bdev_examine", params)
-
-    def nbd_start_disk(self, bdev_name, nbd_device="/dev/nbd0"):
-        params = {
-            "bdev_name": bdev_name,
-            "nbd_device": nbd_device,
-        }
-        return self._request("nbd_start_disk", params)
-
-    def nbd_stop_disk(self, nbd_device):
-        params = {
-            "nbd_device": nbd_device
-        }
-        return self._request("nbd_stop_disk", params)
-
-
     def bdev_jm_unmap_vuid(self, name, vuid):
         params = {"name": name, "vuid": vuid}
         return self._request("bdev_jm_unmap_vuid", params)
-
-    def sock_impl_set_options(self):
-        method = "sock_impl_set_options"
-        params = {"impl_name": "posix", "enable_quickack": True,
-                  "enable_zerocopy_send_server": True,
-                  "enable_zerocopy_send_client": True}
-        return self._request(method, params)
-
-    def nvmf_set_config(self, poll_groups_mask):
-        params = {"poll_groups_mask": poll_groups_mask}
-        return self._request("nvmf_set_config", params)
-
-    def thread_get_stats(self):
-        return self._request("thread_get_stats")
-
-    def thread_set_cpumask(self, app_thread_process_id, app_thread_mask):
-        params = {"id": app_thread_process_id, "cpumask": app_thread_mask}
-        return self._request("thread_set_cpumask", params)

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/__init__.py

@@ -35,6 +35,10 @@ def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, log
     return __run_script(
         ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, graylog_password, cluster_id, log_del_interval, metrics_retention_period])
 
+def apply_dashboard(grafanaPassword):
+    return __run_script(
+        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'apply_dashboard.sh'), grafanaPassword])
+
 
 def deploy_cleaner():
     return __run_script(['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'clean_local_storage_deploy.sh')])

sbcli_pre-1.2.4/simplyblock_core/scripts/alerting/alert_resources.yaml.j2 → sbcli_pre-1.2.5/simplyblock_core/scripts/alerting/alert_resources.yaml

@@ -12,26 +12,15 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: {{ ALERT_TYPE }}
-        {% if ALERT_TYPE == 'slack' %}
+        type: slack
         settings:
           username: grafana_bot
-          url: '{{ CONTACT_POINT }}'
+          url: 'https://hooks.slack.com/services/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'
           title: |
-            {{ '{{' }} template "slack.title" . {{ '}}' }}
+            {{ template "slack.title" . }}
           text: |
-            {{ '{{' }} template "slack.message" . {{ '}}' }}
-        {% else %}
-        settings:
-          addresses: '{{ CONTACT_POINT }}'
-          subject: |
-            {{ '{{' }} template "email.subject" . {{ '}}' }}
-          body: |
-            {{ '{{' }} template "email.body" . {{ '}}' }}
-        {% endif %}
+            {{ template "slack.message" . }}
 
-{% if ALERT_TYPE == 'slack' %}
-{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -49,9 +38,7 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      {% endraw %}
-      *Explore logs:* {{ GRAFANA_ENDPOINT }}
-      {% raw %}
+      *Explore logs:* https://grafanaURL.com/explore?orgId=1
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -78,39 +65,3 @@ templates:
       {{ end }}
 
       {{- end }}
-{% endraw %}
-{% else %}
-{% raw %}
-  - orgId: 1
-    name: email.subject
-    template: |-
-      {{ define "email.subject" -}}
-      [{{ .Status | toUpper }}] Grafana Alert
-      {{- end -}}
-  - orgId: 1
-    name: email.body
-    template: |-
-      {{ define "email.body" -}}
-      Alert: {{ .Labels.alertname }}
-      {{ if .Annotations -}}
-      Summary: {{ .Annotations.summary}}
-      Description: {{ .Annotations.description }}
-      {{ end -}}
-      Log message: {{ index .Labels "message" }}
-      Explore logs: {{ GRAFANA_ENDPOINT }}
-      {{ if .DashboardURL -}}
-      Go to dashboard: {{ .DashboardURL }}
-      {{- end }}
-      {{ if .PanelURL -}}
-      Go to panel: {{ .PanelURL }}
-      {{- end }}
-      Details:
-      {{ range .Labels.SortedPairs -}}
-      - {{ .Name }}: `{{ .Value }}`
-      {{ end -}}
-      {{ if .SilenceURL -}}
-      Silence this alert: {{ .SilenceURL }}
-      {{- end }}
-      {{- end }}
-{% endraw %}
-{% endif %}

sbcli_pre-1.2.5/simplyblock_core/scripts/apply_dashboard.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+TD=$(dirname -- "$(readlink -f -- "$0")")
+
+# Grafana Password
+export grafanaPassword=$1
+
+# Grafana username
+GF_ADMIN_USER=admin
+
+HOST=0.0.0.0:3000
+
+DASHBOARDS="${TD}/dashboards"
+for dashboard in "${DASHBOARDS}/cluster.json" "${DASHBOARDS}/devices.json" "${DASHBOARDS}/nodes.json" "${DASHBOARDS}/lvols.json" "${DASHBOARDS}/pools.json" "${DASHBOARDS}/node-exporter.json"; do
+    echo -e "\nUploading dashboard: ${dashboard}"
+    curl -X POST -H "Content-Type: application/json" \
+        -d "@${dashboard}" \
+        "http://${GF_ADMIN_USER}:${grafanaPassword}@${HOST}/api/dashboards/import"
+    echo ""
+done
+
+echo "Cluster deployment complete."

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/cluster.json

@@ -2354,4 +2354,4 @@
         "version": 5,
         "weekStart": ""
     }
-}
+}

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/deploy_stack.sh

@@ -19,8 +19,6 @@ then
    export FDB_CLUSTER_FILE_CONTENTS=$FDB_CLUSTER_FILE_CONTENTS
 fi
 
-docker network create monitoring-net -d overlay --attachable
-
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -9,8 +9,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   opensearch:
     image: "opensearchproject/opensearch:2.4.0"
@@ -26,8 +24,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   graylog:
     hostname: "server"
@@ -38,7 +34,7 @@ services:
       GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
       GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}"
       GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
-      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost/graylog/"
+      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
       GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
       GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
     ports:
@@ -47,6 +43,7 @@ services:
     - "5140:5140/tcp"   # Syslog
     - "5555:5555/tcp"   # RAW TCP
     - "5555:5555/udp"   # RAW TCP
+    - "9000:9000/tcp"   # Server API
     - "12201:12201/tcp" # GELF TCP
     - "12201:12201/udp" # GELF UDP
     - "13301:13301/tcp" # Forwarder data
@@ -57,8 +54,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   promagent:
     image: simplyblock/promagent
@@ -69,16 +64,12 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   pushgateway:
     image: prom/pushgateway
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   prometheus:
     image: prom/prometheus:v2.44.0
@@ -94,8 +85,6 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   node-exporter:
     image: prom/node-exporter:v1.7.0
@@ -116,9 +105,7 @@ services:
       mode: global
       placement:
         constraints: [node.role == worker]
-    networks:
-      - monitoring-net
-
+        
   grafana:
     image: grafana/grafana:10.0.12
     environment:
@@ -127,16 +114,19 @@ services:
       GF_ALERTING_ENABLED: "true"
       GF_PATHS_PROVISIONING: "/etc/grafana/provisioning"
       GF_INSTALL_PLUGINS: "grafana-opensearch-datasource"
-      GF_SERVER_ROOT_URL: "http://localhost/grafana/"
     volumes:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
+    restart: "always"
+    ports:
+      - target: 3000
+        published: 3000
+        protocol: tcp
+        mode: host
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
 
   CleanupGraylog:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -146,8 +136,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    networks:
-      - monitoring-net
+
+  ### monitoring ###
 
 volumes:
   mongodb_data:
@@ -159,5 +149,6 @@ volumes:
   alertmanager_data:
 
 networks:
-  monitoring-net:
+  hostnet:
     external: true
+    name: host

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -114,7 +114,6 @@ services:
       - 8404:8404
     networks:
       - localnet
-      - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
 
@@ -186,20 +185,9 @@ services:
     networks:
       - hostnet
 
-  TasksRunnerRestart:
+  TasksRunner:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/tasks_runner_restart.py"
-    deploy:
-      placement:
-        constraints: [node.role == manager]
-    volumes:
-      - "/etc/foundationdb:/etc/foundationdb"
-    networks:
-      - hostnet
-
-  TasksRunnerMigration:
-    image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/tasks_runner_migration.py"
+    command: "python simplyblock_core/services/job_tasks.py"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -212,9 +200,6 @@ volumes:
   os_data:
 
 networks:
-  monitoring-net:
-    external: true
-
   hostnet:
     external: true
     name: host

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/haproxy.cfg

@@ -42,16 +42,6 @@ backend wep_api_services
     balance roundrobin
     server-template webapi- 3 WebAppAPI:5000 check resolvers docker init-addr libc,none
 
-backend grafana_services
-    balance roundrobin
-    http-request set-path %[path,regsub(^/grafana/?,/)]
-    server-template grafana- 1 grafana:3000 check resolvers docker init-addr libc,none
-
-backend graylog_services
-    balance roundrobin
-    http-request set-path %[path,regsub(^/graylog/?,/)]
-    server-template graylog- 1 graylog:9000 check resolvers docker init-addr libc,none
-
 frontend stats_front
     bind *:8404
     stats enable
@@ -62,9 +52,4 @@ frontend stats_front
 
 frontend web_api_front
     bind *:80
-
-    use_backend grafana_services if { path /grafana } || { path_beg /grafana/ }
-    use_backend graylog_services if { path /graylog } || { path_beg /graylog/ }
-
     default_backend wep_api_services
-

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/install_deps.sh

@@ -29,7 +29,6 @@ sudo sed -i 's/#X11UseLocalhost yes/X11UseLocalhost no/g' /etc/ssh/sshd_config
 
 sudo service sshd restart
 sudo modprobe nvme-tcp
-sudo modprobe nbd
 
 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
 

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -193,7 +193,7 @@ while True:
                     logger.info(f"Device is skipped: {device.get_id()} status: {device.status}")
                     continue
                 capacity_dict = rpc_client.alceml_get_capacity(device.alceml_bdev)
-                stats_dict = rpc_client.get_device_stats(device.nvme_bdev)
+                stats_dict = rpc_client.get_device_stats(device.alloc_bdev)
                 record = add_device_stats(cl, device, capacity_dict, stats_dict)
                 if record:
                     devices_records.append(record)

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/device_monitor.py

@@ -5,7 +5,7 @@ import sys
 import uuid
 
 from simplyblock_core import constants, kv_store
-from simplyblock_core.controllers import tasks_controller
+from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
@@ -27,6 +27,47 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController()
 
 
+def add_device_to_auto_restart(device):
+    tasks = db_controller.get_job_tasks(device.cluster_id)
+    for task in tasks:
+        if task.device_id == device.get_id():
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return
+
+    ds = JobSchedule()
+    ds.uuid = str(uuid.uuid4())
+    ds.cluster_id = device.cluster_id
+    ds.node_id = device.node_id
+    ds.device_id = device.get_id()
+    ds.date = int(time.time())
+    ds.function_name = "device_restart"
+    ds.status = 'new'
+
+    ds.write_to_db(db_store)
+    return ds.get_id()
+
+
+def add_node_to_auto_restart(node):
+    tasks = db_controller.get_job_tasks(node.cluster_id)
+    for task in tasks:
+        if task.node_id == node.get_id():
+            if task.status != JobSchedule.STATUS_DONE:
+                logger.info(f"Task found, skip adding new task: {task.get_id()}")
+                return
+
+    ds = JobSchedule()
+    ds.uuid = str(uuid.uuid4())
+    ds.cluster_id = node.cluster_id
+    ds.node_id = node.get_id()
+    ds.date = int(time.time())
+    ds.function_name = "node_restart"
+    ds.status = 'new'
+
+    ds.write_to_db(db_store)
+    return ds.get_id()
+
+
 logger.info("Starting Device monitor...")
 while True:
     nodes = db_controller.get_storage_nodes()
@@ -48,9 +89,9 @@ while True:
                 logger.info("Adding device to auto restart")
                 auto_restart_devices.append(dev)
 
-        if len(auto_restart_devices) >= 2 or len(online_devices) == 0:
-            tasks_controller.add_node_to_auto_restart(node)
-        elif len(auto_restart_devices) == 1:
-            tasks_controller.add_device_to_auto_restart(auto_restart_devices[0])
+        if len(auto_restart_devices) == 1:
+            add_device_to_auto_restart(auto_restart_devices[0])
+        elif len(auto_restart_devices) >= 2 and len(online_devices) == 0:
+            add_node_to_auto_restart(node)
 
     time.sleep(constants.DEV_MONITOR_INTERVAL_SEC)

{sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/distr_event_collector.py

@@ -88,7 +88,7 @@ def process_lvol_event(event):
     if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
         vuid = event.object_dict['vuid']
         lvol = None
-        for lv in db_controller.get_lvols():  # pass
+        for lv in db_controller.get_lvols():
             if lv.vuid == vuid:
                 lvol = lv
                 break
@@ -127,6 +127,7 @@ def process_event(event_id):
 
 hostname = utils.get_hostname()
 logger.info("Starting Distr event collector...")
+logger.info(f"Node:{hostname}")
 while True:
     time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
 
@@ -140,13 +141,14 @@ while True:
         snode.rpc_port,
         snode.rpc_username,
         snode.rpc_password,
-        timeout=10, retry=2)
-
+        timeout=3, retry=2
+    )
+    num_of_events = constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS
     try:
-        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
-
+        # events = client.distr_status_events_get()
+        events = client.distr_status_events_discard_then_get(0, num_of_events)
         if not events:
-            logger.debug("no events found")
+            logger.error("Distr events empty")
             continue
 
         logger.info(f"Found events: {len(events)}")
@@ -159,11 +161,10 @@ while True:
         for eid in event_ids:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
-
-            logger.info(f"Discarding events: {len(events)}")
-            client.distr_status_events_discard_then_get(len(events), 0)
+        logger.info(f"Discarding events: {num_of_events}")
+        events = client.distr_status_events_discard_then_get(num_of_events, 0)
 
     except Exception as e:
-        logger.error("Failed to process distr events")
+        logger.error("Failed to get distr events")
         logger.exception(e)
         continue