sbcli-pre 1.3.6__zip → 1.3.7__zip

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/scripts/datasource.yml

@@ -1,10 +1,11 @@
 apiVersion: 1
 datasources:
-- name: Prometheus
+- name: Thanos
   type: prometheus
-  url: http://prometheus:9090
+  url: http://thanos-query:9091
   isDefault: true
   access: proxy
+  uid: PBFA97CFB590B2093
   editable: true
 - name: GRAYLOG
   type: elasticsearch

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/scripts/deploy_stack.sh

@@ -21,13 +21,6 @@ fi
 
 docker network create monitoring-net -d overlay --attachable
 
-INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-
-#if [ -n "$INSTANCE_ID" ]
-#then
-#  export USE_EFS="rexray/efs"
-#fi
-
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -68,6 +68,7 @@ services:
       ClusterIP: "HAProxy"
       ClusterSecret: "${CLUSTER_SECRET}"
     deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -76,6 +77,7 @@ services:
   pushgateway:
     image: prom/pushgateway
     deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -90,9 +92,82 @@ services:
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.no-lockfile"
       - "--storage.tsdb.retention.time=${RETENTION_PERIOD}"
+      - "--storage.tsdb.min-block-duration=2h"
+      - "--storage.tsdb.max-block-duration=2h"
     restart: "always"
     deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-sidecar:
+    image: thanosio/thanos:v0.31.0
+    user: root
+    command:
+      - sidecar
+      - --tsdb.path=/prometheus
+      - --prometheus.url=http://prometheus:9090
+      - --objstore.config-file=/etc/thanos/objstore.yml
+    volumes:
+      - prometheus_data:/prometheus
+      - ./objstore.yml:/etc/thanos/objstore.yml
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-store:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - store
+      - --objstore.config-file=/etc/thanos/objstore.yml
+      - --index-cache-size=500MB
+      - --chunk-pool-size=500MB
+    volumes:
+      - ./objstore.yml:/etc/thanos/objstore.yml
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-query:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - query
+      - --http-address=0.0.0.0:9091
+      - --store=thanos-store:10901
+      - --store=thanos-sidecar:10901
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-compactor:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - compact
+      - --data-dir=/data
+      - --objstore.config-file=/etc/thanos/objstore.yml
+      - --retention.resolution-raw=30d
+      - --retention.resolution-5m=60d
+      - --retention.resolution-1h=90d
+      - --compact.concurrency=1
+      - --wait
+    volumes:
+      - ./objstore.yml:/etc/thanos/objstore.yml
+      - thanos_compactor_data:/data
+    deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -133,6 +208,8 @@ services:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
+      - ./dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml
+      - ./dashboards:/var/lib/grafana/dashboards
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -152,19 +229,13 @@ services:
 
 volumes:
   mongodb_data:
-    driver: ${USE_EFS:-local}
   os_data:
-    driver: ${USE_EFS:-local}
   graylog_data:
-    driver: ${USE_EFS:-local}
   graylog_journal:
-    driver: ${USE_EFS:-local}
   grafana_data:
-    driver: ${USE_EFS:-local}
-  graylog_config:
-    driver: ${USE_EFS:-local}  
+  graylog_config:  
   prometheus_data:
-    driver: ${USE_EFS:-local}
+  thanos_compactor_data:
 
 networks:
   monitoring-net:

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/scripts/install_deps.sh

@@ -15,8 +15,6 @@ sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce do
 sudo systemctl enable docker
 sudo systemctl start docker
 
-#sudo docker plugin install rexray/efs --grant-all-permissions EFS_TAG=$INSTANCE_ID REXRAY_PREEMPT=true
-
 wget https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm -q
 sudo rpm -U foundationdb-clients-7.3.3-1.el7.x86_64.rpm --quiet --reinstall
 rm -f foundationdb-clients-7.3.3-1.el7.x86_64.rpm

sbcli_pre-1.3.7/simplyblock_core/scripts/objstore.yml

@@ -0,0 +1,3 @@
+type: FILESYSTEM
+config:
+  directory: /mnt/thanos

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/services/health_check_service.py

@@ -125,7 +125,7 @@ while True:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                     else:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    node_remote_devices_check &= bool(ret)
+                    # node_remote_devices_check &= bool(ret)
 
                 health_check_status = is_node_online and node_devices_check and node_remote_devices_check
             set_node_health_check(snode, health_check_status)

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_core/storage_node_ops.py

@@ -973,6 +973,12 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
         logger.error(f"Can not remove online node: {node_id}")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not remove storage node")
+        if force_remove is False:
+            return False
+
     if snode.lvols:
         if force_migrate:
             for lvol_id in snode.lvols:
@@ -1064,6 +1070,11 @@ def restart_storage_node(
         logger.error(f"Can not restart online node: {node_id}")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not restart storage node")
+        return False
+
     logger.info("Setting node state to restarting")
     old_status = snode.status
     snode.status = StorageNode.STATUS_RESTARTING
@@ -1469,16 +1480,11 @@ def shutdown_storage_node(node_id, force=False):
         if force is False:
             return False
 
-    # cls = db_controller.get_clusters(id=snode.cluster_id)
-    # snodes = db_controller.get_storage_nodes()
-    # online_nodes = 0
-    # for node in snodes:
-    #     if node.status == node.STATUS_ONLINE:
-    #         online_nodes += 1
-    # if cls[0].ha_type == "ha" and online_nodes <= 3:
-    #     logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
-    #     if force is False:
-    #         return False
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not shutdown storage node")
+        if force is False:
+            return False
 
     logger.info("Shutting down node")
     old_status = snode.status
@@ -1538,7 +1544,14 @@ def suspend_storage_node(node_id, force=False):
     logger.info("Node found: %s in state: %s", snode.hostname, snode.status)
     if snode.status != StorageNode.STATUS_ONLINE:
         logger.error("Node is not in online state")
-        return False
+        if force is False:
+            return False
+
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not suspend storage node")
+        if force is False:
+            return False
 
     cluster = db_controller.get_cluster_by_id(snode.cluster_id)
     snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
@@ -1599,6 +1612,11 @@ def resume_storage_node(node_id):
         logger.error("Node is not in suspended state")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not resume storage node")
+        return False
+
     logger.info("Resuming node")
 
     logger.info("Sending cluster event updates")

{sbcli_pre-1.3.6 → sbcli_pre-1.3.7}/simplyblock_web/blueprints/snode_ops.py

@@ -100,9 +100,9 @@ def spdk_process_start():
             node.remove(force=True)
             time.sleep(2)
 
-    spdk_debug = 0
+    spdk_debug = ""
     if set_debug:
-        spdk_debug = 1
+        spdk_debug = "1"
 
     spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
     if 'spdk_image' in data and data['spdk_image']: