sbcli-pre 1.3.5__zip → 1.3.7__zip

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/scripts/datasource.yml

@@ -1,10 +1,11 @@
 apiVersion: 1
 datasources:
-- name: Prometheus
+- name: Thanos
   type: prometheus
-  url: http://prometheus:9090
+  url: http://thanos-query:9091
   isDefault: true
   access: proxy
+  uid: PBFA97CFB590B2093
   editable: true
 - name: GRAYLOG
   type: elasticsearch

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/scripts/deploy_stack.sh

@@ -21,13 +21,6 @@ fi
 
 docker network create monitoring-net -d overlay --attachable
 
-INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-
-#if [ -n "$INSTANCE_ID" ]
-#then
-#  export USE_EFS="rexray/efs"
-#fi
-
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -68,6 +68,7 @@ services:
       ClusterIP: "HAProxy"
       ClusterSecret: "${CLUSTER_SECRET}"
     deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -76,6 +77,7 @@ services:
   pushgateway:
     image: prom/pushgateway
     deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -90,9 +92,82 @@ services:
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.no-lockfile"
       - "--storage.tsdb.retention.time=${RETENTION_PERIOD}"
+      - "--storage.tsdb.min-block-duration=2h"
+      - "--storage.tsdb.max-block-duration=2h"
     restart: "always"
     deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-sidecar:
+    image: thanosio/thanos:v0.31.0
+    user: root
+    command:
+      - sidecar
+      - --tsdb.path=/prometheus
+      - --prometheus.url=http://prometheus:9090
+      - --objstore.config-file=/etc/thanos/objstore.yml
+    volumes:
+      - prometheus_data:/prometheus
+      - ./objstore.yml:/etc/thanos/objstore.yml
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-store:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - store
+      - --objstore.config-file=/etc/thanos/objstore.yml
+      - --index-cache-size=500MB
+      - --chunk-pool-size=500MB
+    volumes:
+      - ./objstore.yml:/etc/thanos/objstore.yml
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-query:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - query
+      - --http-address=0.0.0.0:9091
+      - --store=thanos-store:10901
+      - --store=thanos-sidecar:10901
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - monitoring-net
+
+  thanos-compactor:
+    image: thanosio/thanos:v0.31.0
+    command:
+      - compact
+      - --data-dir=/data
+      - --objstore.config-file=/etc/thanos/objstore.yml
+      - --retention.resolution-raw=30d
+      - --retention.resolution-5m=60d
+      - --retention.resolution-1h=90d
+      - --compact.concurrency=1
+      - --wait
+    volumes:
+      - ./objstore.yml:/etc/thanos/objstore.yml
+      - thanos_compactor_data:/data
+    deploy:
+      mode: global
       placement:
         constraints: [node.role == manager]
     networks:
@@ -133,6 +208,8 @@ services:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
+      - ./dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml
+      - ./dashboards:/var/lib/grafana/dashboards
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -152,19 +229,13 @@ services:
 
 volumes:
   mongodb_data:
-    driver: ${USE_EFS:-local}
   os_data:
-    driver: ${USE_EFS:-local}
   graylog_data:
-    driver: ${USE_EFS:-local}
   graylog_journal:
-    driver: ${USE_EFS:-local}
   grafana_data:
-    driver: ${USE_EFS:-local}
-  graylog_config:
-    driver: ${USE_EFS:-local}  
+  graylog_config:  
   prometheus_data:
-    driver: ${USE_EFS:-local}
+  thanos_compactor_data:
 
 networks:
   monitoring-net:

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/scripts/install_deps.sh

@@ -15,8 +15,6 @@ sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce do
 sudo systemctl enable docker
 sudo systemctl start docker
 
-#sudo docker plugin install rexray/efs --grant-all-permissions EFS_TAG=$INSTANCE_ID REXRAY_PREEMPT=true
-
 wget https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm -q
 sudo rpm -U foundationdb-clients-7.3.3-1.el7.x86_64.rpm --quiet --reinstall
 rm -f foundationdb-clients-7.3.3-1.el7.x86_64.rpm

sbcli_pre-1.3.7/simplyblock_core/scripts/objstore.yml

@@ -0,0 +1,3 @@
+type: FILESYSTEM
+config:
+  directory: /mnt/thanos

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/services/distr_event_collector.py

@@ -160,8 +160,8 @@ while True:
             logger.info(f"Processing event: {eid}")
             process_event(eid)
 
-            logger.info(f"Discarding events: {len(events)}")
-            client.distr_status_events_discard_then_get(len(events), 0)
+        logger.info(f"Discarding events: {len(events)}")
+        client.distr_status_events_discard_then_get(len(events), 0)
 
     except Exception as e:
         logger.error("Failed to process distr events")

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/services/health_check_service.py

@@ -125,7 +125,7 @@ while True:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... ok")
                     else:
                         logger.info(f"Checking bdev: {remote_device.remote_bdev} ... not found")
-                    node_remote_devices_check &= bool(ret)
+                    # node_remote_devices_check &= bool(ret)
 
                 health_check_status = is_node_online and node_devices_check and node_remote_devices_check
             set_node_health_check(snode, health_check_status)

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/services/tasks_runner_restart.py

@@ -5,7 +5,7 @@ import sys
 
 
 from simplyblock_core import constants, kv_store, storage_node_ops
-from simplyblock_core.controllers import device_controller, tasks_events
+from simplyblock_core.controllers import device_controller, tasks_events, health_controller
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice
 
@@ -169,6 +169,19 @@ def task_runner_node(task):
         task.write_to_db(db_controller.kv_store)
         tasks_events.task_updated(task)
 
+    # is node reachable?
+    ping_check = health_controller._check_node_ping(node.mgmt_ip)
+    logger.info(f"Check: ping mgmt ip {node.mgmt_ip} ... {ping_check}")
+    node_api_check = health_controller._check_node_api(node.mgmt_ip)
+    logger.info(f"Check: node API {node.mgmt_ip}:5000 ... {node_api_check}")
+    if not ping_check or not node_api_check:
+        # node is unreachable, retry
+        logger.info(f"Node is not reachable: {task.node_id}, retry")
+        task.function_result = f"Node is unreachable, retry"
+        task.retry += 1
+        task.write_to_db(db_controller.kv_store)
+        return False
+
     # shutting down node
     logger.info(f"Shutdown node {node.get_id()}")
     ret = storage_node_ops.shutdown_storage_node(node.get_id(), force=True)

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_core/storage_node_ops.py

@@ -973,6 +973,12 @@ def remove_storage_node(node_id, force_remove=False, force_migrate=False):
         logger.error(f"Can not remove online node: {node_id}")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not remove storage node")
+        if force_remove is False:
+            return False
+
     if snode.lvols:
         if force_migrate:
             for lvol_id in snode.lvols:
@@ -1064,6 +1070,11 @@ def restart_storage_node(
         logger.error(f"Can not restart online node: {node_id}")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not restart storage node")
+        return False
+
     logger.info("Setting node state to restarting")
     old_status = snode.status
     snode.status = StorageNode.STATUS_RESTARTING
@@ -1469,16 +1480,11 @@ def shutdown_storage_node(node_id, force=False):
         if force is False:
             return False
 
-    # cls = db_controller.get_clusters(id=snode.cluster_id)
-    # snodes = db_controller.get_storage_nodes()
-    # online_nodes = 0
-    # for node in snodes:
-    #     if node.status == node.STATUS_ONLINE:
-    #         online_nodes += 1
-    # if cls[0].ha_type == "ha" and online_nodes <= 3:
-    #     logger.warning(f"Cluster mode is HA but online storage nodes are less than 3")
-    #     if force is False:
-    #         return False
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not shutdown storage node")
+        if force is False:
+            return False
 
     logger.info("Shutting down node")
     old_status = snode.status
@@ -1538,7 +1544,14 @@ def suspend_storage_node(node_id, force=False):
     logger.info("Node found: %s in state: %s", snode.hostname, snode.status)
     if snode.status != StorageNode.STATUS_ONLINE:
         logger.error("Node is not in online state")
-        return False
+        if force is False:
+            return False
+
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not suspend storage node")
+        if force is False:
+            return False
 
     cluster = db_controller.get_cluster_by_id(snode.cluster_id)
     snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
@@ -1599,6 +1612,11 @@ def resume_storage_node(node_id):
         logger.error("Node is not in suspended state")
         return False
 
+    task_id = tasks_controller.get_active_node_restart_task(snode.cluster_id, snode.get_id())
+    if task_id:
+        logger.error(f"Restart task found: {task_id}, can not resume storage node")
+        return False
+
     logger.info("Resuming node")
 
     logger.info("Sending cluster event updates")

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_web/blueprints/snode_ops.py

@@ -100,9 +100,9 @@ def spdk_process_start():
             node.remove(force=True)
             time.sleep(2)
 
-    spdk_debug = 0
+    spdk_debug = ""
     if set_debug:
-        spdk_debug = 1
+        spdk_debug = "1"
 
     spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
     if 'spdk_image' in data and data['spdk_image']:

{sbcli_pre-1.3.5 → sbcli_pre-1.3.7}/simplyblock_web/blueprints/web_api_deployer.py

@@ -30,8 +30,8 @@ ssm = boto3.client('ssm', region_name=region)
 s3 = boto3.client('s3', region_name=region)
 
 
-def get_instance_tf_engine_instance_id():
-    tag_value = 'tfengine'
+def get_instance_tf_engine_instance_id(workspace: str):
+    tag_value = f'{workspace}-tfengine'
     tag_key = 'Name'
 
     ec2 = boto3.client('ec2', region_name=region)
@@ -145,7 +145,7 @@ def update_cluster(d, kv_store, storage_nodes, availability_zone):
     d.status = "in_progress"
     d.write_to_db(kv_store)
 
-    instance_ids = get_instance_tf_engine_instance_id()
+    instance_ids = get_instance_tf_engine_instance_id(d.tf_workspace)
     if len(instance_ids) == 0:
         # wait for a min and try again before returning error on the API
         print('no instance IDs')