mas-cli 12.0.0__py3-none-any.whl → 13.0.0__py3-none-any.whl

mas/cli/templates/ibm-mas-tekton.yaml

@@ -231,7 +231,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - aibroker
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/appconnect.yaml
@@ -342,7 +342,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - appconnect
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -460,7 +460,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - arcgis
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/cert-manager.yaml
@@ -530,7 +530,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - cert_manager
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 # --------------------------------------------------------------------------------
@@ -596,7 +596,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - common_services
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 # --------------------------------------------------------------------------------
@@ -723,7 +723,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - cos
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -843,7 +843,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - cp4d_service
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 # --------------------------------------------------------------------------------
@@ -960,7 +960,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - cp4d_service
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -1066,7 +1066,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - cp4d
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/db2.yaml
@@ -1388,7 +1388,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - db2
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -1498,7 +1498,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - eck
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/gencfg-workspace.yaml
@@ -1587,7 +1587,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - gencfg_workspace
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -1691,7 +1691,7 @@ spec:
         - -c
       name: gitops-bootstrap
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -1773,7 +1773,7 @@ spec:
         - -c
       name: gitops-cis-compliance
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -1879,9 +1879,17 @@ spec:
     - name: group_sync_operator_isv_groups
       type: string
       default: ''
+
     - name: ibm_rbac_binding_to_group
       type: string
       default: ''
+    
+    - name: falcon_operator_cloud_region
+      type: string
+      default: ''
+    - name: falcon_operator_node_sensor
+      type: string
+      default: ''
 
   stepTemplate:
     name: gitops-cluster
@@ -1959,8 +1967,14 @@ spec:
         value: $(params.group_sync_operator_isv_tenant_url)
       - name: GROUP_SYNC_OPERATOR_ISV_GROUPS
         value: $(params.group_sync_operator_isv_groups)
+
       - name: IBM_RBAC_BINDING_TO_GROUP
         value: $(params.ibm_rbac_binding_to_group)
+      
+      - name: FALCON_OPERATOR_CLOUD_REGION
+        value: $(params.falcon_operator_cloud_region)
+      - name: FALCON_OPERATOR_NODE_SENSOR
+        value: $(params.falcon_operator_node_sensor)
 
     envFrom:
       - configMapRef:
@@ -2002,6 +2016,13 @@ spec:
           export INSTALL_IBM_RBAC=true
         fi
 
+        if [[ -n "${FALCON_OPERATOR_CLOUD_REGION}" ]];then
+          echo "Setting var INSTALL_FALCON_OPERATOR to true as var FALCON_OPERATOR_CLOUD_REGION is set"
+          export INSTALL_FALCON_OPERATOR=true
+        else
+          echo "Not setting var INSTALL_FALCON_OPERATOR to true as var FALCON_OPERATOR_CLOUD_REGION is not set"
+        fi
+
         mkdir -p /tmp/init-cluster
         mas gitops-cluster -a $ACCOUNT -c $CLUSTER_NAME \
         --dir /tmp/init-cluster \
@@ -2018,7 +2039,7 @@ spec:
         - -c
       name: gitops-cluster
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -2121,7 +2142,7 @@ spec:
         - -c
       name: gitops-cos
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -2269,7 +2290,7 @@ spec:
         - -c
       name: gitops-cp4d-service
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -2398,7 +2419,7 @@ spec:
         - -c
       name: gitops-cp4d
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -2659,7 +2680,7 @@ spec:
         - -c
       name: gitops-db2u-database
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -2757,7 +2778,7 @@ spec:
         - -c
       name: gitops-db2u
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -2884,7 +2905,7 @@ spec:
         - -c
       name: gitops-delete-jdbc-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -2982,7 +3003,7 @@ spec:
         - -c
       name: gitops-delete-kafka-config
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -3074,7 +3095,7 @@ spec:
         - -c
       name: gitops-deprovision-app-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -3159,7 +3180,7 @@ spec:
         - -c
       name: gitops-deprovision-app-install
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -3244,7 +3265,7 @@ spec:
         - -c
       name: gitops-deprovision-cluster
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -3363,7 +3384,7 @@ spec:
         - -c
       name: gitops-deprovision-cos
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -3451,7 +3472,7 @@ spec:
         - -c
       name: gitops-deprovision-db2u-database
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -3534,7 +3555,7 @@ spec:
         - -c
       name: gitops-deprovision-db2u
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -3642,7 +3663,7 @@ spec:
         - -c
       name: gitops-deprovision-efs
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -3752,7 +3773,7 @@ spec:
         - -c
       name: gitops-deprovision-kafka
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -3855,7 +3876,7 @@ spec:
         - -c
       name: gitops-deprovision-mongo
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -3912,7 +3933,7 @@ spec:
         - -c
       name: gitops-deprovision-rosa
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4117,7 +4138,7 @@ spec:
         - -c
       name: gitops-deprovision-suite-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -4215,7 +4236,7 @@ spec:
         - -c
       name: gitops-deprovision-suite-idp-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4307,7 +4328,127 @@ spec:
         - -c
       name: gitops-deprovision-suite-objectstorage-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
+  workspaces:
+    - name: configs
+# --------------------------------------------------------------------------------
+# /home/runner/work/cli/cli/tekton/target/tasks/gitops-deprovision-suite-sendgrid-subuser.yaml
+# --------------------------------------------------------------------------------
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: gitops-deprovision-suite-sendgrid-subuser
+spec:
+  params:
+    - name: cluster_name
+      type: string
+    - name: account
+      type: string
+    - name: mas_instance_id
+      type: string
+    - name: icn
+      type: string
+    - name: cis_mas_domain
+      type: string
+    - name: cis_crn
+      type: string
+    - name: avp_aws_secret_region
+      type: string
+  stepTemplate:
+    name: gitops-deprovision-suite-sendgrid-subuser
+    env:
+      - name: CLUSTER_ID
+        value: $(params.cluster_name)
+      - name: ACCOUNT_ID
+        value: $(params.account)
+      - name: MAS_INSTANCE_ID
+        value: $(params.mas_instance_id)
+
+      - name: ICN
+        value: $(params.icn)
+      - name: CIS_MAS_DOMAIN
+        value: $(params.cis_mas_domain)
+      - name: CIS_CRN
+        value: $(params.cis_crn)
+
+      - name: SM_AWS_REGION
+        value: $(params.avp_aws_secret_region)
+    envFrom:
+      - configMapRef:
+          name: environment-properties
+          optional: true
+      - secretRef:
+          name: secure-properties
+  steps:
+    - args:
+      - |-
+
+        # Expected secure-properties (sourced from IBM Cloud SM):
+        # -------------------
+        # SM_AWS_ACCESS_KEY_ID
+        # SM_AWS_SECRET_ACCESS_KEY
+        # SENDGRID_API_KEY
+
+
+        # Teardown of smtp config in gitops-envs is handled by gitops-deprovision-suite-smtp-config task
+        # All we need to do here is make sure we clean up SendGrid, CIS and the sendgrid_subuser secret in AWS SM (if the subuser was deleted)
+        source /mascli/functions/gitops_utils
+        export AVP_TYPE="aws"
+        sm_login || exit 1
+
+        # Fetch CIS API Key from AWS SM.
+        # This is suitable for use with CIS instances used by MAS instances in the cluster we are targetting
+        # This secret is only required if the instance is configured with CIS
+        if [[ -n "${CIS_CRN}" ]]; then
+          SECRET_NAME_CIS="${ACCOUNT_ID}/${CLUSTER_ID}/cis"
+          echo "Getting ${SECRET_NAME_CIS} from AWS SM"
+          export CIS_APIKEY="$(sm_get_secret_value "${SECRET_NAME_CIS}" "ibm_apikey")" # pragma: allowlist secret
+          if [[ -z "${CIS_APIKEY}" || "${CIS_APIKEY}" == "null" ]]; then
+            echo "Required AWS SM secret "${SECRET_NAME_CIS}" not found or invalid"
+            exit 1
+          fi
+        fi
+
+        mkdir -p /tmp/gitops-deprovision-suite-sendgrid-subuser
+        OUTPUT_FILE="/tmp/gitops-deprovision-suite-sendgrid-subuser/mas-saas-sendgrid-subuser-output.yaml"
+        rm "${OUTPUT_FILE}"
+
+        mas-saas-sendgrid-subuser \
+          --customer-id "${ICN}" \
+          --mas-account-id "${ACCOUNT_ID}" \
+          --mas-cluster-id "${CLUSTER_ID}" \
+          --mas-instance-id "${MAS_INSTANCE_ID}" \
+          --cis-crn "${CIS_CRN}" \
+          --cis-mas-domain "${CIS_MAS_DOMAIN}" \
+          --output-file "${OUTPUT_FILE}" \
+          --action delete
+        
+        rc="$?"
+
+        if [[ "${rc}" != "0" ]]; then
+          echo "mas-saas-sendgrid-subuser failed with rc ${rc}"
+          exit ${rc}
+        fi
+
+        # Pull values out of the outfile and set them as environment vars
+        if [[ -f "${OUTPUT_FILE}" ]]; then
+          echo "Reading outputs from ${OUTPUT_FILE}"
+          export DELETED_SUBUSER="$(yq '.deleted_subuser // ""' "${OUTPUT_FILE}")"
+        fi
+
+        if [[ "${DELETED_SUBUSER}" == "true" ]]; then
+          SECRET_NAME_SENDGRID="ibm-customer/${ICN}/sendgrid_subuser"
+          echo "Subuser was deleted, cleaning up ${SECRET_NAME_SENDGRID} secret"
+          sm_delete_secret "${SECRET_NAME_SENDGRID}"
+        fi
+
+      command:
+        - /bin/sh
+        - -c
+      name: gitops-deprovision-suite-sendgrid-subuser
+      imagePullPolicy: IfNotPresent
+      image: docker-na-public.artifactory.swg-devops.com/wiotp-docker-local/mas/saas-task:latest
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4399,7 +4540,7 @@ spec:
         - -c
       name: gitops-deprovision-suite-smtp-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4492,7 +4633,7 @@ spec:
         - -c
       name: gitops-deprovision-suite-watson-studio-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4583,7 +4724,7 @@ spec:
         - -c
       name: gitops-deprovision-suite-workspace
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4676,7 +4817,7 @@ spec:
         - -c
       name: gitops-deprovision-suite
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -4789,7 +4930,7 @@ spec:
         - -c
       name: gitops-dro
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -4906,7 +5047,7 @@ spec:
         - -c
       name: gitops-efs
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -5041,7 +5182,7 @@ spec:
         - -c
       name: gitops-jdbc-config
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -5137,7 +5278,7 @@ spec:
         - -c
       name: gitops-kafka-config
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -5259,7 +5400,7 @@ spec:
         - -c
       name: gitops-kafka
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -5385,7 +5526,7 @@ spec:
         - -c
       name: gitops-license
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: shared-entitlement
 
@@ -5559,6 +5700,14 @@ spec:
       type: string
       default: ""
 
+    - name: icn
+      type: string
+    - name: avp_aws_secret_region
+      type: string
+    - name: use_sendgrid
+      type: string
+      default: ""
+
   stepTemplate:
     name: gitops-mas-fvt-preparer
     env:
@@ -5684,6 +5833,13 @@ spec:
       - name: LDAP_CERT_ALIAS
         value: ldap
 
+      - name: ICN
+        value: $(params.icn)
+      - name: SM_AWS_REGION
+        value: $(params.avp_aws_secret_region)
+      - name: USE_SENDGRID
+        value: $(params.use_sendgrid)
+
     envFrom:
       - configMapRef:
           name: environment-properties
@@ -5775,6 +5931,52 @@ spec:
           check_argo_app_healthy "${SUITE_APP_NAME}" 30
           check_argo_app_healthy "${WORKSPACE_APP}" 30
         fi
+
+
+        # If use_sendgrid: true, disable the subuser so we do not accidentally send out real emails when running tests against the instance
+        # NOTE: Many of the FVT suites will fail unless the suite is configured to use Mailhog for SMTP, so we only plan to have smtp.use_sendgrid: true set for fvtsaastran
+        #       We only run the catalogapi FVT suite there at present, and that suite does not depend on Mailhog.
+        # NOTE: we deliberately perform this step *after* checking application health, since disabling the sendgrid subuser will cause the validation step in the
+        #       SMTP entity manager to fail (unfortunately, there is no way to configure a SendGrid subuser to silently drop emails without the client call reporting failure)
+        # NOTE: although we don't check the health of the SMTP app explicitly, this is not necessary since the WORKSPACE_APP (which we do check above) sync will be blocked until the SMTP
+        #       app becomes healthy (if SMTP is configured)
+        if [[ "${USE_SENDGRID}" == "true" ]]; then
+
+          echo "Disabling sendgrid subuser to prevent the suite from sending out emails during test execution"
+
+          export AVP_TYPE="aws" # required by sm_login (only AWS supported at present)
+          sm_login || exit 1
+
+          # lookup ibm-customer/<ICN>/sendgrid_subuser#username from AWS SM
+          SECRET_NAME_SENDGRID="ibm-customer/${ICN}/sendgrid_subuser"
+          echo "Getting ${SECRET_NAME_SENDGRID} from AWS SM"
+          export SENDGRID_SUBUSER_USERNAME="$(sm_get_secret_value "${SECRET_NAME_SENDGRID}" "username")" # pragma: allowlist secret
+          echo "Subuser username: ${SENDGRID_SUBUSER_USERNAME}"
+          if [[ -z "${SENDGRID_SUBUSER_USERNAME}" || "${SENDGRID_SUBUSER_USERNAME}" == "null" ]]; then
+            echo "Required AWS SM secret "${SECRET_NAME_SENDGRID}" not found or invalid"
+            exit 1
+          fi
+
+          curl -X PATCH \
+            https://api.sendgrid.com/v3/subusers/${SENDGRID_SUBUSER_USERNAME} \
+            --fail \
+            -H "Authorization: Bearer ${SENDGRID_API_KEY}" \
+            -H "Content-Type: application/json" \
+            -d '{"disabled": true}'
+          CURL_RC=$?
+          if [ $CURL_RC -ne 0 ]; then
+            echo "Failed to disable SendGrid subuser, aborting test"
+            echo "WARNING: until the SendGrid subuser is disabled, the suite will be capable of sending emails for real!"
+            echo "         do not attempt to run any tests against the environment until the subuser is successfully disabled!"
+            exit 1
+          fi
+          echo "SendGrid subuser ${SENDGRID_SUBUSER_USERNAME} disabled successfully!"
+          echo "It is now safe to run tests against the environment; the suite is no longer capable of sending emails for real."
+
+        fi
+
+        # NOTE: verified that subuser teardown (including deletion of its API keys, authenticated domains and DNS records) still works as expected against a disabled subuser
+        #       so there is no need to re-enable the subuser before attempting to deprovision the MAS instance.
       
         if [[ "$LAUNCHER_ID" == "apps" ]]; then
           # The following order is defined by the sync wave order in https://github.com/ibm-mas/gitops/tree/main/root-applications/ibm-mas-instance-root/templates
@@ -5888,7 +6090,7 @@ spec:
         - -c
       name: gitops-mas-fvt-preparer
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-additional-configs
@@ -6330,7 +6532,7 @@ spec:
         - -c
       name: gitops-mas-initiator
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -6438,7 +6640,7 @@ spec:
         - -c
       name: gitops-mongo
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -6560,7 +6762,7 @@ spec:
         - -c
       name: gitops-nvidia-gpu
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -6678,7 +6880,7 @@ spec:
         - -c
       name: gitops-process-mongo-user
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -6734,7 +6936,7 @@ spec:
         - -c
       name: gitops-rosa
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -6921,7 +7123,7 @@ spec:
         - -c
       name: gitops-suite-app-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -7079,7 +7281,7 @@ spec:
         - -c
       name: gitops-suite-app-install
       imagePullPolicy: Always
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -7185,7 +7387,7 @@ spec:
         - -c
       name: gitops-suite-certs
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: certificates
@@ -7355,7 +7557,7 @@ spec:
         - -c
       name: gitops-suite-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-additional-configs
@@ -7455,7 +7657,7 @@ spec:
         - -c
       name: gitops-suite-dns
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -7579,7 +7781,7 @@ spec:
         - -c
       name: gitops-suite-idp-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-additional-configs
@@ -7680,10 +7882,268 @@ spec:
         - -c
       name: gitops-suite-objectstorage-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
+# --------------------------------------------------------------------------------
+# /home/runner/work/cli/cli/tekton/target/tasks/gitops-suite-smtp-config-sendgrid.yaml
+# --------------------------------------------------------------------------------
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: gitops-suite-smtp-config-sendgrid
+spec:
+  params:
+    - name: cluster_name
+      type: string
+    - name: account
+      type: string
+    - name: secrets_path
+      type: string
+    - name: mas_instance_id
+      type: string
+    - name: git_branch
+      type: string
+    - name: github_org
+      type: string
+    - name: github_repo
+      type: string
+    - name: github_host
+      type: string
+    - name: avp_aws_secret_region
+      type: string
+    - name: mas_smtpcfg_pod_template_yaml
+      type: string
+      default: ""
+
+    - name: icn
+      type: string
+    - name: mas_domain
+      type: string
+    - name: cis_mas_domain
+      type: string
+    - name: cis_crn
+      type: string
+  stepTemplate:
+    name: gitops-suite-smtp-config
+    env:
+      - name: CLUSTER_ID
+        value: $(params.cluster_name)
+      - name: ACCOUNT_ID
+        value: $(params.account)
+      - name: SECRETS_PATH
+        value: $(params.secrets_path)
+      - name: MAS_INSTANCE_ID
+        value: $(params.mas_instance_id)
+      - name: GIT_BRANCH
+        value: $(params.git_branch)
+      - name: GITHUB_ORG
+        value: $(params.github_org)
+      - name: GITHUB_HOST
+        value: $(params.github_host)
+      - name: GITHUB_REPO
+        value: $(params.github_repo)
+      - name: SM_AWS_REGION
+        value: $(params.avp_aws_secret_region)
+      - name: MAS_SMTPCFG_POD_TEMPLATE_YAML
+        value: $(params.mas_smtpcfg_pod_template_yaml)
+
+      - name: ICN
+        value: $(params.icn)
+      - name: MAS_DOMAIN
+        value: $(params.mas_domain)
+      - name: CIS_MAS_DOMAIN
+        value: $(params.cis_mas_domain)
+      - name: CIS_CRN
+        value: $(params.cis_crn)
+    envFrom:
+      - configMapRef:
+          name: environment-properties
+          optional: true
+      - secretRef:
+          name: secure-properties
+
+  steps:
+    - args:
+      - |-
+
+        # Expected secure-properties (sourced from IBM Cloud SM):
+        # -------------------
+        # SM_AWS_ACCESS_KEY_ID
+        # SM_AWS_SECRET_ACCESS_KEY
+        # SENDGRID_API_KEY
+        # GITHUB_PAT
+
+        source /mascli/functions/gitops_utils
+
+        export AVP_TYPE="aws"
+        sm_login || exit 1
+
+        # Fetch CIS API Key from AWS SM.
+        # This is suitable for use with CIS instances used by MAS instances in the cluster we are targetting
+        SECRET_NAME_CIS="${ACCOUNT_ID}/${CLUSTER_ID}/cis"
+        echo "Getting ${SECRET_NAME_CIS} from AWS SM"
+        export CIS_APIKEY="$(sm_get_secret_value "${SECRET_NAME_CIS}" "ibm_apikey")" # pragma: allowlist secret
+        if [[ -z "${CIS_APIKEY}" || "${CIS_APIKEY}" == "null" ]]; then
+          echo "Required AWS SM secret "${SECRET_NAME_CIS}" not found or invalid"
+          exit 1
+        fi
+
+        mkdir -p /tmp/init-suite-smtp-config
+
+        git config --global user.name "MAS Automation"
+        git config --global user.email "you@example.com"
+        git config --global user.password "${GITHUB_PAT}"
+
+        OUTPUT_FILE="/tmp/init-suite-smtp-config/mas-saas-sendgrid-subuser-output.yaml"
+        rm "${OUTPUT_FILE}"
+
+        mas-saas-sendgrid-subuser \
+          --customer-id "${ICN}" \
+          --cis-mas-domain "${CIS_MAS_DOMAIN}" \
+          --mas-account-id "${ACCOUNT_ID}" \
+          --mas-cluster-id "${CLUSTER_ID}" \
+          --mas-instance-id "${MAS_INSTANCE_ID}" \
+          --cis-crn "${CIS_CRN}" \
+          --output-file "${OUTPUT_FILE}" \
+          --action create
+        
+        rc="$?"
+
+        echo "mas-saas-sendgrid-subuser rc: ${rc}"
+
+        # Pull values out of the outfile and set them as environment vars
+        if [[ -f "${OUTPUT_FILE}" ]]; then
+          echo "Reading outputs from ${OUTPUT_FILE}"
+          export SUBUSER_USERNAME="$(yq '.subuser.username // ""' "${OUTPUT_FILE}")" # pragma: allowlist secret
+          export SUBUSER_PASSWORD="$(yq '.subuser.password // ""' "${OUTPUT_FILE}")" # pragma: allowlist secret
+          export SENDGRID_APIKEY_USERNAME="apikey" # pragma: allowlist secret
+          export SENDGRID_APIKEY_PASSWORD="$(yq '.apikey // ""' "${OUTPUT_FILE}")" # pragma: allowlist secret
+        fi
+
+        # If an API key was generated by the script (even if something went wrong in later step)
+        # this is our only opportunity to obtain its value so we need to register it in secrets manager now
+        # so it'll be present when whatever the issue is resolved and the instance pipeline is rerun
+        update_apikey_secret_rc="0"
+        if [[ -n "${SENDGRID_APIKEY_PASSWORD}" ]]; then
+          SECRET_NAME_SMTP="${ACCOUNT_ID}/${CLUSTER_ID}/${MAS_INSTANCE_ID}/smtp"
+          echo "Putting generated API Key in ${SECRET_NAME_SMTP}"
+
+          # NOTE: deliberately not using $SMTP_USERNAME or $SMTP_PASSWORD here, since we don't need/want the gitops-mas-config to repeat this step
+          TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_suite_smtp_config_sendgrid\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]"
+          # NOTE: running function in a subshell so we don't exit this script if something goes wrong (necessary due to the use of "set +o pipefail" in sm_update_secret)
+          (sm_update_secret "${SECRET_NAME_SMTP}" "{\"username\": \"$SENDGRID_APIKEY_USERNAME\", \"password\": \"$SENDGRID_APIKEY_PASSWORD\"}" "${TAGS}")
+
+          # defer exiting the script if the sm_update_secret call above failed, so we have an opportunity to register other generated secrets
+          update_apikey_secret_rc="$?"
+        fi
+
+        # If a new subuser was created by the script (even if something went wrong in a later step)
+        # this is our only opportunity to store the password we generated for the subuser
+        update_subuser_secret_rc="0"
+        if [[ -n "${SUBUSER_PASSWORD}" ]]; then
+          SECRET_NAME_SENDGRID="ibm-customer/${ICN}/sendgrid_subuser"
+          TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_suite_smtp_config_sendgrid\"}]"
+          echo "Putting generated subuser credentials in ${SECRET_NAME_SENDGRID}"
+          (sm_update_secret "${SECRET_NAME_SENDGRID}" "{\"username\": \"$SUBUSER_USERNAME\", \"password\": \"$SUBUSER_PASSWORD\"}" "${TAGS}")
+
+          # defer exiting the script if the sm_update_secret call above failed, so we have an opportunity to register other generated secrets
+          update_subuser_secret_rc="$?"
+        fi
+
+        echo "update_apikey_secret_rc: ${update_apikey_secret_rc}"
+        echo "update_subuser_secret_rc: ${update_subuser_secret_rc}"
+
+        if [[ "${update_apikey_secret_rc}" != "0" ]]; then
+          # We could add additional automation to deal with the (rare) edge case of the SM update failing.
+          # i.e. add a "force_regenerate_api_key" flag that can be set before rerunning the pipeline
+          # but I think - given the rarity - just documenting the manual resolution steps below will suffice.
+          # another mitigation would be to add retry logic to the sm_update_secret call above
+          echo "Failed to write generated API Key to AWS Secrets Manager!"
+          echo "The API Key value is now lost and cannot be retrieved, manual intervention is required before rerunning the pipeline:"
+          echo "  - Log in to the SendGrid parent account"
+          echo "  - Switch to the subuser's account (which will start with ${ICN}, the full value will be in the logs above)"
+          echo "  - Locate and delete the API Key for this MAS instance - its name will be in the logs above"
+          echo "  - A new API Key for the MAS instance will be generated by this Task in the next pipeline run"
+          rc="${update_apikey_secret_rc}"
+        fi
+
+
+
+        if [[ "${update_subuser_secret_rc}" != "0" ]]; then
+          # We could add additional automation to deal with the (rare) edge case of the SM update failing.
+          # but I think - given the rarity - just documenting the manual resolution steps below will suffice.
+          echo "Failed to write generated subuser username and password to AWS Secrets Manager!"
+          echo "The generated password value is now lost and cannot be retrieved, manual intervention is required before rerunning the pipeline:"
+          echo "  - Log in to the SendGrid parent account"
+          echo "  - Go to Settings -> Subuser management"
+          echo "  - Locate and delete the subuser generated by the script above. The username will start with ${ICN}, the full value will be in the logs above."
+          echo "  - A new subuser will be created for the customer in the next pipeline run"
+          rc="${update_subuser_secret_rc}"
+        fi
+
+
+        if [[ "${rc}" != "0" ]]; then
+          echo "mas-saas-sendgrid-subuser failed with rc ${rc}"
+          exit ${rc}
+        fi
+
+        export SMTP_DISPLAY_NAME="SendGrid"
+        export SMTP_HOST="smtp.sendgrid.net"
+        export SMTP_PORT="465"
+        export SMTP_SECURITY="SSL"
+        export SMTP_AUTHENTICATION="true"
+        export SMTP_DEFAULT_SENDER_EMAIL="mas.admin@${CIS_MAS_DOMAIN:-${MAS_DOMAIN}}" # use dns.cis.mas_domain if set, otherwise fallback to mas_instance.mas_domain
+        export SMTP_DEFAULT_SENDER_NAME="MASAdmin"
+        export SMTP_DEFAULT_RECIPIENT_EMAIL="mas.ms.smtp@ibm.com"
+        export SMTP_DEFAULT_SHOULD_EMAIL_PASSWORDS="true"
+
+        # Ensure any SMTP_USERNAME/SMTP_PASSWORD values set in pipeline context properties
+        # are not used by the gitops-mas-config script to override the values in the AWS SM Secret
+        unset SMTP_USERNAME
+        unset SMTP_PASSWORD
+
+        mas gitops-mas-config \
+        --account-id "$ACCOUNT_ID" \
+        --cluster-id "$CLUSTER_ID" \
+        --mas-instance-id "$MAS_INSTANCE_ID" \
+        --secrets-path "$SECRETS_PATH" \
+        --github-push \
+        --github-host "$GITHUB_HOST" \
+        --github-org "$GITHUB_ORG" \
+        --github-repo "$GITHUB_REPO" \
+        --git-branch "$GIT_BRANCH" \
+        --config-action upsert \
+        --mas-config-scope system \
+        --mas-config-type smtp \
+        --dir /tmp/init-suite-smtp-config \
+        --smtp-display-name "${SMTP_DISPLAY_NAME}" \
+        --smtp-host "${SMTP_HOST}" \
+        --smtp-port "${SMTP_PORT}" \
+        --smtp-security "${SMTP_SECURITY}" \
+        --smtp-authentication "${SMTP_AUTHENTICATION}" \
+        --smtp-default-sender-email "${SMTP_DEFAULT_SENDER_EMAIL}" \
+        --smtp-default-sender-name "${SMTP_DEFAULT_SENDER_NAME}" \
+        --smtp-default-recipient-email "${SMTP_DEFAULT_RECIPIENT_EMAIL}" \
+        --smtp-default-should-email-passwords "${SMTP_DEFAULT_SHOULD_EMAIL_PASSWORDS}" \
+        --mas-smtpcfg-pod-template-yaml "${MAS_SMTPCFG_POD_TEMPLATE_YAML}"
+
+        exit $?
+        
+      command:
+        - /bin/sh
+        - -c
+      name: gitops-suite-smtp-config-sendgrid
+      imagePullPolicy: IfNotPresent
+      image: docker-na-public.artifactory.swg-devops.com/wiotp-docker-local/mas/saas-task:latest
+  workspaces:
+    - name: configs
+
+
+
+
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/gitops-suite-smtp-config.yaml
 # --------------------------------------------------------------------------------
@@ -7821,7 +8281,7 @@ spec:
         - -c
       name: gitops-suite-smtp-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 
@@ -7930,7 +8390,7 @@ spec:
         - -c
       name: gitops-suite-watson-studio-config
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -8027,7 +8487,7 @@ spec:
         - -c
       name: gitops-suite-workspace
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
 # --------------------------------------------------------------------------------
@@ -8307,7 +8767,7 @@ spec:
         - -c
       name: gitops-suite
       imagePullPolicy: IfNotPresent
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
   workspaces:
     - name: configs
     - name: shared-gitops-configs
@@ -8357,7 +8817,7 @@ spec:
 
   steps:
     - name: grafana
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -8491,7 +8951,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - ibm_catalogs
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 # --------------------------------------------------------------------------------
@@ -8733,7 +9193,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - kafka
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -8949,7 +9409,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - kmodels
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/mongodb.yaml
@@ -9127,7 +9587,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - mongodb
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -9170,7 +9630,7 @@ spec:
         - $(params.base_output_dir)
         - --extra-namespaces
         - selenium
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       env:
         - name: DEVOPS_MONGO_URI
@@ -9287,7 +9747,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - nvidia_gpu
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -9323,7 +9783,7 @@ spec:
     # Verify Cluster
     # -------------------------------------------------------------------------
     - name: ocp-verify-cluster
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -9365,7 +9825,7 @@ spec:
     # Verify Catalogs
     # -------------------------------------------------------------------------
     - name: ocp-verify-catalogs
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -9407,7 +9867,7 @@ spec:
     # Verify Subscriptions
     # -------------------------------------------------------------------------
     - name: ocp-verify-subscriptions
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -9449,7 +9909,7 @@ spec:
     # Verify Workloads
     # -------------------------------------------------------------------------
     - name: ocp-verify-workloads
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -9491,7 +9951,7 @@ spec:
     # Verify Catalogs - Ingress TLS
     # -------------------------------------------------------------------------
     - name: ocp-verify-ingress
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/run-role.sh
@@ -9617,7 +10077,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - ocp_verify
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 # --------------------------------------------------------------------------------
@@ -9685,7 +10145,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - ocs
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -9921,7 +10381,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - odh
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/sls-registry-update.yaml
@@ -10097,7 +10557,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - sls
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -10486,12 +10946,12 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_config
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 
-    # If configmap/approval-app-cfg-$(params.mas_app_id) exists then set CONFIGMAP_KEY=pending and wait for it to be changed to "approved"
+    # If configmap/approval-app-cfg-$(params.mas_app_id) exists then set STATUS=pending and wait for it to be changed to "approved"
     - name: app-cfg-post-verify
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       command:
         - /opt/app-root/src/wait-for-configmap.sh
@@ -10500,12 +10960,6 @@ spec:
           value: $(context.taskRun.namespace)
         - name: CONFIGMAP_NAME
           value: approval-app-cfg-$(params.mas_app_id)
-        - name: CONFIGMAP_KEY
-          valueFrom:
-            configMapKeyRef:
-              name: approval-app-cfg-$(params.mas_app_id)
-              key: CONFIGMAP_KEY
-              optional: true
         - name: CONFIGMAP_INITIAL_VALUE
           value: pending
         - name: CONFIGMAP_TARGET_VALUE
@@ -10692,7 +11146,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_install
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 
   workspaces:
@@ -10781,7 +11235,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_rollback
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-app-uninstall.yaml
@@ -10844,7 +11298,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_uninstall
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-app-upgrade.yaml
@@ -10919,7 +11373,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_upgrade
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-app-verify.yaml
@@ -11020,7 +11474,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_app_verify
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-certs.yaml
@@ -11163,7 +11617,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_certs
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 
   workspaces:
@@ -11230,7 +11684,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_config
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -11301,7 +11755,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_db2_setup_for_manage
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-dns.yaml
@@ -11491,7 +11945,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_dns
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-install.yaml
@@ -11740,7 +12194,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_install
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
 
@@ -11828,7 +12282,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_rollback
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-uninstall.yaml
@@ -11893,7 +12347,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_uninstall
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-upgrade.yaml
@@ -11963,7 +12417,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_upgrade
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/suite-verify.yaml
@@ -12030,12 +12484,12 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - suite_verify
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 
-    # If configmap/approval-suite-verify exists then set CONFIGMAP_KEY=pending and wait for it to be changed to "approved"
+    # If configmap/approval-suite-verify exists then set STATUS=pending and wait for it to be changed to "approved"
     - name: suite-post-verify
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       script: |
         #!/usr/bin/env bash
@@ -12047,12 +12501,6 @@ spec:
           value: $(context.taskRun.namespace)
         - name: CONFIGMAP_NAME
           value: approval-suite-verify
-        - name: CONFIGMAP_KEY
-          valueFrom:
-            configMapKeyRef:
-              name: approval-suite-verify
-              key: CONFIGMAP_KEY
-              optional: true
         - name: CONFIGMAP_INITIAL_VALUE
           value: pending
         - name: CONFIGMAP_TARGET_VALUE
@@ -12169,7 +12617,7 @@ spec:
       command:
         - /opt/app-root/src/run-role.sh
         - turbonomic
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/uds.yaml
@@ -12250,7 +12698,7 @@ spec:
     # IBM User Data Services (UDS)
     # -------------------------------------------------------------------------
     - name: uds
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
       command:
@@ -12313,7 +12761,7 @@ spec:
     # IBM Data Reporter Operator (DRO)
     # -------------------------------------------------------------------------
     - name: dro
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       imagePullPolicy: $(params.image_pull_policy)
       workingDir: /workspace/configs
       command:
@@ -12387,31 +12835,104 @@ metadata:
 spec:
   params:
     # What configmap to update
+    - name: configmap_namespace
+      type: string
+      description: "The namespace of the configmap to update"
+      default: $(context.taskRun.namespace)
     - name: configmap_name
       type: string
       description: "The name of the configmap to update"
-    - name: configmap_key
-      type: string
-      description: "The key in the configmap to update"
     - name: configmap_value
       type: string
       description: "The value to set"
+  steps:
+    - name: update-configmap
+      image: quay.io/ibmmas/cli:13.0.0
+      command:
+        - /opt/app-root/src/update-configmap.sh
+      env:
+        # What to wait for
+        - name: NAMESPACE
+          value: $(params.configmap_namespace)
+        - name: CONFIGMAP_NAME
+          value: $(params.configmap_name)
+        - name: CONFIGMAP_VALUE
+          value: $(params.configmap_value)
+# --------------------------------------------------------------------------------
+# /home/runner/work/cli/cli/tekton/target/tasks/wait-for-configmap-v2.yaml
+# --------------------------------------------------------------------------------
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: mas-devops-wait-for-configmap-v2
+spec:
+  params:
+    - name: configmap_namespace
+      type: string
+      description: "The namespace of the configmap to wait for"
+      default: $(context.taskRun.namespace)
+    - name: configmap_name
+      type: string
+      description: "The name of the configmap to wait for"
+    - name: configmap_initial_value
+      type: string
+      description: "Optional value to initialise the configmap with if it does not already exist"
+      default: ""
+    - name: configmap_target_value
+      type: string
+      description: "The value to wait for the configmap to change to (defaults to 'approved')"
+      default: "approved"
+
+    # Optional escape route
+    - name: escape_configmap_name
+      type: string
+      description: "The name of the configmap wait for"
+      default: ""
+      description: "The key in the configmap to watch for"
+      default: ""
+
+  steps:
+    - name: wait-for-configmap
+      image: quay.io/ibmmas/cli:13.0.0
+      command:
+        - /opt/app-root/src/wait-for-configmap.sh
+      env:
+        # What to wait for
+        - name: NAMESPACE
+          value: $(params.configmap_namespace)
+        - name: CONFIGMAP_NAME
+          value: $(params.configmap_name)
+        - name: CONFIGMAP_INITIAL_VALUE
+          value: $(params.configmap_initial_value)
+        - name: CONFIGMAP_TARGET_VALUE
+          value: $(params.configmap_target_value)
+
+        # How long to wait
+        - name: DELAY
+          valueFrom:
+            configMapKeyRef:
+              name: $(params.configmap_name)
+              key: DELAY
+              optional: true
+        - name: MAX_RETRIES
+          valueFrom:
+            configMapKeyRef:
+              name: $(params.configmap_name)
+              key: MAX_RETRIES
+              optional: true
 
-  steps:
-    - name: update-configmap
-      image: quay.io/ibmmas/cli:12.0.0
-      command:
-        - /opt/app-root/src/update-configmap.sh
-      env:
-        # What to wait for
-        - name: NAMESPACE
-          value: $(context.taskRun.namespace)
-        - name: CONFIGMAP_NAME
-          value: $(params.configmap_name)
-        - name: CONFIGMAP_KEY
-          value: $(params.configmap_key)
-        - name: CONFIGMAP_VALUE
-          value: $(params.configmap_value)
+        # Optional escape route
+        - name: ESCAPE_CONFIGMAP_NAME
+          value: $(params.escape_configmap_name)
+
+        # How to handle errors
+        - name: IGNORE_FAILURE
+          valueFrom:
+            configMapKeyRef:
+              name: $(params.configmap_name)
+              key: IGNORE_FAILURE
+              optional: true
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/tasks/wait-for-configmap.yaml
 # --------------------------------------------------------------------------------
@@ -12426,9 +12947,6 @@ spec:
     - name: configmap_name
       type: string
       description: "The name of the configmap wait for"
-    - name: configmap_key
-      type: string
-      description: "The key in the configmap to watch for"
     - name: configmap_target_value
       type: string
       description: "The value to wait for the configmap to change to"
@@ -12438,10 +12956,6 @@ spec:
       type: string
       description: "The name of the configmap wait for"
       default: ""
-    - name: escape_configmap_key
-      type: string
-      description: "The key in the configmap to watch for"
-      default: ""
 
     # How long to wait
     - name: delay
@@ -12461,7 +12975,7 @@ spec:
 
   steps:
     - name: wait-for-configmap
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       command:
         - /opt/app-root/src/wait-for-configmap.sh
       env:
@@ -12470,16 +12984,12 @@ spec:
           value: $(context.taskRun.namespace)
         - name: CONFIGMAP_NAME
           value: $(params.configmap_name)
-        - name: CONFIGMAP_KEY
-          value: $(params.configmap_key)
         - name: CONFIGMAP_TARGET_VALUE
           value: $(params.configmap_target_value)
 
         # Optional escape route
         - name: ESCAPE_CONFIGMAP_NAME
           value: $(params.escape_configmap_name)
-        - name: ESCAPE_CONFIGMAP_KEY
-          value: $(params.escape_configmap_key)
 
         # How long to wait
         - name: DELAY
@@ -12525,7 +13035,7 @@ spec:
 
   steps:
     - name: wait
-      image: quay.io/ibmmas/cli:12.0.0
+      image: quay.io/ibmmas/cli:13.0.0
       command:
         - /opt/app-root/src/wait-for-tekton.sh
       env:
@@ -13464,8 +13974,18 @@ spec:
     - name: jdbc_route_manage
       type: string
       default: ""
+
+    - name: sls_license_icn
+      type: string
+    - name: cis_crn
+      type: string
+    - name: cis_mas_domain
+      type: string
+    - name: smtp_use_sendgrid
+      type: string
+      default: ""
   tasks:
-    # 1. Deprovision workspace
+    # Deprovision workspace
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-workspace
       params:
@@ -13506,7 +14026,45 @@ spec:
         - name: configs
           workspace: configs
 
-    # 2. Deprovision SMTP config
+    # Deprovision SendGrid subuser
+    # -------------------------------------------------------------------------
+    - name: gitops-deprovision-suite-sendgrid-subuser
+      params:
+        - name: cluster_name
+          value: $(params.cluster_name)
+        - name: account
+          value: $(params.account)
+        - name: mas_instance_id
+          value: $(params.mas_instance_id)
+        - name: icn
+          value: $(params.sls_license_icn)
+        - name: cis_mas_domain
+          value: $(params.cis_mas_domain)
+        - name: cis_crn
+          value: $(params.cis_crn)
+        - name: avp_aws_secret_region
+          value: $(params.avp_aws_secret_region)
+      taskRef:
+        kind: Task
+        name: gitops-deprovision-suite-sendgrid-subuser
+      when:
+        # Task uses an IBM internal image and cannot be used outside of the internal IBM Toolchains
+        # To make sure this task is only run when this is the case, reference a field that 
+        # but corresponds to another IBM-internal only feature that will also still be present after smtp config is removed
+        # sls.license.icn is a good choice for this, it also happens to be a required input to the sendgrid scripts
+        - input: "$(params.sls_license_icn)"
+          operator: notin
+          values: [""]
+
+        # only attempt Subuser teardown if the instance was configured to use automated sendgrid subuser management at the time of deprovisioning
+        - input: "$(params.smtp_use_sendgrid)"
+          operator: in
+          values: ["true"]
+      workspaces:
+        - name: configs
+          workspace: configs
+
+    # Deprovision SMTP config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-smtp-config
       runAfter:
@@ -13545,7 +14103,7 @@ spec:
         - name: configs
           workspace: configs
 
-    # 3. Deprovision IDP config
+    # Deprovision IDP config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-idp-config
       runAfter:
@@ -13584,7 +14142,7 @@ spec:
         - name: configs
           workspace: configs
 
-    # 4. Deprovision Suite config
+    # Deprovision Suite config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-config
       runAfter:
@@ -13634,7 +14192,7 @@ spec:
         name: gitops-deprovision-suite-config
         kind: Task
 
-    # 5. Deprovision Suite config
+    # Deprovision Suite config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite
       runAfter:
@@ -13675,7 +14233,7 @@ spec:
         - name: configs
           workspace: configs
 
-    # 8. Deprovision Kafka config
+    # Deprovision Kafka config
     # -------------------------------------------------------------------------
     - name: gitops-delete-kafka-config
       runAfter:
@@ -13716,7 +14274,7 @@ spec:
         kind: Task
         name: gitops-delete-kafka-config
 
-    # 9. Deprovision Objectstorage config
+    # Deprovision Objectstorage config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-objectstorage-config
       runAfter:
@@ -13755,7 +14313,7 @@ spec:
         - name: configs
           workspace: configs
 
-    # 10. Deprovision Watson Studio config
+    # Deprovision Watson Studio config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-watson-studio-config
       runAfter:
@@ -13794,7 +14352,7 @@ spec:
         - name: configs
           workspace: configs
 
-    # 11. Deprovision DB2U Operator
+    # Deprovision DB2U Operator
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-db2u
       runAfter:
@@ -17634,6 +18192,7 @@ spec:
     - name: ingress
       type: string
       default: "false"
+
     - name: group_sync_operator_cron_schedule
       type: string
       default: ''
@@ -17643,9 +18202,17 @@ spec:
     - name: group_sync_operator_isv_groups
       type: string
       default: ''
+
     - name: ibm_rbac_binding_to_group
       type: string
       default: ''
+    
+    - name: falcon_operator_cloud_region
+      type: string
+      default: ''
+    - name: falcon_operator_node_sensor
+      type: string
+      default: ''
   tasks:
     - name: gitops-cluster
       params:
@@ -17718,14 +18285,21 @@ spec:
           value: $(params.dns_provider)
         - name: ingress
           value: $(params.ingress)
+
         - name: group_sync_operator_cron_schedule
           value: $(params.group_sync_operator_cron_schedule)
         - name: group_sync_operator_isv_tenant_url
           value: $(params.group_sync_operator_isv_tenant_url)
         - name: group_sync_operator_isv_groups
           value: $(params.group_sync_operator_isv_groups)
+
         - name: ibm_rbac_binding_to_group
           value: $(params.ibm_rbac_binding_to_group)
+
+        - name: falcon_operator_cloud_region
+          value: $(params.falcon_operator_cloud_region)
+        - name: falcon_operator_node_sensor
+          value: $(params.falcon_operator_node_sensor)
       taskRef:
         kind: Task
         name: gitops-cluster
@@ -18569,6 +19143,12 @@ spec:
       type: string
       default: ""
 
+    - name: sls_license_icn
+      type: string
+    - name: smtp_use_sendgrid
+      type: string
+      default: ""
+
   tasks:
     - name: mas-launchfvt
       params:
@@ -18694,6 +19274,13 @@ spec:
         - name: ldap_userid_map
           value: $(params.ldap_userid_map)
 
+        - name: icn
+          value: $(params.sls_license_icn)
+        - name: avp_aws_secret_region
+          value: $(params.avp_aws_secret_region)
+        - name: use_sendgrid
+          value: $(params.smtp_use_sendgrid)
+
       workspaces:
         - name: configs
           workspace: configs
@@ -19211,6 +19798,9 @@ spec:
       type: string
     - name: smtp_default_should_email_passwords
       type: string
+    - name: smtp_use_sendgrid
+      type: string
+
     - name: ldap_url
       type: string
     - name: ldap_basedn
@@ -19379,7 +19969,7 @@ spec:
         kind: Task
         name: gitops-license
       when:
-        - input: "$(params.sls_license_app_points)"
+        - input: "$(params.sls_license_icn)"
           operator: in
           values: [""]
 
@@ -19408,7 +19998,7 @@ spec:
         kind: Task
         name: gitops-license-generator
       when:
-        - input: "$(params.sls_license_app_points)"
+        - input: "$(params.sls_license_icn)"
           operator: notin
           values: [""]
 
@@ -19670,6 +20260,56 @@ spec:
         - input: "$(params.smtp_host)"
           operator: notin
           values: [""]
+        - input: "$(params.smtp_use_sendgrid)"
+          operator: notin
+          values: ["true"]
+      workspaces:
+        - name: configs
+          workspace: configs
+
+    - name: gitops-suite-smtp-config-sendgrid
+      runAfter:
+        - gitops-suite-config
+      params:
+        - name: cluster_name
+          value: $(params.cluster_name)
+        - name: account
+          value: $(params.account)
+        - name: secrets_path
+          value: $(params.secrets_path)
+        - name: mas_instance_id
+          value: $(params.mas_instance_id)
+        - name: git_branch
+          value: $(params.git_branch)
+        - name: github_org
+          value: $(params.github_org)
+        - name: github_repo
+          value: $(params.github_repo)
+        - name: github_host
+          value: $(params.github_host)
+        - name: avp_aws_secret_region
+          value: $(params.avp_aws_secret_region)
+        - name: mas_smtpcfg_pod_template_yaml
+          value: $(params.mas_smtpcfg_pod_template_yaml)
+        
+        - name: icn
+          value: $(params.sls_license_icn)
+        - name: mas_domain
+          value: $(params.mas_domain)
+        - name: cis_mas_domain
+          value: $(params.cis_mas_domain)
+        - name: cis_crn
+          value: $(params.cis_crn)
+      taskRef:
+        kind: Task
+        name: gitops-suite-smtp-config-sendgrid
+      when:
+        - input: "$(params.smtp_host)"
+          operator: in
+          values: [""]
+        - input: "$(params.smtp_use_sendgrid)"
+          operator: in
+          values: ["true"]
       workspaces:
         - name: configs
           workspace: configs
@@ -19730,6 +20370,45 @@ spec:
     # Deprovision task section
     # -------------------------------------------------------------------------
 
+    # 1. Deprovision SendGrid subuser
+    # -------------------------------------------------------------------------
+    - name: gitops-deprovision-suite-sendgrid-subuser
+      params:
+        - name: cluster_name
+          value: $(params.cluster_name)
+        - name: account
+          value: $(params.account)
+        - name: mas_instance_id
+          value: $(params.mas_instance_id)
+        - name: icn
+          value: $(params.sls_license_icn)
+        - name: cis_mas_domain
+          value: $(params.cis_mas_domain)
+        - name: cis_crn
+          value: $(params.cis_crn)
+        - name: avp_aws_secret_region
+          value: $(params.avp_aws_secret_region)
+      taskRef:
+        kind: Task
+        name: gitops-deprovision-suite-sendgrid-subuser
+      when:
+        # Always run this task when smtp.use_sendgrid is unset (and we're running in an internal IBM toolchain - see below)
+        # Task will still complete successfully even if it is a no-op
+        - input: "$(params.smtp_use_sendgrid)"
+          operator: notin
+          values: ["true"]
+
+        # Task uses an IBM internal image and cannot be used outside of the internal IBM Toolchains
+        # To make sure this task is only run when this is the case, reference a field that 
+        # but corresponds to another IBM-internal only feature that will also still be present after smtp config is removed
+        # sls.license.icn is a good choice for this, it also happens to be a required input to the sendgrid scripts
+        - input: "$(params.sls_license_icn)"
+          operator: notin
+          values: [""]
+      workspaces:
+        - name: configs
+          workspace: configs
+
     # 2. Deprovision SMTP config
     # -------------------------------------------------------------------------
     - name: gitops-deprovision-suite-smtp-config
@@ -19767,6 +20446,9 @@ spec:
         - input: "$(params.smtp_host)"
           operator: in
           values: [""]
+        - input: "$(params.smtp_use_sendgrid)"
+          operator: notin
+          values: ["true"]
       workspaces:
         - name: configs
           workspace: configs
@@ -19811,6 +20493,7 @@ spec:
       workspaces:
         - name: configs
           workspace: configs
+
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/pipelines/install.yaml
 # --------------------------------------------------------------------------------
@@ -22753,6 +23436,7 @@ spec:
         - suite-db2-setup-system
         - suite-db2-setup-manage
         - suite-config-watson-studio
+        - suite-config-cos
 
     # 8.3 Configure Manage workspace
     - name: app-cfg-manage
@@ -23521,8 +24205,6 @@ spec:
           value: $(params.image_pull_policy)
         - name: configmap_name
           value: sync-install
-        - name: configmap_key
-          value: INSTALL_STATUS
         - name: configmap_value
           # An aggregate status of all the pipelineTasks under the tasks section (excluding the finally section).
           # This variable is only available in the finally tasks and can have any one of the values (Succeeded, Failed, Completed, or None)
@@ -24167,13 +24849,28 @@ spec:
     # 8. Uninstall IBM Catalogs
     # 9. Uninstall Cluster Monitoring
 
-    # 1. Uninstall IBM Maximo Application Suite Applications
+
+    # 0. Wait for approval before we change anything
     # -------------------------------------------------------------------------
-    # 1.0 Start uninstall
+    - name: waitfor-approval
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-wait-for-configmap-v2
+      params:
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: configmap_name
+          value: approval-uninstall
+
 
+    # 1. Uninstall IBM Maximo Application Suite Applications
+    # -------------------------------------------------------------------------
     # 1.1 Uninstall Visual Inspection
     - name: app-uninstall-visualinspection
       timeout: "0"
+      runAfter:
+        - waitfor-approval
       taskRef:
         kind: Task
         name: mas-devops-suite-app-uninstall
@@ -24188,6 +24885,8 @@ spec:
     # 1.2 Uninstall Assist
     - name: app-uninstall-assist
       timeout: "0"
+      runAfter:
+        - waitfor-approval
       taskRef:
         kind: Task
         name: mas-devops-suite-app-uninstall
@@ -24202,6 +24901,8 @@ spec:
     # 1.3 Uninstall Optimizer
     - name: app-uninstall-optimizer
       timeout: "0"
+      runAfter:
+        - waitfor-approval
       taskRef:
         kind: Task
         name: mas-devops-suite-app-uninstall
@@ -24216,6 +24917,8 @@ spec:
     # 1.4 Uninstall Predict
     - name: app-uninstall-predict
       timeout: "0"
+      runAfter:
+        - waitfor-approval
       taskRef:
         kind: Task
         name: mas-devops-suite-app-uninstall
@@ -24275,6 +24978,7 @@ spec:
         - name: devops_suite_name
           value: app-iot-uninstall
 
+
     # 2. Uninstall IBM Maximo Application Suite
     # -------------------------------------------------------------------------
     - name: uninstall-suite
@@ -24298,6 +25002,7 @@ spec:
         - name: devops_suite_name
           value: suite-uninstall
 
+
     # 3. Uninstall IBM Suite Licensing Service
     # -------------------------------------------------------------------------
     - name: uninstall-sls
@@ -24319,6 +25024,7 @@ spec:
       runAfter:
         - uninstall-suite
 
+
     # 4. Uninstall MongoDB
     # -------------------------------------------------------------------------
     - name: uninstall-mongodb
@@ -24340,6 +25046,7 @@ spec:
       runAfter:
         - uninstall-sls
 
+
     # 5. Uninstall IBM User Data Services (or DRO)
     # -------------------------------------------------------------------------
     - name: uninstall-uds
@@ -24363,6 +25070,7 @@ spec:
       runAfter:
         - uninstall-mongodb
 
+
     # 6. Uninstall IBM Cert Manager
     # -------------------------------------------------------------------------
     - name: uninstall-cert-manager
@@ -24386,6 +25094,7 @@ spec:
       runAfter:
         - uninstall-uds
 
+
     # 7. Uninstall IBM Common Services
     # -------------------------------------------------------------------------
     - name: uninstall-common-services
@@ -24407,6 +25116,7 @@ spec:
       runAfter:
         - uninstall-cert-manager
 
+
     # 8. Uninstall IBM Catalogs
     # -------------------------------------------------------------------------
     - name: uninstall-ibm-catalogs
@@ -24428,6 +25138,7 @@ spec:
       runAfter:
         - uninstall-common-services
 
+
     # 9. Uninstall Grafana
     # -------------------------------------------------------------------------
     - name: uninstall-grafana
@@ -24462,8 +25173,6 @@ spec:
           value: $(params.image_pull_policy)
         - name: configmap_name
           value: sync-uninstall
-        - name: configmap_key
-          value: UNINSTALL_STATUS
         - name: configmap_value
           # An aggregate status of all the pipelineTasks under the tasks section (excluding the finally section).
           # This variable is only available in the finally tasks and can have any one of the values (Succeeded, Failed, Completed, or None)
@@ -24477,9 +25186,7 @@ kind: Pipeline
 metadata:
   name: mas-update
 spec:
-
   params:
-
     # Tekton Pipeline Configuration
     # -------------------------------------------------------------------------
     - name: image_pull_policy
@@ -24661,9 +25368,19 @@ spec:
       description: ReadWriteMany storage class
 
   tasks:
-
-    # 1. Verify health of the cluster before we change anything
+    # 1. Wait for approval & verify health of the cluster before we change anything
     # -------------------------------------------------------------------------
+    - name: waitfor-approval
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-wait-for-configmap-v2
+      params:
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: configmap_name
+          value: approval-update
+
     - name: pre-update-check
       timeout: "0"
       taskRef:
@@ -24684,9 +25401,11 @@ spec:
           operator: notin
           values: ["True", "true"]
 
+      runAfter:
+        - waitfor-approval
 
 
-    # 3. Run the catalog update
+    # 2. Run the catalog update
     # -------------------------------------------------------------------------
     - name: update-catalog
       timeout: "0"
@@ -24698,7 +25417,6 @@ spec:
       params:
         - name: devops_suite_name
           value: update-catalog
-
         - name: mas_catalog_version
           value: $(params.mas_catalog_version)
 
@@ -24708,7 +25426,8 @@ spec:
         - name: artifactory_token
           value: $(params.artifactory_token)
 
-    # 4. Verify health of the cluster before we consider the update complete
+
+    # 3. Verify health of the cluster before we consider the update complete
     # -------------------------------------------------------------------------
     - name: post-update-verify
       timeout: "0"
@@ -24733,7 +25452,8 @@ spec:
       runAfter:
         - update-catalog
 
-    # 5. Update Dependencies
+
+    # 4. Update Dependencies
     # ---------------------------------------------------------------------------
     - name: update-ocs
       timeout: "0"
@@ -24886,9 +25606,10 @@ spec:
         - name: devops_suite_name
           value: update-uds
 
-    # 6 Update Cloud Pak for Data
+
+    # 5 Update Cloud Pak for Data
     # -------------------------------------------------------------------------
-    # 6.1 Cloud Pak for Data Platform
+    # 5.1 Cloud Pak for Data Platform
     - name: update-cp4d
       timeout: "0"
       params:
@@ -24931,7 +25652,7 @@ spec:
       runAfter:
         - update-cert-manager # CP4D requires cert-manager
         
-    # 6.2 Watson Studio
+    # 5.2 Watson Studio
     - name: update-watson-studio
       timeout: "0"
       params:
@@ -24961,7 +25682,7 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 6.3 Watson Machine Learning
+    # 5.3 Watson Machine Learning
     - name: update-watson-machine-learning
       timeout: "0"
       params:
@@ -24991,7 +25712,7 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 6.4 Analytics Service (Spark)
+    # 5.4 Analytics Service (Spark)
     - name: update-analytics-engine
       timeout: "0"
       params:
@@ -25021,7 +25742,7 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 6.5 Watson OpenScale
+    # 5.5 Watson OpenScale
     - name: update-aiopenscale
       timeout: "0"
       params:
@@ -25051,7 +25772,7 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 6.6 SPSS Statistics
+    # 5.6 SPSS Statistics
     - name: update-spss
       timeout: "0"
       params:
@@ -25081,7 +25802,7 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 6.7 Cognos Analytics
+    # 5.7 Cognos Analytics
     - name: update-cognos
       timeout: "0"
       params:
@@ -25111,7 +25832,8 @@ spec:
       runAfter:
         - update-cp4d
 
-    # 7. Verify health of the cluster after dependencies updates
+
+    # 6. Verify health of the cluster after dependencies updates
     # -------------------------------------------------------------------------
     - name: post-deps-update-verify
       timeout: "0"
@@ -25146,7 +25868,23 @@ spec:
         - update-spss
         - update-cognos
 
-
+  finally:
+    # Update synchronization configmap
+    # -------------------------------------------------------------------------
+    - name: sync-update
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-update-configmap
+      params:
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: configmap_name
+          value: sync-update
+        - name: configmap_value
+          # An aggregate status of all the pipelineTasks under the tasks section (excluding the finally section).
+          # This variable is only available in the finally tasks and can have any one of the values (Succeeded, Failed, Completed, or None)
+          value: $(tasks.status)
 # --------------------------------------------------------------------------------
 # /home/runner/work/cli/cli/tekton/target/pipelines/upgrade.yaml
 # --------------------------------------------------------------------------------
@@ -25156,7 +25894,6 @@ kind: Pipeline
 metadata:
   name: mas-upgrade
 spec:
-
   params:
     # 1. Common Parameters
     # -------------------------------------------------------------------------
@@ -25204,7 +25941,6 @@ spec:
       type: string
       default: ""
 
-
     # MAS Configuration - Application Channels
     # -------------------------------------------------------------------------
     - name: mas_app_channel_iot
@@ -25242,9 +25978,19 @@ spec:
 
 
   tasks:
-
-    # 1. Verify health of the cluster before we change anything
+    # 1. Wait for approval & verify health of the cluster before we change anything
     # -------------------------------------------------------------------------
+    - name: waitfor-approval
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-wait-for-configmap-v2
+      params:
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: configmap_name
+          value: approval-upgrade
+
     - name: pre-upgrade-check
       timeout: "0"
       taskRef:
@@ -25265,9 +26011,11 @@ spec:
           operator: notin
           values: ["True", "true"]
 
+      runAfter:
+        - waitfor-approval
 
 
-    # 3. Suite Upgrade (Phase 1)
+    # 2. Suite Upgrade (Phase 1)
     # -------------------------------------------------------------------------
     - name: core-upgrade
       timeout: "0"
@@ -25304,9 +26052,9 @@ spec:
         - core-upgrade
 
 
-    # 4. IoT Upgrade (Phase 2)
+    # 3. IoT Upgrade (Phase 2)
     # -------------------------------------------------------------------------
-    - name: iot-upgrade
+    - name: app-iot-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25318,7 +26066,7 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: iot-upgrade
+          value: app-iot-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
@@ -25326,9 +26074,9 @@ spec:
         - core-verify
 
 
-    # 5. Manage Upgrade (Phase 2)
+    # 4. Manage Upgrade (Phase 2)
     # -------------------------------------------------------------------------
-    - name: manage-upgrade
+    - name: app-manage-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25340,7 +26088,7 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: manage-upgrade
+          value: app-manage-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
@@ -25348,9 +26096,9 @@ spec:
         - core-verify
 
 
-    # 6. Visual Inspection Upgrade (Phase 2)
+    # 5. Visual Inspection Upgrade (Phase 2)
     # -------------------------------------------------------------------------
-    - name: visualinspection-upgrade
+    - name: app-visualinspection-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25362,7 +26110,7 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: visualinspection-upgrade
+          value: app-visualinspection-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
@@ -25370,9 +26118,9 @@ spec:
         - core-verify
 
 
-    # 7. Assist Upgrade (Phase 2)
+    # 6. Assist Upgrade (Phase 2)
     # -------------------------------------------------------------------------
-    - name: assist-upgrade
+    - name: app-assist-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25384,7 +26132,7 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: assist-upgrade
+          value: app-assist-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
@@ -25392,9 +26140,9 @@ spec:
         - core-verify
 
 
-    # 8. Optimizer Upgrade (Phase 2)
+    # 7. Optimizer Upgrade (Phase 2)
     # -------------------------------------------------------------------------
-    - name: optimizer-upgrade
+    - name: app-optimizer-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25406,7 +26154,7 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: optimizer-upgrade
+          value: app-optimizer-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
@@ -25414,9 +26162,9 @@ spec:
         - core-verify
 
 
-    # 9. Monitor Upgrade (Phase 3 - after IoT)
+    # 8. Monitor Upgrade (Phase 3 - after IoT)
     # -------------------------------------------------------------------------
-    - name: monitor-upgrade
+    - name: app-monitor-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25428,17 +26176,17 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: monitor-upgrade
+          value: app-monitor-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
       runAfter:
-        - iot-upgrade
+        - app-iot-upgrade
 
 
-    # 10. Predict Upgrade (Phase 3 - after Manage)
+    # 9. Predict Upgrade (Phase 3 - after Manage)
     # -------------------------------------------------------------------------
-    - name: predict-upgrade
+    - name: app-predict-upgrade
       timeout: "0"
       params:
         - name: mas_instance_id
@@ -25450,14 +26198,15 @@ spec:
         - name: skip_compatibility_check
           value: $(params.skip_compatibility_check)
         - name: devops_suite_name
-          value: predict-upgrade
+          value: app-predict-upgrade
       taskRef:
         kind: Task
         name: mas-devops-suite-app-upgrade
       runAfter:
-        - manage-upgrade
+        - app-manage-upgrade
+
 
-    # 11. Verify health of the cluster after upgrade
+    # 10. Verify health of the cluster after upgrade
     # -------------------------------------------------------------------------
     - name: post-upgrade-verify
       timeout: "0"
@@ -25481,11 +26230,27 @@ spec:
 
       runAfter:
         # Phase 2 apps that don't have a phase 3 app following it
-        - assist-upgrade
-        - optimizer-upgrade
-        - visualinspection-upgrade
+        - app-assist-upgrade
+        - app-optimizer-upgrade
+        - app-visualinspection-upgrade
         # Phase 3 apps
-        - predict-upgrade
-        - monitor-upgrade
-
+        - app-predict-upgrade
+        - app-monitor-upgrade
 
+  finally:
+    # Update synchronization configmap
+    # -------------------------------------------------------------------------
+    - name: sync-upgrade
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-update-configmap
+      params:
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: configmap_name
+          value: sync-upgrade
+        - name: configmap_value
+          # An aggregate status of all the pipelineTasks under the tasks section (excluding the finally section).
+          # This variable is only available in the finally tasks and can have any one of the values (Succeeded, Failed, Completed, or None)
+          value: $(tasks.status)