sqlmesh 0.231.2.dev5__tar.gz → 0.233.0__tar.gz

sqlmesh-0.233.0/.github/scripts/install-prerequisites.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# This script is intended to be run by an Ubuntu CI build agent
+# The goal is to install OS-level dependencies that are required before trying to install Python dependencies
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "USAGE: $0 <engine>"
+    exit 1
+fi
+
+ENGINE="$1"
+
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev"
+ENGINE_DEPENDENCIES=""
+
+if [ "$ENGINE" == "spark" ]; then
+    ENGINE_DEPENDENCIES="default-jdk"
+elif [ "$ENGINE" == "fabric" ]; then
+    echo "Installing Microsoft package repository"
+
+    # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+    curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+    sudo dpkg -i packages-microsoft-prod.deb
+    rm packages-microsoft-prod.deb
+
+    ENGINE_DEPENDENCIES="msodbcsql18"
+fi
+
+ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"
+
+echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
+
+sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
+
+if [ "$ENGINE" == "spark" ]; then
+    echo "Using Java version for spark:"
+    java -version
+fi
+
+echo "All done"

sqlmesh-0.233.0/.github/scripts/manage-test-db.sh

@@ -0,0 +1,172 @@
+#!/bin/bash
+
+# The purpose of this script is to create and destroy temporary test databases on the cloud engines
+# The idea is that a database is created, the integration tests are run on that database and then the database is dropped
+# This allows builds for multiple PR's to run concurrently without the tests clobbering each other and also gives each set of tests a fresh environment
+
+# Note: It is expected that the environment variables defined in 'tests/core/engine_adapter/config.yaml' for each cloud engine are set
+
+set -e
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "USAGE: $0 <engine> <db_name> <up/down>"
+    exit 1
+fi
+
+ENGINE="$1"
+DB_NAME="$2"
+DIRECTION="$3"
+
+function_exists() {
+    declare -f -F $1 > /dev/null
+    return $?
+}
+
+# Snowflake
+snowflake_init() {
+    echo "Installing Snowflake CLI"
+    pip install "snowflake-cli"
+}
+
+snowflake_up() {
+    snow sql -q "create database if not exists $1" --temporary-connection
+}
+
+snowflake_down() {
+    snow sql -q "drop database if exists $1" --temporary-connection
+}
+
+# Databricks
+databricks_init() {
+    echo "Installing Databricks CLI"
+    curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh || true
+}
+
+databricks_up() {
+    databricks catalogs create $1 || true
+}
+
+databricks_down() {
+    databricks catalogs delete $1 --force || true
+}
+
+# Redshift
+redshift_init() {
+    psql --version
+}
+
+redshift_exec() {
+    PGPASSWORD=$REDSHIFT_PASSWORD psql -h $REDSHIFT_HOST -p $REDSHIFT_PORT -U $REDSHIFT_USER -c "$1" dev
+}
+
+redshift_up() {
+    redshift_exec "create database $1"
+}
+
+redshift_down() {
+    # try to prevent a "database is being accessed by other users" error when running DROP DATABASE
+    EXIT_CODE=1
+    ATTEMPTS=0
+    while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+
+        # perform drop
+        redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+            echo "Unable to drop database; retrying..."
+            ATTEMPTS=$((ATTEMPTS + 1))
+            sleep 5
+        fi
+    done
+}
+
+# BigQuery
+bigquery_init() {
+    # Write out the keyfile for the integration tests to pick up
+    echo "Writing out keyfile to $BIGQUERY_KEYFILE"
+    echo "$BIGQUERY_KEYFILE_CONTENTS" > $BIGQUERY_KEYFILE
+}
+
+
+# Clickhouse cloud
+clickhouse-cloud_init() {
+    # note: the ping endpoint doesnt seem to need any API keys
+    until curl https://$CLICKHOUSE_CLOUD_HOST:8443/ping
+    do
+        echo "Pinging Clickhouse Cloud service to ensure it's not in idle mode..."
+        sleep 5
+    done
+    echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
+}
+
+# GCP Postgres
+gcp-postgres_init() {
+    # Download Cloud SQL Proxy if not already present
+    if [ ! -f cloud-sql-proxy ]; then
+        curl -fsSL -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.18.0/cloud-sql-proxy.linux.amd64
+        chmod +x cloud-sql-proxy
+    fi
+    echo "$GCP_POSTGRES_KEYFILE_JSON" > /tmp/keyfile.json
+    if ! pgrep -x cloud-sql-proxy > /dev/null; then
+        ./cloud-sql-proxy --credentials-file /tmp/keyfile.json $GCP_POSTGRES_INSTANCE_CONNECTION_STRING &
+        sleep 5
+    fi
+}
+
+gcp-postgres_exec() {
+    PGPASSWORD=$GCP_POSTGRES_PASSWORD psql -h 127.0.0.1 -U $GCP_POSTGRES_USER -c "$1" postgres
+}
+
+gcp-postgres_up() {
+    gcp-postgres_exec "create database $1"
+}
+
+gcp-postgres_down() {
+    gcp-postgres_exec "drop database $1"
+}
+
+# Fabric
+fabric_init() {
+    python --version #note: as at 2025-08-20, ms-fabric-cli is pinned to Python >= 3.10, <3.13
+    pip install ms-fabric-cli
+
+    # to prevent the '[EncryptionFailed] An error occurred with the encrypted cache.' error
+    # ref: https://microsoft.github.io/fabric-cli/#switch-to-interactive-mode-optional
+    fab config set encryption_fallback_enabled true
+
+    echo "Logging in to Fabric"
+    fab auth login -u $FABRIC_CLIENT_ID -p $FABRIC_CLIENT_SECRET --tenant $FABRIC_TENANT_ID
+}
+
+fabric_up() {
+    fab create "SQLMesh CircleCI.Workspace/$1.Warehouse"
+}
+
+fabric_down() {
+    fab rm -f "SQLMesh CircleCI.Workspace/$1.Warehouse" || true
+}
+
+INIT_FUNC="${ENGINE}_init"
+UP_FUNC="${ENGINE}_up"
+DOWN_FUNC="${ENGINE}_down"
+
+# If called with an unimplemented / unsupported engine, just exit
+if ! function_exists $INIT_FUNC ; then
+    echo "WARN: $INIT_FUNC not implemeted; exiting"
+    exit 0
+fi
+
+echo "Initializing $ENGINE"
+$INIT_FUNC
+
+if [ "$DIRECTION" == "up" ] && function_exists $UP_FUNC; then
+    echo "Creating database $DB_NAME"
+    $UP_FUNC $DB_NAME
+elif [ "$DIRECTION" == "down" ] && function_exists $DOWN_FUNC; then
+    echo "Dropping database $DB_NAME"
+    $DOWN_FUNC $DB_NAME
+fi
+
+echo "All done"

sqlmesh-0.233.0/.github/scripts/test_migration.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ -z $(git tag --points-at HEAD) ]]; then
+    # If the current commit is not tagged, we need to find the last tag
+    LAST_TAG=$(git describe --tags --abbrev=0)
+else
+    # If the current commit is tagged, we need to find the previous tag
+    LAST_TAG=$(git tag --sort=-creatordate | head -n 2 | tail -n 1)
+fi
+
+if [ "$1" == "" ]; then
+    echo "Usage: $0 <example name> <sqlmesh opts>"
+    echo "eg $0 sushi '--gateway duckdb_persistent'"
+    exit 1
+fi
+
+
+TMP_DIR=$(mktemp -d)
+EXAMPLE_NAME="$1"
+SQLMESH_OPTS="$2"
+EXAMPLE_DIR="./examples/$EXAMPLE_NAME"
+TEST_DIR="$TMP_DIR/$EXAMPLE_NAME"
+
+echo "Running migration test for '$EXAMPLE_NAME' in '$TEST_DIR' for example project '$EXAMPLE_DIR' using options '$SQLMESH_OPTS'"
+
+# Copy the example project from the *current* checkout so it's stable across old/new SQLMesh versions
+cp -r "$EXAMPLE_DIR" "$TEST_DIR"
+
+git checkout $LAST_TAG
+
+# Install dependencies from the previous release.
+uv venv .venv --clear
+source .venv/bin/activate
+make install-dev
+
+# this is only needed temporarily until the released tag for $LAST_TAG includes this config
+if [ "$EXAMPLE_NAME" == "sushi_dbt" ]; then
+    echo 'migration_test_config = sqlmesh_config(Path(__file__).parent, dbt_target_name="duckdb")' >> $TEST_DIR/config.py
+fi
+
+# Run initial plan
+pushd $TEST_DIR
+rm -rf ./data/*
+sqlmesh $SQLMESH_OPTS plan --no-prompts --auto-apply
+rm -rf .cache
+popd
+
+# Switch back to the starting state of the repository
+git checkout -
+
+# Install updated dependencies.
+uv venv .venv --clear
+source .venv/bin/activate
+make install-dev
+
+# Migrate and make sure the diff is empty
+pushd $TEST_DIR
+sqlmesh $SQLMESH_OPTS migrate
+sqlmesh $SQLMESH_OPTS diff prod
+popd

sqlmesh-0.233.0/.github/scripts/wait-for-db.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# The purpose of this script is to be called after `docker compose up -d` has been run for a given database
+# The idea is to block until the database is available to serve requests. Once the database can serve requests,
+# the integration tests can be run.
+# Therefore, the ports etc are tightly coupled with the compose.yml files under tests/core/engine_adapter/docker/
+#
+# Note that if the docker daemon is not running `localhost`, you can set the DOCKER_HOSTNAME environment variable to the
+# correct host Docker is running on
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "USAGE: $0 <engine>"
+    exit 1
+fi
+
+ENGINE="$1"
+
+function_exists() {
+    declare -f -F $1 > /dev/null
+    return $?
+}
+
+probe_port() {
+    HOSTNAME=${DOCKER_HOSTNAME:-localhost}
+    echo "Probing '$HOSTNAME' on port $1"
+    while ! nc -z $HOSTNAME $1; do
+        sleep 1
+    done
+}
+
+clickhouse_ready() {
+    probe_port 8123
+}
+
+postgres_ready() {
+    probe_port 5432
+}
+
+mssql_ready() {
+    probe_port 1433
+}
+
+mysql_ready() {
+    probe_port 3306
+}
+
+spark_ready() {
+    probe_port 15002
+}
+
+trino_ready() {
+    # Trino has a built-in healthcheck script, just call that
+    docker compose -f tests/core/engine_adapter/integration/docker/compose.trino.yaml exec trino /bin/bash -c '/usr/lib/trino/bin/health-check'
+}
+
+risingwave_ready() {
+    probe_port 4566
+}
+
+echo "Waiting for $ENGINE to be ready..."
+
+READINESS_FUNC="${ENGINE}_ready"
+
+# If called with an unimplemented / unsupported engine, just exit
+if ! function_exists $READINESS_FUNC ; then
+    echo "WARN: $READINESS_FUNC not implemeted; exiting"
+    exit 0
+fi
+
+EXIT_CODE=1
+
+while [ $EXIT_CODE -ne 0 ]; do
+    echo "Checking $ENGINE"
+    $READINESS_FUNC && EXIT_CODE=$? || EXIT_CODE=$?
+    if [ $EXIT_CODE -ne 0 ]; then
+        echo "$ENGINE not ready; sleeping"
+        sleep 5
+    fi
+done
+
+echo "$ENGINE is ready!"