sqlmesh 0.220.2.dev7__tar.gz → 0.228.6.dev2__tar.gz

sqlmesh-0.228.6.dev2/.circleci/continue_config.yml

@@ -0,0 +1,331 @@
+version: 2.1
+
+parameters:
+  client:
+    type: boolean
+    default: false
+  common:
+    type: boolean
+    default: false
+  python:
+    type: boolean
+    default: false
+
+orbs:
+  windows: circleci/windows@5.0
+
+commands:
+  halt_unless_core:
+    steps:
+      - unless:
+          condition:
+            or:
+              - << pipeline.parameters.common >>
+              - << pipeline.parameters.python >>
+              - equal: [main, << pipeline.git.branch >>]
+          steps:
+            - run: circleci-agent step halt
+  halt_unless_client:
+    steps:
+      - unless:
+          condition:
+            or:
+              - << pipeline.parameters.common >>
+              - << pipeline.parameters.client >>
+              - equal: [main, << pipeline.git.branch >>]
+          steps:
+            - run: circleci-agent step halt
+
+jobs:
+  vscode_test:
+    docker:
+      - image: cimg/node:20.19.1-browsers
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - run:
+          name: Run VSCode extension CI
+          command: |
+            cd vscode/extension
+            pnpm run ci
+  doc_tests:
+    docker:
+      - image: cimg/python:3.10
+    resource_class: small
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install dependencies
+          command: make install-dev install-doc
+      - run:
+          name: Run doc tests
+          command: make doc-test
+
+  style_and_cicd_tests:
+    parameters:
+      python_version:
+        type: string
+    docker:
+      - image: cimg/python:<< parameters.python_version >>
+    resource_class: large
+    environment:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 8
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install OpenJDK
+          command: sudo apt-get update && sudo apt-get install default-jdk
+      - run:
+          name: Install ODBC
+          command: sudo apt-get install unixodbc-dev
+      - run:
+          name: Install SQLMesh dev dependencies
+          command: make install-dev
+      - run:
+          name: Fix Git URL override
+          command: git config --global --unset url."ssh://git@github.com".insteadOf
+      - run:
+          name: Run linters and code style checks
+          command: make py-style
+      - unless:
+          condition:
+            equal: ["3.9", << parameters.python_version >>]
+          steps:
+            - run:
+                name: Exercise the benchmarks
+                command: make benchmark-ci
+      - run:
+          name: Run cicd tests
+          command: make cicd-test
+      - store_test_results:
+          path: test-results
+
+  cicd_tests_windows:
+    executor:
+      name: windows/default
+      size: large
+    steps:
+      - halt_unless_core
+      - run:
+          name: Enable symlinks in git config
+          command: git config --global core.symlinks true
+      - checkout
+      - run:
+          name: Install System Dependencies
+          command: |
+            choco install make which -y
+            refreshenv
+      - run:
+          name: Install SQLMesh dev dependencies
+          command: |
+            python -m venv venv
+            . ./venv/Scripts/activate
+            python.exe -m pip install --upgrade pip
+            make install-dev
+      - run:
+          name: Run fast unit tests
+          command: |
+            . ./venv/Scripts/activate
+            which python
+            python --version
+            make fast-test
+      - store_test_results:
+          path: test-results
+
+  migration_test:
+    docker:
+      - image: cimg/python:3.10
+    resource_class: small
+    environment:
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Run the migration test - sushi
+          command: ./.circleci/test_migration.sh sushi "--gateway duckdb_persistent"
+      - run:
+          name: Run the migration test - sushi_dbt
+          command: ./.circleci/test_migration.sh sushi_dbt "--config migration_test_config"
+
+  ui_style:
+    docker:
+      - image: cimg/node:20.19.0
+    resource_class: small
+    steps:
+      - checkout
+      - restore_cache:
+          name: Restore pnpm Package Cache
+          keys:
+            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - save_cache:
+          name: Save pnpm Package Cache
+          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+          paths:
+            - .pnpm-store
+      - run:
+          name: Run linters and code style checks
+          command: pnpm run lint
+
+  ui_test:
+    docker:
+      - image: mcr.microsoft.com/playwright:v1.54.1-jammy
+    resource_class: medium
+    steps:
+      - halt_unless_client
+      - checkout
+      - restore_cache:
+          name: Restore pnpm Package Cache
+          keys:
+            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+      - run:
+          name: Install pnpm package manager
+          command: |
+            npm install --global corepack@latest
+            corepack enable
+            corepack prepare pnpm@latest-10 --activate
+            pnpm config set store-dir .pnpm-store
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - save_cache:
+          name: Save pnpm Package Cache
+          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+          paths:
+            - .pnpm-store
+      - run:
+          name: Run tests
+          command: npm --prefix web/client run test
+
+  engine_tests_docker:
+    parameters:
+      engine:
+        type: string
+    machine:
+      image: ubuntu-2404:2024.05.1
+      docker_layer_caching: true
+    resource_class: large
+    environment:
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install OS-level dependencies
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
+      - run:
+          name: Run tests
+          command: make << parameters.engine >>-test
+          no_output_timeout: 20m
+      - store_test_results:
+          path: test-results
+
+  engine_tests_cloud:
+    parameters:
+      engine:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    resource_class: medium
+    environment:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 4
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install OS-level dependencies
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
+      - run:
+          name: Generate database name
+          command: |
+            UUID=`cat /proc/sys/kernel/random/uuid`
+            TEST_DB_NAME="circleci_${UUID:0:8}"
+            echo "export TEST_DB_NAME='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export SNOWFLAKE_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+
+            # Make snowflake private key available            
+            echo $SNOWFLAKE_PRIVATE_KEY_RAW | base64 -d > /tmp/snowflake-keyfile.p8
+            echo "export SNOWFLAKE_PRIVATE_KEY_FILE='/tmp/snowflake-keyfile.p8'" >> "$BASH_ENV"
+      - run:
+          name: Create test database
+          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
+      - run:
+          name: Run tests
+          command: |
+            make << parameters.engine >>-test
+          no_output_timeout: 20m
+      - run:
+          name: Tear down test database
+          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" down
+          when: always
+      - store_test_results:
+          path: test-results
+
+workflows:
+  main_pr:
+    jobs:
+      - doc_tests
+      - style_and_cicd_tests:
+          matrix:
+            parameters:
+              python_version:
+                - "3.9"
+                - "3.10"
+                - "3.11"
+                - "3.12"
+                - "3.13"
+      - cicd_tests_windows
+      - engine_tests_docker:
+          name: engine_<< matrix.engine >>
+          matrix:
+            parameters:
+              engine:
+                - duckdb
+                - postgres
+                - mysql
+                - mssql
+                - trino
+                - spark
+                - clickhouse
+                - risingwave
+      - engine_tests_cloud:
+          name: cloud_engine_<< matrix.engine >>
+          context:
+            - sqlmesh_cloud_database_integration
+          requires:
+            - engine_tests_docker
+          matrix:
+            parameters:
+              engine:
+                - snowflake
+                - databricks
+                - redshift
+                - bigquery
+                - clickhouse-cloud
+                - athena
+                - fabric
+                - gcp-postgres
+          filters:
+            branches:
+              only:
+                - main
+      - ui_style
+      - ui_test
+      - vscode_test
+      - migration_test

sqlmesh-0.228.6.dev2/.circleci/manage-test-db.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+
+# The purpose of this script is to create and destroy temporary test databases on the cloud engines
+# The idea is that a database is created, the integration tests are run on that database and then the database is dropped
+# This allows builds for multiple PR's to run concurrently without the tests clobbering each other and also gives each set of tests a fresh environment
+
+# Note: It is expected that the environment variables defined in 'tests/core/engine_adapter/config.yaml' for each cloud engine are set
+
+set -e
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "USAGE: $0 <engine> <db_name> <up/down>"
+    exit 1
+fi
+
+ENGINE="$1"
+DB_NAME="$2"
+DIRECTION="$3"
+
+function_exists() {
+    declare -f -F $1 > /dev/null
+    return $?
+}
+
+# Snowflake
+snowflake_init() {
+    echo "Installing Snowflake CLI"
+    pip install "snowflake-cli"
+}
+
+snowflake_up() {
+    snow sql -q "create database if not exists $1" --temporary-connection
+}
+
+snowflake_down() {
+    snow sql -q "drop database if exists $1" --temporary-connection
+}
+
+# Databricks
+databricks_init() {
+    echo "Installing Databricks CLI"
+    curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh || true
+}
+
+databricks_up() {
+    databricks catalogs create $1 || true
+}
+
+databricks_down() {
+    databricks catalogs delete $1 --force || true
+}
+
+# Redshift
+redshift_init() {
+    psql --version
+}
+
+redshift_exec() {
+    PGPASSWORD=$REDSHIFT_PASSWORD psql -h $REDSHIFT_HOST -p $REDSHIFT_PORT -U $REDSHIFT_USER -c "$1" dev
+}
+
+redshift_up() {
+    redshift_exec "create database $1"
+}
+
+redshift_down() {
+    # try to prevent a "database is being accessed by other users" error when running DROP DATABASE
+    EXIT_CODE=1
+    ATTEMPTS=0
+    while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges 
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+        
+        # perform drop
+        redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+            echo "Unable to drop database; retrying..."
+            ATTEMPTS=$((ATTEMPTS + 1))
+            sleep 5
+        fi
+    done
+}
+
+# BigQuery
+bigquery_init() {
+    # Write out the keyfile for the integration tests to pick up
+    echo "Writing out keyfile to $BIGQUERY_KEYFILE"
+    echo "$BIGQUERY_KEYFILE_CONTENTS" > $BIGQUERY_KEYFILE
+}
+
+
+# Clickhouse cloud
+clickhouse-cloud_init() {
+    # note: the ping endpoint doesnt seem to need any API keys
+    until curl https://$CLICKHOUSE_CLOUD_HOST:8443/ping
+    do
+        echo "Pinging Clickhouse Cloud service to ensure it's not in idle mode..."
+        sleep 5
+    done
+    echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
+}
+
+# GCP Postgres
+gcp-postgres_init() {
+    # Download and start Cloud SQL Proxy
+    curl -fsSL -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.18.0/cloud-sql-proxy.linux.amd64
+    chmod +x cloud-sql-proxy
+    echo "$GCP_POSTGRES_KEYFILE_JSON" > /tmp/keyfile.json
+    ./cloud-sql-proxy --credentials-file /tmp/keyfile.json $GCP_POSTGRES_INSTANCE_CONNECTION_STRING &
+
+    # Wait for proxy to start
+    sleep 5
+}
+
+gcp-postgres_exec() {
+    PGPASSWORD=$GCP_POSTGRES_PASSWORD psql -h 127.0.0.1 -U $GCP_POSTGRES_USER -c "$1" postgres
+}
+
+gcp-postgres_up() {
+    gcp-postgres_exec "create database $1"
+}
+
+gcp-postgres_down() {
+    gcp-postgres_exec "drop database $1"
+}
+
+# Fabric
+fabric_init() {    
+    python --version #note: as at 2025-08-20, ms-fabric-cli is pinned to Python >= 3.10, <3.13
+    pip install ms-fabric-cli
+    
+    # to prevent the '[EncryptionFailed] An error occurred with the encrypted cache.' error
+    # ref: https://microsoft.github.io/fabric-cli/#switch-to-interactive-mode-optional
+    fab config set encryption_fallback_enabled true 
+
+    echo "Logging in to Fabric"
+    fab auth login -u $FABRIC_CLIENT_ID -p $FABRIC_CLIENT_SECRET --tenant $FABRIC_TENANT_ID
+}
+
+fabric_up() {
+    fab create "SQLMesh CircleCI.Workspace/$1.Warehouse"
+}
+
+fabric_down() {
+    fab rm -f "SQLMesh CircleCI.Workspace/$1.Warehouse" || true
+}
+
+INIT_FUNC="${ENGINE}_init"
+UP_FUNC="${ENGINE}_up"
+DOWN_FUNC="${ENGINE}_down"
+
+# If called with an unimplemented / unsupported engine, just exit
+if ! function_exists $INIT_FUNC ; then
+    echo "WARN: $INIT_FUNC not implemeted; exiting"
+    exit 0
+fi
+
+echo "Initializing $ENGINE"
+$INIT_FUNC
+
+if [ "$DIRECTION" == "up" ] && function_exists $UP_FUNC; then
+    echo "Creating database $DB_NAME"
+    $UP_FUNC $DB_NAME
+elif [ "$DIRECTION" == "down" ] && function_exists $DOWN_FUNC; then
+    echo "Dropping database $DB_NAME"
+    $DOWN_FUNC $DB_NAME
+fi
+
+echo "All done"

sqlmesh-0.228.6.dev2/.circleci/test_migration.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ -z $(git tag --points-at HEAD) ]]; then
+    # If the current commit is not tagged, we need to find the last tag
+    LAST_TAG=$(git describe --tags --abbrev=0)
+else
+    # If the current commit is tagged, we need to find the previous tag
+    LAST_TAG=$(git tag --sort=-creatordate | head -n 2 | tail -n 1)
+fi
+
+if [ "$1" == "" ]; then
+    echo "Usage: $0 <example name> <sqlmesh opts>"
+    echo "eg $0 sushi '--gateway duckdb_persistent'"
+    exit 1
+fi
+
+
+TMP_DIR=$(mktemp -d)
+EXAMPLE_NAME="$1"
+SQLMESH_OPTS="$2"
+EXAMPLE_DIR="./examples/$EXAMPLE_NAME"
+TEST_DIR="$TMP_DIR/$EXAMPLE_NAME"
+
+echo "Running migration test for '$EXAMPLE_NAME' in '$TEST_DIR' for example project '$EXAMPLE_DIR' using options '$SQLMESH_OPTS'"
+
+# Copy the example project from the *current* checkout so it's stable across old/new SQLMesh versions
+cp -r "$EXAMPLE_DIR" "$TEST_DIR"
+
+git checkout $LAST_TAG
+
+# Install dependencies from the previous release.
+make install-dev
+
+# this is only needed temporarily until the released tag for $LAST_TAG includes this config
+if [ "$EXAMPLE_NAME" == "sushi_dbt" ]; then
+    echo 'migration_test_config = sqlmesh_config(Path(__file__).parent, dbt_target_name="duckdb")' >> $TEST_DIR/config.py
+fi    
+
+# Run initial plan
+pushd $TEST_DIR
+rm -rf ./data/*
+sqlmesh $SQLMESH_OPTS plan --no-prompts --auto-apply
+rm -rf .cache
+popd
+
+# Switch back to the starting state of the repository    
+git checkout -
+
+# Install updated dependencies.
+make install-dev
+
+# Migrate and make sure the diff is empty
+pushd $TEST_DIR
+sqlmesh $SQLMESH_OPTS migrate
+sqlmesh $SQLMESH_OPTS diff prod
+popd

sqlmesh-0.228.6.dev2/.github/workflows/pr.yaml

@@ -0,0 +1,148 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+concurrency:
+  group: 'pr-${{ github.event.pull_request.number }}'
+  cancel-in-progress: true
+permissions:
+  contents: read
+jobs:
+  test-vscode:
+    env:
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+      - uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install dependencies
+        run: pnpm install
+      - name: Run CI
+        run: pnpm run ci
+  test-vscode-e2e:
+    runs-on:
+      labels: [ubuntu-2204-8]
+    # As at 2026-01-12 this job flakes 100% of the time. It needs investigation
+    if: false
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+      - uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install dependencies
+        run: pnpm install
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install python dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          make install-dev
+      - name: Install code-server
+        run: curl -fsSL https://code-server.dev/install.sh | sh
+      - name: Install Playwright browsers
+        working-directory: ./vscode/extension
+        run: pnpm exec playwright install
+      - name: Run e2e tests
+        working-directory: ./vscode/extension
+        timeout-minutes: 30
+        run: |
+          source ../../.venv/bin/activate
+          pnpm run test:e2e
+      - uses: actions/upload-artifact@v5
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: vscode/extension/playwright-report/
+          retention-days: 30
+  test-dbt-versions:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        dbt-version: ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10']
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install SQLMesh dev dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          UV=1 make install-dev-dbt-${{ matrix.dbt-version }}
+      - name: Run dbt tests
+        # We can't run slow tests across all engines due to tests requiring DuckDB and old versions
+        # of DuckDB require a version of DuckDB we no longer support
+        run: |
+          source .venv/bin/activate
+
+          # Remove semantic_models and metrics sections for DBT versions < 1.6.0
+          # Using explicit list to avoid version comparison issues
+          if [[ "${{ matrix.dbt-version }}" == "1.3" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.4" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.5" ]]; then
+            
+            echo "DBT version is ${{ matrix.dbt-version }} (< 1.6.0), removing semantic_models and metrics sections..."
+            
+            schema_file="tests/fixtures/dbt/sushi_test/models/schema.yml"
+            if [[ -f "$schema_file" ]]; then
+              echo "Modifying $schema_file..."
+              
+              # Create a temporary file
+              temp_file=$(mktemp)
+              
+              # Use awk to remove semantic_models and metrics sections
+              awk '
+                /^semantic_models:/ { in_semantic=1; next }
+                /^metrics:/ { in_metrics=1; next }
+                /^[^ ]/ && (in_semantic || in_metrics) { 
+                  in_semantic=0; 
+                  in_metrics=0 
+                }
+                !in_semantic && !in_metrics { print }
+              ' "$schema_file" > "$temp_file"
+              
+              # Move the temp file back
+              mv "$temp_file" "$schema_file"
+              
+              echo "Successfully removed semantic_models and metrics sections"
+            else
+              echo "Schema file not found at $schema_file, skipping..."
+            fi
+          else
+            echo "DBT version is ${{ matrix.dbt-version }} (>= 1.6.0), keeping semantic_models and metrics sections"
+          fi
+
+          make dbt-fast-test
+      - name: Test SQLMesh info in sushi_dbt
+        working-directory: ./examples/sushi_dbt
+        run: |
+          source ../../.venv/bin/activate
+          sed -i 's/target: in_memory/target: postgres/g' profiles.yml
+          if [[ $(echo -e "${{ matrix.dbt-version }}\n1.5.0" | sort -V | head -n1) == "${{ matrix.dbt-version }}" ]] && [[ "${{ matrix.dbt-version }}" != "1.5.0" ]]; then
+            echo "DBT version is ${{ matrix.dbt-version }} (< 1.5.0), removing version parameters..."
+            sed -i -e 's/, version=1) }}/) }}/g' -e 's/, v=1) }}/) }}/g' models/top_waiters.sql
+          else
+            echo "DBT version is ${{ matrix.dbt-version }} (>= 1.5.0), keeping version parameters"
+          fi
+
+          sqlmesh info --skip-connection