sqlmesh 0.209.1.dev69__tar.gz → 0.209.2__tar.gz

sqlmesh-0.209.2/.circleci/continue_config.yml

@@ -0,0 +1,313 @@
+version: 2.1
+
+parameters:
+  client:
+    type: boolean
+    default: false
+  common:
+    type: boolean
+    default: false
+  python:
+    type: boolean
+    default: false
+
+orbs:
+  windows: circleci/windows@5.0
+
+commands:
+  halt_unless_core:
+    steps:
+      - unless:
+          condition:
+            or:
+              - << pipeline.parameters.common >>
+              - << pipeline.parameters.python >>
+              - equal: [main, << pipeline.git.branch >>]
+          steps:
+            - run: circleci-agent step halt
+  halt_unless_client:
+    steps:
+      - unless:
+          condition:
+            or:
+              - << pipeline.parameters.common >>
+              - << pipeline.parameters.client >>
+              - equal: [main, << pipeline.git.branch >>]
+          steps:
+            - run: circleci-agent step halt
+
+jobs:
+  vscode_test:
+    docker:
+      - image: cimg/node:20.19.1-browsers
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - run:
+          name: Run VSCode extension CI
+          command: |
+            cd vscode/extension
+            pnpm run ci
+  doc_tests:
+    docker:
+      - image: cimg/python:3.10
+    resource_class: small
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install dependencies
+          command: make install-dev install-doc
+      - run:
+          name: Run doc tests
+          command: make doc-test
+
+  style_and_cicd_tests:
+    parameters:
+      python_version:
+        type: string
+    docker:
+      - image: cimg/python:<< parameters.python_version >>
+    resource_class: large
+    environment:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 8
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install OpenJDK
+          command: sudo apt-get update && sudo apt-get install default-jdk
+      - run:
+          name: Install ODBC
+          command: sudo apt-get install unixodbc-dev
+      - run:
+          name: Install SQLMesh dev dependencies
+          command: make install-dev
+      - run:
+          name: Fix Git URL override
+          command: git config --global --unset url."ssh://git@github.com".insteadOf
+      - run:
+          name: Run linters and code style checks
+          command: make py-style
+      - run:
+          name: Exercise the benchmarks
+          command: make benchmark-ci
+      - run:
+          name: Run cicd tests
+          command: make cicd-test
+      - store_test_results:
+          path: test-results
+
+  cicd_tests_windows:
+    executor:
+      name: windows/default
+      size: large
+    steps:
+      - halt_unless_core
+      - run:
+          name: Enable symlinks in git config
+          command: git config --global core.symlinks true
+      - checkout
+      - run:
+          name: Install System Dependencies
+          command: |
+            choco install make which -y
+            refreshenv
+      - run:
+          name: Install SQLMesh dev dependencies
+          command: |
+            python -m venv venv
+            . ./venv/Scripts/activate
+            python.exe -m pip install --upgrade pip
+            make install-dev
+      - run:
+          name: Run fast unit tests
+          command: |
+            . ./venv/Scripts/activate
+            which python
+            python --version
+            make fast-test
+      - store_test_results:
+          path: test-results
+
+  migration_test:
+    docker:
+      - image: cimg/python:3.10
+    resource_class: small
+    environment:
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Run the migration test
+          command: ./.circleci/test_migration.sh
+
+  ui_style:
+    docker:
+      - image: cimg/node:20.19.0
+    resource_class: small
+    steps:
+      - checkout
+      - restore_cache:
+          name: Restore pnpm Package Cache
+          keys:
+            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - save_cache:
+          name: Save pnpm Package Cache
+          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+          paths:
+            - .pnpm-store
+      - run:
+          name: Run linters and code style checks
+          command: pnpm run lint
+
+  ui_test:
+    docker:
+      - image: mcr.microsoft.com/playwright:v1.54.1-jammy
+    resource_class: medium
+    steps:
+      - halt_unless_client
+      - checkout
+      - restore_cache:
+          name: Restore pnpm Package Cache
+          keys:
+            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+      - run:
+          name: Install pnpm package manager
+          command: |
+            npm install --global corepack@latest
+            corepack enable
+            corepack prepare pnpm@latest-10 --activate
+            pnpm config set store-dir .pnpm-store
+      - run:
+          name: Install Dependencies
+          command: |
+            pnpm install
+      - save_cache:
+          name: Save pnpm Package Cache
+          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
+          paths:
+            - .pnpm-store
+      - run:
+          name: Run tests
+          command: npm --prefix web/client run test
+
+  engine_tests_docker:
+    parameters:
+      engine:
+        type: string
+    machine:
+      image: ubuntu-2404:2024.05.1
+      docker_layer_caching: true
+    resource_class: large
+    environment:
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Install OS-level dependencies
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
+      - run:
+          name: Run tests
+          command: make << parameters.engine >>-test
+          no_output_timeout: 20m
+      - store_test_results:
+          path: test-results
+
+  engine_tests_cloud:
+    parameters:
+      engine:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    resource_class: medium
+    environment:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 4
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
+    steps:
+      - halt_unless_core
+      - checkout
+      - run:
+          name: Generate database name
+          command: |
+            UUID=`cat /proc/sys/kernel/random/uuid`
+            TEST_DB_NAME="circleci_${UUID:0:8}"
+            echo "export TEST_DB_NAME='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export SNOWFLAKE_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+      - run:
+          name: Create test database
+          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
+      - run:
+          name: Run tests
+          command: |
+            make << parameters.engine >>-test
+          no_output_timeout: 20m
+      - run:
+          name: Tear down test database
+          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" down
+          when: always
+      - store_test_results:
+          path: test-results
+
+workflows:
+  main_pr:
+    jobs:
+      - doc_tests
+      - style_and_cicd_tests:
+          matrix:
+            parameters:
+              python_version:
+                - "3.9"
+                - "3.10"
+                - "3.11"
+                - "3.12"
+                - "3.13"
+      - cicd_tests_windows
+      - engine_tests_docker:
+          name: engine_<< matrix.engine >>
+          matrix:
+            parameters:
+              engine:
+                - duckdb
+                - postgres
+                - mysql
+                - mssql
+                - trino
+                - spark
+                - clickhouse
+                - risingwave
+      - engine_tests_cloud:
+          name: cloud_engine_<< matrix.engine >>
+          context:
+            - sqlmesh_cloud_database_integration
+          requires:
+            - engine_tests_docker
+          matrix:
+            parameters:
+              engine:
+                - snowflake
+                - databricks
+                - redshift
+                - bigquery
+                - clickhouse-cloud
+                - athena
+          filters:
+            branches:
+              only:
+                - main
+      - ui_style
+      - ui_test
+      - vscode_test
+      - migration_test

sqlmesh-0.209.2/.circleci/install-prerequisites.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# This script is intended to be run by an Ubuntu build agent on CircleCI
+# The goal is to install OS-level dependencies that are required before trying to install Python dependencies
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "USAGE: $0 <engine>"
+    exit 1
+fi
+
+ENGINE="$1"
+
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional"
+ENGINE_DEPENDENCIES=""
+
+if [ "$ENGINE" == "spark" ]; then
+    ENGINE_DEPENDENCIES="default-jdk"
+fi
+
+ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"
+
+echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
+
+sudo apt-get clean && sudo apt-get -y update && sudo apt-get -y install $ALL_DEPENDENCIES
+
+echo "All done"

sqlmesh-0.209.2/.circleci/manage-test-db.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# The purpose of this script is to create and destroy temporary test databases on the cloud engines
+# The idea is that a database is created, the integration tests are run on that database and then the database is dropped
+# This allows builds for multiple PR's to run concurrently without the tests clobbering each other and also gives each set of tests a fresh environment
+
+# Note: It is expected that the environment variables defined in 'tests/core/engine_adapter/config.yaml' for each cloud engine are set
+
+set -e
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "USAGE: $0 <engine> <db_name> <up/down>"
+    exit 1
+fi
+
+ENGINE="$1"
+DB_NAME="$2"
+DIRECTION="$3"
+
+function_exists() {
+    declare -f -F $1 > /dev/null
+    return $?
+}
+
+# Snowflake
+snowflake_init() {
+    echo "Installing Snowflake CLI"
+    pip install "snowflake-cli-labs<3.8.0"
+}
+
+snowflake_up() {
+    snow sql -q "create database if not exists $1" --temporary-connection
+}
+
+snowflake_down() {
+    snow sql -q "drop database if exists $1" --temporary-connection
+}
+
+# Databricks
+databricks_init() {
+    echo "Installing Databricks CLI"
+    curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh || true
+
+    echo "Writing out Databricks CLI config file"
+    echo -e "[DEFAULT]\nhost = $DATABRICKS_SERVER_HOSTNAME\ntoken = $DATABRICKS_ACCESS_TOKEN" > ~/.databrickscfg
+
+    # this takes a path like 'sql/protocolv1/o/2934659247569/0723-005339-foobar' and extracts '0723-005339-foobar' from it
+    CLUSTER_ID=${DATABRICKS_HTTP_PATH##*/}
+
+    echo "Extracted cluster id: $CLUSTER_ID from '$DATABRICKS_HTTP_PATH'"
+
+    # Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests
+    echo "Ensuring cluster is running"
+    databricks clusters start $CLUSTER_ID || true
+}
+
+databricks_up() {
+    databricks catalogs create $1 || true
+}
+
+databricks_down() {
+    databricks catalogs delete $1 --force || true
+}
+
+# Redshift
+redshift_init() {
+    psql --version
+}
+
+redshift_exec() {
+    PGPASSWORD=$REDSHIFT_PASSWORD psql -h $REDSHIFT_HOST -p $REDSHIFT_PORT -U $REDSHIFT_USER -c "$1" dev
+}
+
+redshift_up() {
+    redshift_exec "create database $1"
+}
+
+redshift_down() {
+    # try to prevent a "database is being accessed by other users" error when running DROP DATABASE
+    EXIT_CODE=1
+    ATTEMPTS=0
+    while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'"
+        redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+            echo "Unable to drop database; retrying..."
+            ATTEMPTS=$((ATTEMPTS + 1))
+            sleep 5
+        fi
+    done
+}
+
+# BigQuery
+bigquery_init() {
+    # Write out the keyfile for the integration tests to pick up
+    echo "Writing out keyfile to $BIGQUERY_KEYFILE"
+    echo "$BIGQUERY_KEYFILE_CONTENTS" > $BIGQUERY_KEYFILE
+}
+
+
+# Clickhouse cloud
+clickhouse-cloud_init() {
+    # note: the ping endpoint doesnt seem to need any API keys
+    until curl https://$CLICKHOUSE_CLOUD_HOST:8443/ping
+    do
+        echo "Pinging Clickhouse Cloud service to ensure it's not in idle mode..."
+        sleep 5
+    done
+    echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
+}
+
+INIT_FUNC="${ENGINE}_init"
+UP_FUNC="${ENGINE}_up"
+DOWN_FUNC="${ENGINE}_down"
+
+# If called with an unimplemented / unsupported engine, just exit
+if ! function_exists $INIT_FUNC ; then
+    echo "WARN: $INIT_FUNC not implemeted; exiting"
+    exit 0
+fi
+
+echo "Initializing $ENGINE"
+$INIT_FUNC
+
+if [ "$DIRECTION" == "up" ] && function_exists $UP_FUNC; then
+    echo "Creating database $DB_NAME"
+    $UP_FUNC $DB_NAME
+elif [ "$DIRECTION" == "down" ] && function_exists $DOWN_FUNC; then
+    echo "Dropping database $DB_NAME"
+    $DOWN_FUNC $DB_NAME
+fi
+
+echo "All done"

sqlmesh-0.209.2/.github/workflows/pr.yaml

@@ -0,0 +1,68 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+concurrency:
+  group: 'pr-${{ github.event.pull_request.number }}'
+  cancel-in-progress: true
+jobs:
+  test-vscode:
+    env:
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install dependencies
+        run: pnpm install
+      - name: Run CI
+        run: pnpm run ci
+  test-vscode-e2e:
+    runs-on:
+      labels: [ubuntu-2204-8]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install dependencies
+        run: pnpm install
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install python dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          make install-dev
+      - name: Install code-server
+        run: curl -fsSL https://code-server.dev/install.sh | sh
+      - name: Install Playwright browsers
+        working-directory: ./vscode/extension
+        run: pnpm exec playwright install
+      - name: Run e2e tests
+        working-directory: ./vscode/extension
+        timeout-minutes: 90
+        run: |
+          source ../../.venv/bin/activate
+          pnpm run test:e2e
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: vscode/extension/playwright-report/
+          retention-days: 30

sqlmesh-0.209.2/.github/workflows/private-repo-test.yaml

@@ -0,0 +1,96 @@
+name: Private Repo Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: 'private-test-${{ github.event.pull_request.number }}'
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  trigger-private-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Set up Node.js for UI build
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install UI dependencies
+        run: pnpm install
+      - name: Build UI
+        run: pnpm --prefix web/client run build
+      - name: Install Python dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install build twine setuptools_scm
+      - name: Generate development version
+        id: version
+        run: |
+          source .venv/bin/activate
+          # Generate a PEP 440 compliant unique version including run attempt
+          BASE_VERSION=$(python .github/scripts/get_scm_version.py)
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          # Use PEP 440 compliant format: base.devN+pr.sha.attempt
+          UNIQUE_VERSION="${BASE_VERSION}+pr${{ github.event.pull_request.number }}.${COMMIT_SHA}.run${{ github.run_attempt }}"
+          echo "version=$UNIQUE_VERSION" >> $GITHUB_OUTPUT
+          echo "Generated unique version with run attempt: $UNIQUE_VERSION"
+      - name: Build package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Configure PyPI for private repository
+        env:
+          TOBIKO_PRIVATE_PYPI_URL: ${{ secrets.TOBIKO_PRIVATE_PYPI_URL }}
+          TOBIKO_PRIVATE_PYPI_KEY: ${{ secrets.TOBIKO_PRIVATE_PYPI_KEY }}
+        run: ./.circleci/update-pypirc.sh
+      - name: Publish to private PyPI
+        run: |
+          source .venv/bin/activate
+          python -m twine upload -r tobiko-private dist/*
+      - name: Publish Python Tests package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          source .venv/bin/activate
+          unset TWINE_USERNAME TWINE_PASSWORD && make publish-tests
+      - name: Get GitHub App token
+        id: get_token
+        uses: actions/create-github-app-token@v1
+        with:
+          private-key: ${{ secrets.TOBIKO_RENOVATE_BOT_PRIVATE_KEY }}
+          app-id: ${{ secrets.TOBIKO_RENOVATE_BOT_APP_ID }}
+          owner: ${{ secrets.PRIVATE_REPO_OWNER }}
+      - name: Trigger private repository workflow
+        uses: convictional/trigger-workflow-and-wait@v1.6.5
+        with:
+          owner: ${{ secrets.PRIVATE_REPO_OWNER }}
+          repo: ${{ secrets.PRIVATE_REPO_NAME }}
+          github_token: ${{ steps.get_token.outputs.token }}
+          workflow_file_name: ${{ secrets.PRIVATE_WORKFLOW_FILE }}
+          client_payload: |
+            {
+              "package_version": "${{ steps.version.outputs.version }}",
+              "pr_number": "${{ github.event.pull_request.number }}"
+            }

sqlmesh-0.209.2/.github/workflows/release_extension.yaml

@@ -0,0 +1,56 @@
+name: Release VSCode Extension
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., 1.0.0)'
+        required: true
+        type: string
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Check branch is main
+        run: |
+          if [[ "${{ github.ref }}" != "refs/heads/main" ]]; then
+            echo "Error: This workflow can only be run from the main branch"
+            exit 1
+          fi
+          echo "Branch check passed: running from main branch"
+      - name: Validate version format
+        run: |
+          version="${{ github.event.inputs.version }}"
+          if ! [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+            echo "Error: Version must be a valid semantic version (e.g., 1.0.0, 1.0.0-beta.1, 1.0.0+build.1)"
+            exit 1
+          fi
+          echo "Version format is valid: $version"
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Update package.json version
+        working-directory: vscode/extension
+        run: |
+          npm version ${{ github.event.inputs.version }} --no-git-tag-version
+      - name: Build extension
+        working-directory: vscode/extension
+        run: pnpm run vscode:package
+      - name: Upload extension to Marketplace
+        working-directory: vscode/extension
+        run: |
+          pnpx vsce publish --packagePath sqlmesh-${{ github.event.inputs.version }}.vsix
+        env:
+          VSCE_PAT: ${{ secrets.VSCE_PAT }}
+      - name: Upload extension to OpenVSX
+        working-directory: vscode/extension
+        run: |
+          pnpx ovsx publish -p ${{ secrets.OPEN_VSX_TOKEN }} sqlmesh-${{ github.event.inputs.version }}.vsix