sqlmesh 0.236.1__tar.gz → 0.236.3__tar.gz

sqlmesh-0.236.3/.devcontainer/devcontainer-lock.json

@@ -0,0 +1,19 @@
+{
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "version": "2.5.9",
+      "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a",
+      "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {
+      "version": "4.0.0",
+      "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:4fa87399214366e320d489991769c4f3f461e1ffe461f54eea78a41b34945bb5",
+      "integrity": "sha256:4fa87399214366e320d489991769c4f3f461e1ffe461f54eea78a41b34945bb5"
+    },
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "1.8.0",
+      "resolved": "ghcr.io/devcontainers/features/python@sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511",
+      "integrity": "sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511"
+    }
+  }
+}

sqlmesh-0.236.3/.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+  "name": "SQLMesh Python Dev",
+  "image": "mcr.microsoft.com/devcontainers/base:ubuntu-24.04",
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {},
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.12"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {}
+  },
+  "postCreateCommand": "bash .devcontainer/post-create-command.sh",
+  "customizations": {
+    "vscode": {
+      "extensions": ["ms-python.python", "ms-python.vscode-pylance"]
+    }
+  },
+  "remoteUser": "vscode"
+}

sqlmesh-0.236.3/.devcontainer/post-create-command.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# This script is intended to be run by an Ubuntu development container
+
+# Exit immediately if any command returns a non-zero code
+set -e
+
+# Install OS-level dependencies
+
+## Ensure that the Microsoft package repository is available as it is required for msodbcsql18. Note that the repository
+## may have already been added by a development container feature (e.g. docker-in-docker or java).
+if apt-cache policy | grep -q 'packages.microsoft.com'; then
+    echo "Microsoft package repository already present"
+else
+    echo "Microsoft package repository not present, it will be added."
+    
+    # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+    curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+    sudo dpkg -i packages-microsoft-prod.deb
+    rm packages-microsoft-prod.deb
+fi
+
+ALL_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev default-jdk msodbcsql18"
+sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
+
+# Install Python dependencies
+make install-dev

sqlmesh-0.236.3/.github/scripts/install-prerequisites.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# This script is intended to be run by an Ubuntu CI build agent
+# The goal is to install OS-level dependencies that are required before trying to install Python dependencies
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "USAGE: $0 <engine>"
+    exit 1
+fi
+
+ENGINE="$1"
+
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev"
+ENGINE_DEPENDENCIES=""
+
+if [ "$ENGINE" == "spark" ]; then
+    ENGINE_DEPENDENCIES="default-jdk"
+elif [ "$ENGINE" == "db2" ]; then
+    ENGINE_DEPENDENCIES="libxml2-dev build-essential"
+elif [ "$ENGINE" == "fabric" ]; then
+    echo "Installing Microsoft package repository"
+
+    # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+    curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+    sudo dpkg -i packages-microsoft-prod.deb
+    rm packages-microsoft-prod.deb
+
+    ENGINE_DEPENDENCIES="msodbcsql18"
+fi
+
+ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"
+
+echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
+
+sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
+
+if [ "$ENGINE" == "spark" ]; then
+    echo "Using Java version for spark:"
+    java -version
+fi
+
+echo "All done"

sqlmesh-0.236.3/.github/scripts/wait-for-db.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# The purpose of this script is to be called after `docker compose up -d` has been run for a given database
+# The idea is to block until the database is available to serve requests. Once the database can serve requests,
+# the integration tests can be run.
+# Therefore, the ports etc are tightly coupled with the compose.yml files under tests/core/engine_adapter/docker/
+#
+# Note that if the docker daemon is not running `localhost`, you can set the DOCKER_HOSTNAME environment variable to the
+# correct host Docker is running on
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "USAGE: $0 <engine>"
+    exit 1
+fi
+
+ENGINE="$1"
+
+function_exists() {
+    declare -f -F $1 > /dev/null
+    return $?
+}
+
+probe_port() {
+    HOSTNAME=${DOCKER_HOSTNAME:-localhost}
+    echo "Probing '$HOSTNAME' on port $1"
+    while ! nc -z $HOSTNAME $1; do
+        sleep 1
+    done
+}
+
+clickhouse_ready() {
+    probe_port 8123
+}
+
+postgres_ready() {
+    probe_port 5432
+}
+
+mssql_ready() {
+    probe_port 1433
+}
+
+mysql_ready() {
+    probe_port 3306
+}
+
+spark_ready() {
+    probe_port 15002
+}
+
+starrocks_ready() {
+    probe_port 9030
+
+    echo "Checking for 1 alive StarRocks backends..."
+    sleep 5
+
+    while true; do
+        echo "Checking StarRocks backends..."
+        ALIVE_BACKENDS=$(docker exec -i starrocks-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" | grep -c "^ *Alive: true *$")
+
+        # fallback value if failed to get number
+        if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then
+            echo "WARN: Unable to parse number of alive backends, got: '$ALIVE_BACKENDS'"
+            ALIVE_BACKENDS=0
+        fi
+
+        echo "Found $ALIVE_BACKENDS alive backends"
+
+        if [ "$ALIVE_BACKENDS" -ge 1 ]; then
+            echo "StarRocks has 1 or more alive backends"
+            break
+        fi
+
+        echo "Waiting for more backends to become alive..."
+        sleep 5
+    done
+
+    # set default replication num to 1 (there is only one be in the docker compose file)
+    docker exec -i starrocks-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET frontend config ('default_replication_num' = '1');"
+}
+
+trino_ready() {
+    # Trino has a built-in healthcheck script, just call that
+    docker compose -f tests/core/engine_adapter/integration/docker/compose.trino.yaml exec trino /bin/bash -c '/usr/lib/trino/bin/health-check'
+}
+
+risingwave_ready() {
+    probe_port 4566
+}
+
+db2_ready() {
+    probe_port 50001
+
+    echo "Waiting for Db2 to finish initialising (this can take 2-4 minutes)..."
+    while true; do
+        if docker exec db2 su - db2inst1 -c "db2 connect to TESTDB" > /dev/null 2>&1; then
+            echo "Db2 is accepting connections"
+            break
+        fi
+        echo "Db2 not yet ready; sleeping 15s..."
+        sleep 15
+    done
+}
+
+echo "Waiting for $ENGINE to be ready..."
+
+READINESS_FUNC="${ENGINE}_ready"
+
+# If called with an unimplemented / unsupported engine, just exit
+if ! function_exists $READINESS_FUNC ; then
+    echo "WARN: $READINESS_FUNC not implemeted; exiting"
+    exit 0
+fi
+
+EXIT_CODE=1
+
+while [ $EXIT_CODE -ne 0 ]; do
+    echo "Checking $ENGINE"
+    $READINESS_FUNC && EXIT_CODE=$? || EXIT_CODE=$?
+    if [ $EXIT_CODE -ne 0 ]; then
+        echo "$ENGINE not ready; sleeping"
+        sleep 5
+    fi
+done
+
+echo "$ENGINE is ready!"

sqlmesh-0.236.3/.github/workflows/db2.yaml

@@ -0,0 +1,65 @@
+name: db2
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'sqlmesh/core/engine_adapter/db2.py'
+      - 'tests/core/engine_adapter/test_db2.py'
+      - 'tests/core/engine_adapter/integration/test_integration_db2.py'
+      - 'tests/core/engine_adapter/integration/docker/compose.db2.yaml'
+      - '.github/workflows/db2.yaml'
+      - 'pyproject.toml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'sqlmesh/core/engine_adapter/db2.py'
+      - 'tests/core/engine_adapter/test_db2.py'
+      - 'tests/core/engine_adapter/integration/test_integration_db2.py'
+      - 'tests/core/engine_adapter/integration/docker/compose.db2.yaml'
+      - '.github/workflows/db2.yaml'
+      - 'pyproject.toml'
+
+concurrency:
+  group: db2-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  db2-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    env:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 2
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: '1'
+      UV: '1'
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install SQLMesh dev dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          make install-dev
+      - name: Install OS-level dependencies
+        run: ./.github/scripts/install-prerequisites.sh "db2"
+      - name: Run tests
+        run: |
+          source .venv/bin/activate
+          make db2-test
+      - name: Upload test results
+        uses: actions/upload-artifact@v7
+        if: ${{ !cancelled() }}
+        with:
+          name: test-results-docker-db2
+          path: test-results/
+          retention-days: 7