sqlmesh 0.207.1.dev6__tar.gz → 0.207.1.dev8__tar.gz

{sqlmesh-0.207.1.dev6/.circleci → sqlmesh-0.207.1.dev8/.github/scripts}/get_scm_version.py

@@ -1,4 +1,4 @@
 from setuptools_scm import get_version
 
-version = get_version(root='../', relative_to=__file__)
+version = get_version(root='../../', relative_to=__file__)
 print(version.split('+')[0])

sqlmesh-0.207.1.dev8/.github/workflows/private-repo-test.yaml

@@ -0,0 +1,96 @@
+name: Private Repo Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: 'private-test-${{ github.event.pull_request.number }}'
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  trigger-private-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Set up Node.js for UI build
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install UI dependencies
+        run: pnpm install
+      - name: Build UI
+        run: pnpm --prefix web/client run build
+      - name: Install Python dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install build twine setuptools_scm
+      - name: Generate development version
+        id: version
+        run: |
+          source .venv/bin/activate
+          # Generate a PEP 440 compliant unique version including run attempt
+          BASE_VERSION=$(python .github/scripts/get_scm_version.py)
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          # Use PEP 440 compliant format: base.devN+pr.sha.attempt
+          UNIQUE_VERSION="${BASE_VERSION}+pr${{ github.event.pull_request.number }}.${COMMIT_SHA}.run${{ github.run_attempt }}"
+          echo "version=$UNIQUE_VERSION" >> $GITHUB_OUTPUT
+          echo "Generated unique version with run attempt: $UNIQUE_VERSION"
+      - name: Build package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Configure PyPI for private repository
+        env:
+          TOBIKO_PRIVATE_PYPI_URL: ${{ secrets.TOBIKO_PRIVATE_PYPI_URL }}
+          TOBIKO_PRIVATE_PYPI_KEY: ${{ secrets.TOBIKO_PRIVATE_PYPI_KEY }}
+        run: ./.circleci/update-pypirc.sh
+      - name: Publish to private PyPI
+        run: |
+          source .venv/bin/activate
+          python -m twine upload -r tobiko-private dist/*
+      - name: Publish Python Tests package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          source .venv/bin/activate
+          unset TWINE_USERNAME TWINE_PASSWORD && make publish-tests
+      - name: Get GitHub App token
+        id: get_token
+        uses: actions/create-github-app-token@v1
+        with:
+          private-key: ${{ secrets.TOBIKO_RENOVATE_BOT_PRIVATE_KEY }}
+          app-id: ${{ secrets.TOBIKO_RENOVATE_BOT_APP_ID }}
+          owner: ${{ secrets.PRIVATE_REPO_OWNER }}
+      - name: Trigger private repository workflow
+        uses: convictional/trigger-workflow-and-wait@v1.6.5
+        with:
+          owner: ${{ secrets.PRIVATE_REPO_OWNER }}
+          repo: ${{ secrets.PRIVATE_REPO_NAME }}
+          github_token: ${{ steps.get_token.outputs.token }}
+          workflow_file_name: ${{ secrets.PRIVATE_WORKFLOW_FILE }}
+          client_payload: |
+            {
+              "package_version": "${{ steps.version.outputs.version }}",
+              "pr_number": "${{ github.event.pull_request.number }}"
+            }

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.207.1.dev6
+Version: 0.207.1.dev8
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.207.1.dev6'
-__version_tuple__ = version_tuple = (0, 207, 1, 'dev6')
+__version__ = version = '0.207.1.dev8'
+__version_tuple__ = version_tuple = (0, 207, 1, 'dev8')

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh/core/plan/stages.py

@@ -8,7 +8,6 @@ from sqlmesh.core.scheduler import merged_missing_intervals, SnapshotToIntervals
 from sqlmesh.core.snapshot.definition import (
     DeployabilityIndex,
     Snapshot,
-    SnapshotChangeCategory,
     SnapshotTableInfo,
     SnapshotId,
     Interval,
@@ -251,18 +250,7 @@ class PlanStagesBuilder:
             deployability_index = DeployabilityIndex.all_deployable()
 
             snapshots_with_schema_migration = [
-                s
-                for s in snapshots.values()
-                if s.is_paused
-                and s.is_model
-                and not s.is_symbolic
-                and (
-                    not deployability_index_for_creation.is_representative(s)
-                    or (
-                        s.is_view
-                        and s.change_category == SnapshotChangeCategory.INDIRECT_NON_BREAKING
-                    )
-                )
+                s for s in snapshots.values() if s.requires_schema_migration_in_prod
             ]
 
         snapshots_to_intervals = self._missing_intervals(

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh/core/snapshot/definition.py

@@ -1352,6 +1352,21 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             check_categorical_relative_expression=False,
         )
 
+    @property
+    def requires_schema_migration_in_prod(self) -> bool:
+        """Returns whether or not this snapshot requires a schema migration when deployed to production."""
+        return (
+            self.is_paused
+            and self.is_model
+            and not self.is_symbolic
+            and (
+                (self.previous_version and self.previous_version.version == self.version)
+                or self.model.forward_only
+                or bool(self.model.physical_version)
+                or self.is_view
+            )
+        )
+
     @property
     def ttl_ms(self) -> int:
         return self.expiration_ts - self.updated_ts

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh/core/snapshot/evaluator.py

@@ -918,11 +918,7 @@ class SnapshotEvaluator:
         adapter: EngineAdapter,
         deployability_index: DeployabilityIndex,
     ) -> None:
-        if (
-            not snapshot.is_paused
-            or not snapshot.is_model
-            or (deployability_index.is_representative(snapshot) and not snapshot.is_view)
-        ):
+        if not snapshot.requires_schema_migration_in_prod:
             return
 
         deployability_index = DeployabilityIndex.all_deployable()

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.207.1.dev6
+Version: 0.207.1.dev8
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/sqlmesh.egg-info/SOURCES.txt

@@ -16,7 +16,6 @@ pnpm-workspace.yaml
 pyproject.toml
 .circleci/config.yml
 .circleci/continue_config.yml
-.circleci/get_scm_version.py
 .circleci/install-prerequisites.sh
 .circleci/manage-test-db.sh
 .circleci/test_migration.sh
@@ -26,7 +25,9 @@ pyproject.toml
 .claude/agents/developer.md
 .claude/agents/technical-writer.md
 .github/dependabot.yml
+.github/scripts/get_scm_version.py
 .github/workflows/pr.yaml
+.github/workflows/private-repo-test.yaml
 .github/workflows/release_extension.yaml
 benchmarks/lsp_render_model_bench.py
 examples/__init__.py

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/tests/core/state_sync/test_state_sync.py

@@ -2089,7 +2089,7 @@ def test_version_schema(state_sync: EngineAdapterStateSync, tmp_path) -> None:
 
     with pytest.raises(
         SQLMeshError,
-        match=rf"SQLMesh \(local\) is using version '{SQLMESH_VERSION}' which is ahead of '0.0.0' \(remote\). Please run a migration \('sqlmesh migrate' command\).",
+        match=rf"SQLMesh \(local\) is using version '{re.escape(SQLMESH_VERSION)}' which is ahead of '0.0.0' \(remote\). Please run a migration \('sqlmesh migrate' command\).",
     ):
         state_sync.get_versions()
 
@@ -2099,7 +2099,7 @@ def test_version_schema(state_sync: EngineAdapterStateSync, tmp_path) -> None:
     state_sync.version_state.update_versions(schema_version=SCHEMA_VERSION + 1)
     error = (
         rf"SQLMesh \(local\) is using version '{SCHEMA_VERSION}' which is behind '{SCHEMA_VERSION + 1}' \(remote\). "
-        rf"""Please upgrade SQLMesh \('pip install --upgrade "sqlmesh=={SQLMESH_VERSION}"' command\)."""
+        rf"""Please upgrade SQLMesh \('pip install --upgrade "sqlmesh=={re.escape(SQLMESH_VERSION)}"' command\)."""
     )
 
     with pytest.raises(SQLMeshError, match=error):
@@ -2136,7 +2136,7 @@ def test_version_sqlmesh(state_sync: EngineAdapterStateSync) -> None:
     # sqlmesh version is behind
     sqlmesh_version_minor_bump = f"{major}.{int(minor) + 1}.{patch}"
     error = (
-        rf"SQLMesh \(local\) is using version '{SQLMESH_VERSION}' which is behind '{sqlmesh_version_minor_bump}' \(remote\). "
+        rf"SQLMesh \(local\) is using version '{re.escape(SQLMESH_VERSION)}' which is behind '{sqlmesh_version_minor_bump}' \(remote\). "
         rf"""Please upgrade SQLMesh \('pip install --upgrade "sqlmesh=={sqlmesh_version_minor_bump}"' command\)."""
     )
     state_sync.version_state.update_versions(sqlmesh_version=sqlmesh_version_minor_bump)
@@ -2146,7 +2146,7 @@ def test_version_sqlmesh(state_sync: EngineAdapterStateSync) -> None:
 
     # sqlmesh version is ahead
     sqlmesh_version_minor_decrease = f"{major}.{int(minor) - 1}.{patch}"
-    error = rf"SQLMesh \(local\) is using version '{SQLMESH_VERSION}' which is ahead of '{sqlmesh_version_minor_decrease}'"
+    error = rf"SQLMesh \(local\) is using version '{re.escape(SQLMESH_VERSION)}' which is ahead of '{sqlmesh_version_minor_decrease}'"
     state_sync.version_state.update_versions(sqlmesh_version=sqlmesh_version_minor_decrease)
     with pytest.raises(SQLMeshError, match=error):
         state_sync.get_versions()

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/tests/core/test_integration.py

@@ -1381,6 +1381,35 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     )
 
 
+@time_machine.travel("2023-01-08 15:00:00 UTC", tick=True)
+def test_metadata_change_after_forward_only_results_in_migration(init_and_plan_context: t.Callable):
+    context, plan = init_and_plan_context("examples/sushi")
+    context.apply(plan)
+
+    # Make a forward-only change
+    model = context.get_model("sushi.waiter_revenue_by_day")
+    model = model.copy(update={"kind": model.kind.copy(update={"forward_only": True})})
+    model = add_projection_to_model(t.cast(SqlModel, model))
+    context.upsert_model(model)
+    plan = context.plan("dev", skip_tests=True, auto_apply=True, no_prompts=True)
+    assert len(plan.new_snapshots) == 2
+    assert all(s.change_category == SnapshotChangeCategory.FORWARD_ONLY for s in plan.new_snapshots)
+
+    # Follow-up with a metadata change in the same environment
+    model = model.copy(update={"owner": "new_owner"})
+    context.upsert_model(model)
+    plan = context.plan("dev", skip_tests=True, auto_apply=True, no_prompts=True)
+    assert len(plan.new_snapshots) == 2
+    assert all(s.change_category == SnapshotChangeCategory.METADATA for s in plan.new_snapshots)
+
+    # Deploy the latest change to prod
+    context.plan("prod", skip_tests=True, auto_apply=True, no_prompts=True)
+
+    # Check that the new column was added in prod
+    columns = context.engine_adapter.columns("sushi.waiter_revenue_by_day")
+    assert "one" in columns
+
+
 @time_machine.travel("2023-01-08 15:00:00 UTC")
 def test_forward_only_precedence_over_indirect_non_breaking(init_and_plan_context: t.Callable):
     context, plan = init_and_plan_context("examples/sushi")

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/tests/core/test_plan_stages.py

@@ -1211,93 +1211,6 @@ def test_build_plan_stages_environment_suffix_target_changed(
     )
 
 
-def test_build_plan_stages_indirect_non_breaking_no_migration(
-    snapshot_a: Snapshot, snapshot_b: Snapshot, make_snapshot, mocker: MockerFixture
-) -> None:
-    # Categorize snapshot_a as forward-only
-    new_snapshot_a = make_snapshot(
-        snapshot_a.model.copy(update={"stamp": "new_version"}),
-    )
-    new_snapshot_a.previous_versions = snapshot_a.all_versions
-    new_snapshot_a.categorize_as(SnapshotChangeCategory.NON_BREAKING)
-
-    new_snapshot_b = make_snapshot(
-        snapshot_b.model.copy(),
-        nodes={'"a"': new_snapshot_a.model},
-    )
-    new_snapshot_b.previous_versions = snapshot_b.all_versions
-    new_snapshot_b.change_category = SnapshotChangeCategory.INDIRECT_NON_BREAKING
-    new_snapshot_b.version = new_snapshot_b.previous_version.data_version.version
-
-    state_reader = mocker.Mock(spec=StateReader)
-    state_reader.get_snapshots.return_value = {}
-    existing_environment = Environment(
-        name="prod",
-        snapshots=[snapshot_a.table_info, snapshot_b.table_info],
-        start_at="2023-01-01",
-        end_at="2023-01-02",
-        plan_id="previous_plan",
-        previous_plan_id=None,
-        promoted_snapshot_ids=[snapshot_a.snapshot_id, snapshot_b.snapshot_id],
-        finalized_ts=to_timestamp("2023-01-02"),
-    )
-    state_reader.get_environment.return_value = existing_environment
-
-    # Create environment
-    environment = Environment(
-        name="prod",
-        snapshots=[new_snapshot_a.table_info, new_snapshot_b.table_info],
-        start_at="2023-01-01",
-        end_at="2023-01-02",
-        plan_id="test_plan",
-        previous_plan_id="previous_plan",
-        promoted_snapshot_ids=[new_snapshot_a.snapshot_id, new_snapshot_b.snapshot_id],
-    )
-
-    # Create evaluatable plan
-    plan = EvaluatablePlan(
-        start="2023-01-01",
-        end="2023-01-02",
-        new_snapshots=[new_snapshot_a, new_snapshot_b],
-        environment=environment,
-        no_gaps=False,
-        skip_backfill=False,
-        empty_backfill=False,
-        restatements={},
-        is_dev=False,
-        allow_destructive_models=set(),
-        forward_only=False,
-        end_bounded=False,
-        ensure_finalized_snapshots=False,
-        directly_modified_snapshots=[new_snapshot_a.snapshot_id],
-        indirectly_modified_snapshots={
-            new_snapshot_a.name: [new_snapshot_b.snapshot_id],
-        },
-        metadata_updated_snapshots=[],
-        removed_snapshots=[],
-        requires_backfill=True,
-        models_to_backfill=None,
-        execution_time="2023-01-02",
-        disabled_restatement_models=set(),
-        environment_statements=None,
-        user_provided_flags=None,
-    )
-
-    # Build plan stages
-    stages = build_plan_stages(plan, state_reader, None)
-
-    # Verify stages
-    assert len(stages) == 7
-
-    assert isinstance(stages[0], CreateSnapshotRecordsStage)
-    assert isinstance(stages[1], PhysicalLayerUpdateStage)
-    assert isinstance(stages[2], BackfillStage)
-    assert isinstance(stages[3], EnvironmentRecordUpdateStage)
-    assert isinstance(stages[4], UnpauseStage)
-    assert isinstance(stages[5], VirtualLayerUpdateStage)
-    assert isinstance(stages[6], FinalizeEnvironmentStage)
-
-
 def test_build_plan_stages_indirect_non_breaking_view_migration(
     snapshot_a: Snapshot, snapshot_c: Snapshot, make_snapshot, mocker: MockerFixture
 ) -> None:

{sqlmesh-0.207.1.dev6 → sqlmesh-0.207.1.dev8}/tests/core/test_snapshot_evaluator.py

@@ -1180,6 +1180,7 @@ def test_migrate(mocker: MockerFixture, make_snapshot):
     )
     snapshot = make_snapshot(model, version="1")
     snapshot.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot.previous_versions = snapshot.all_versions
 
     evaluator.migrate([snapshot], {}, deployability_index=DeployabilityIndex.none_deployable())
 
@@ -1217,6 +1218,7 @@ def test_migrate_missing_table(mocker: MockerFixture, make_snapshot):
     )
     snapshot = make_snapshot(model, version="1")
     snapshot.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot.previous_versions = snapshot.all_versions
 
     evaluator.migrate([snapshot], {}, deployability_index=DeployabilityIndex.none_deployable())
 
@@ -1714,6 +1716,7 @@ def test_on_destructive_change_runtime_check(
     )
     snapshot = make_snapshot(model, version="1")
     snapshot.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot.previous_versions = snapshot.all_versions
 
     with pytest.raises(NodeExecutionFailedError) as ex:
         evaluator.migrate([snapshot], {}, deployability_index=DeployabilityIndex.none_deployable())
@@ -1735,6 +1738,7 @@ def test_on_destructive_change_runtime_check(
     )
     snapshot = make_snapshot(model, version="1")
     snapshot.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot.previous_versions = snapshot.all_versions
 
     logger = logging.getLogger("sqlmesh.core.snapshot.evaluator")
     with patch.object(logger, "warning") as mock_logger:
@@ -3654,6 +3658,7 @@ def test_migrate_snapshot(snapshot: Snapshot, mocker: MockerFixture, adapter_moc
 
     new_snapshot = make_snapshot(updated_model)
     new_snapshot.categorize_as(SnapshotChangeCategory.FORWARD_ONLY)
+    new_snapshot.previous_versions = snapshot.all_versions
     new_snapshot.version = snapshot.version
 
     assert new_snapshot.table_name() == snapshot.table_name()
@@ -3724,6 +3729,7 @@ def test_migrate_managed(adapter_mock, make_snapshot, mocker: MockerFixture):
     )
     snapshot: Snapshot = make_snapshot(model)
     snapshot.categorize_as(SnapshotChangeCategory.FORWARD_ONLY)
+    snapshot.previous_versions = snapshot.all_versions
 
     # no schema changes - no-op
     adapter_mock.get_alter_expressions.return_value = []
@@ -3925,6 +3931,7 @@ def test_multiple_engine_migration(mocker: MockerFixture, adapter_mock, make_sna
     )
     snapshot_1 = make_snapshot(model, version="1")
     snapshot_1.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot_1.previous_versions = snapshot_1.all_versions
     model_2 = SqlModel(
         name="test_schema.test_model_2",
         kind=IncrementalByTimeRangeKind(
@@ -3935,6 +3942,7 @@ def test_multiple_engine_migration(mocker: MockerFixture, adapter_mock, make_sna
     )
     snapshot_2 = make_snapshot(model_2, version="1")
     snapshot_2.change_category = SnapshotChangeCategory.FORWARD_ONLY
+    snapshot_2.previous_versions = snapshot_2.all_versions
     evaluator.migrate(
         [snapshot_1, snapshot_2], {}, deployability_index=DeployabilityIndex.none_deployable()
     )