sqlmesh 0.211.3.dev8__tar.gz → 0.211.3.dev10__tar.gz

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.211.3.dev8
+Version: 0.211.3.dev10
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.211.3.dev8'
-__version_tuple__ = version_tuple = (0, 211, 3, 'dev8')
+__version__ = version = '0.211.3.dev10'
+__version_tuple__ = version_tuple = (0, 211, 3, 'dev10')
 
-__commit_id__ = commit_id = 'gbba2e3ac9'
+__commit_id__ = commit_id = 'g4c94ee13d'

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/sqlmesh/core/snapshot/evaluator.py

@@ -2300,13 +2300,19 @@ class ViewStrategy(PromotableStrategy):
         render_kwargs: t.Dict[str, t.Any],
         **kwargs: t.Any,
     ) -> None:
-        snapshot = kwargs["snapshot"]
+        # We should recreate MVs across supported engines (Snowflake, BigQuery etc) because
+        # if upstream tables were recreated (e.g FULL models), the MVs would be silently invalidated.
+        # The only exception to that rule is RisingWave which doesn't support CREATE OR REPLACE, so upstream
+        # models don't recreate their physical tables for the MVs to be invalidated.
+        # However, even for RW we still want to recreate MVs to avoid stale references, as is the case with normal views.
+        # The flag is_first_insert is used for that matter as a signal to recreate the MV if the snapshot's intervals
+        # have been cleared by `should_force_rebuild`
+        is_materialized_view = self._is_materialized_view(model)
+        must_recreate_view = not self.adapter.HAS_VIEW_BINDING or (
+            is_materialized_view and is_first_insert
+        )
 
-        if (
-            not snapshot.is_materialized_view
-            and self.adapter.HAS_VIEW_BINDING
-            and self.adapter.table_exists(table_name)
-        ):
+        if self.adapter.table_exists(table_name) and not must_recreate_view:
             logger.info("Skipping creation of the view '%s'", table_name)
             return
 
@@ -2315,8 +2321,8 @@ class ViewStrategy(PromotableStrategy):
             table_name,
             query_or_df,
             model.columns_to_types,
-            replace=not self.adapter.HAS_VIEW_BINDING,
-            materialized=self._is_materialized_view(model),
+            replace=must_recreate_view,
+            materialized=is_materialized_view,
             view_properties=kwargs.get("physical_properties", model.physical_properties),
             table_description=model.description,
             column_descriptions=model.column_descriptions,

sqlmesh-0.211.3.dev10/sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py

@@ -0,0 +1,48 @@
+"""
+Warns dbt users about potential diffs due to corrected data_type handling.
+
+SQLMesh previously treated dbt's schema.yml data_type field as columns_to_types, which
+doesn't match dbt's behavior. dbt only uses data_type for contracts/validation, not DDL.
+This fix may cause diffs if tables were created with incorrect types.
+
+More context: https://github.com/TobikoData/sqlmesh/pull/5231
+"""
+
+import json
+
+from sqlglot import exp
+
+from sqlmesh.core.console import get_console
+
+SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    warning = (
+        "SQLMesh previously misinterpreted dbt's schema.yml 'data_type' field as actual "
+        "column types, but dbt only uses these for contracts/validation, not in actual "
+        "DDL statements. This has been fixed to match dbt's actual behavior. Your existing "
+        "tables may have been created with incorrect column types. After this migration, run "
+        "'sqlmesh diff prod' to check for column type differences, and if any are found, "
+        "apply a plan to correct the table schemas. For more details, see: "
+        "https://github.com/TobikoData/sqlmesh/pull/5231."
+    )
+
+    for (snapshot,) in engine_adapter.fetchall(
+        exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+
+        jinja_macros = node.get("jinja_macros") or {}
+        create_builtins_module = jinja_macros.get("create_builtins_module") or ""
+
+        if create_builtins_module == SQLMESH_DBT_PACKAGE and node.get("columns"):
+            get_console().log_warning(warning)
+            return

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.211.3.dev8
+Version: 0.211.3.dev10
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/sqlmesh.egg-info/SOURCES.txt

@@ -467,6 +467,7 @@ sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py
 sqlmesh/migrations/v0089_add_virtual_environment_mode.py
 sqlmesh/migrations/v0090_add_forward_only_column.py
 sqlmesh/migrations/v0091_on_additive_change.py
+sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py
 sqlmesh/schedulers/__init__.py
 sqlmesh/utils/__init__.py
 sqlmesh/utils/aws.py

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/tests/core/engine_adapter/integration/test_integration.py

@@ -7,7 +7,10 @@ import sys
 import typing as t
 import shutil
 from datetime import datetime, timedelta, date
+from unittest import mock
 from unittest.mock import patch
+import logging
+
 import numpy as np  # noqa: TID253
 import pandas as pd  # noqa: TID253
 import pytest
@@ -3748,3 +3751,65 @@ def test_janitor(
         "incremental_model",
         "seed_model",
     ]
+
+
+def test_materialized_view_evaluation(ctx: TestContext, mocker: MockerFixture):
+    adapter = ctx.engine_adapter
+    dialect = ctx.dialect
+
+    if not adapter.SUPPORTS_MATERIALIZED_VIEWS:
+        pytest.skip(f"Skipping engine {dialect} as it does not support materialized views")
+    elif dialect in ("snowflake", "databricks"):
+        pytest.skip(f"Skipping {dialect} as they're not enabled on standard accounts")
+
+    model_name = ctx.table("test_tbl")
+    mview_name = ctx.table("test_mview")
+
+    sqlmesh = ctx.create_context()
+
+    sqlmesh.upsert_model(
+        load_sql_based_model(
+            d.parse(
+                f"""
+                MODEL (name {model_name}, kind FULL);
+
+                SELECT 1 AS col
+                """
+            )
+        )
+    )
+
+    sqlmesh.upsert_model(
+        load_sql_based_model(
+            d.parse(
+                f"""
+                MODEL (name {mview_name}, kind VIEW (materialized true));
+
+                SELECT * FROM {model_name}
+                """
+            )
+        )
+    )
+
+    def _assert_mview_value(value: int):
+        df = adapter.fetchdf(f"SELECT * FROM {mview_name.sql(dialect=dialect)}")
+        assert df["col"][0] == value
+
+    # Case 1: Ensure that plan is successful and we can query the materialized view
+    sqlmesh.plan(auto_apply=True, no_prompts=True)
+
+    _assert_mview_value(value=1)
+
+    # Case 2: Ensure that we can change the underlying table and the materialized view is recreated
+    sqlmesh.upsert_model(
+        load_sql_based_model(d.parse(f"""MODEL (name {model_name}, kind FULL); SELECT 2 AS col"""))
+    )
+
+    logger = logging.getLogger("sqlmesh.core.snapshot.evaluator")
+
+    with mock.patch.object(logger, "info") as mock_logger:
+        sqlmesh.plan(auto_apply=True, no_prompts=True)
+
+        assert any("Replacing view" in call[0][0] for call in mock_logger.call_args_list)
+
+    _assert_mview_value(value=2)

{sqlmesh-0.211.3.dev8 → sqlmesh-0.211.3.dev10}/tests/core/engine_adapter/integration/test_integration_bigquery.py

@@ -441,53 +441,6 @@ def test_table_diff_table_name_matches_column_name(ctx: TestContext):
     assert row_diff.full_match_count == 1
 
 
-def test_materialized_view_evaluation(ctx: TestContext, engine_adapter: BigQueryEngineAdapter):
-    model_name = ctx.table("test_tbl")
-    mview_name = ctx.table("test_mview")
-
-    sqlmesh = ctx.create_context()
-
-    sqlmesh.upsert_model(
-        load_sql_based_model(
-            d.parse(
-                f"""
-                MODEL (name {model_name}, kind FULL);
-
-                SELECT 1 AS col
-                """
-            )
-        )
-    )
-
-    sqlmesh.upsert_model(
-        load_sql_based_model(
-            d.parse(
-                f"""
-                MODEL (name {mview_name}, kind VIEW (materialized true));
-
-                SELECT * FROM {model_name}
-                """
-            )
-        )
-    )
-
-    # Case 1: Ensure that plan is successful and we can query the materialized view
-    sqlmesh.plan(auto_apply=True, no_prompts=True)
-
-    df = engine_adapter.fetchdf(f"SELECT * FROM {mview_name.sql(dialect=ctx.dialect)}")
-    assert df["col"][0] == 1
-
-    # Case 2: Ensure that we can change the underlying table and the materialized view is recreated
-    sqlmesh.upsert_model(
-        load_sql_based_model(d.parse(f"""MODEL (name {model_name}, kind FULL); SELECT 2 AS col"""))
-    )
-
-    sqlmesh.plan(auto_apply=True, no_prompts=True)
-
-    df = engine_adapter.fetchdf(f"SELECT * FROM {mview_name.sql(dialect=ctx.dialect)}")
-    assert df["col"][0] == 2
-
-
 def test_correlation_id_in_job_labels(ctx: TestContext):
     model_name = ctx.table("test")