sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev20__py3-none-any.whl

sqlmesh/core/engine_adapter/mixins.py

@@ -7,9 +7,10 @@ from dataclasses import dataclass
 
 from sqlglot import exp, parse_one
 from sqlglot.helper import seq_get
+from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 
 from sqlmesh.core.engine_adapter.base import EngineAdapter
-from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, SourceQuery
+from sqlmesh.core.engine_adapter.shared import DataObjectType
 from sqlmesh.core.node import IntervalUnit
 from sqlmesh.core.dialect import schema_
 from sqlmesh.core.schema_diff import TableAlterOperation
@@ -17,7 +18,12 @@ from sqlmesh.utils.errors import SQLMeshError
 
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import TableName
-    from sqlmesh.core.engine_adapter._typing import DF
+    from sqlmesh.core.engine_adapter._typing import (
+        DCL,
+        DF,
+        GrantsConfig,
+        QueryOrDF,
+    )
     from sqlmesh.core.engine_adapter.base import QueryOrDF
 
 logger = logging.getLogger(__name__)
@@ -75,52 +81,6 @@ class PandasNativeFetchDFSupportMixin(EngineAdapter):
         return df
 
 
-class InsertOverwriteWithMergeMixin(EngineAdapter):
-    def _insert_overwrite_by_condition(
-        self,
-        table_name: TableName,
-        source_queries: t.List[SourceQuery],
-        target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
-        where: t.Optional[exp.Condition] = None,
-        insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
-        **kwargs: t.Any,
-    ) -> None:
-        """
-        Some engines do not support `INSERT OVERWRITE` but instead support
-        doing an "INSERT OVERWRITE" using a Merge expression but with the
-        predicate being `False`.
-        """
-        target_columns_to_types = target_columns_to_types or self.columns(table_name)
-        for source_query in source_queries:
-            with source_query as query:
-                query = self._order_projections_and_filter(
-                    query, target_columns_to_types, where=where
-                )
-                columns = [exp.column(col) for col in target_columns_to_types]
-                when_not_matched_by_source = exp.When(
-                    matched=False,
-                    source=True,
-                    condition=where,
-                    then=exp.Delete(),
-                )
-                when_not_matched_by_target = exp.When(
-                    matched=False,
-                    source=False,
-                    then=exp.Insert(
-                        this=exp.Tuple(expressions=columns),
-                        expression=exp.Tuple(expressions=columns),
-                    ),
-                )
-                self._merge(
-                    target_table=table_name,
-                    query=query,
-                    on=exp.false(),
-                    whens=exp.Whens(
-                        expressions=[when_not_matched_by_source, when_not_matched_by_target]
-                    ),
-                )
-
-
 class HiveMetastoreTablePropertiesMixin(EngineAdapter):
     MAX_TABLE_COMMENT_LENGTH = 4000
     MAX_COLUMN_COMMENT_LENGTH = 4000
@@ -595,3 +555,137 @@ class RowDiffMixin(EngineAdapter):
 
     def _normalize_boolean_value(self, expr: exp.Expression) -> exp.Expression:
         return exp.cast(expr, "INT")
+
+
+class GrantsFromInfoSchemaMixin(EngineAdapter):
+    CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("current_user")
+    SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = False
+    USE_CATALOG_IN_GRANTS = False
+    GRANT_INFORMATION_SCHEMA_TABLE_NAME = "table_privileges"
+
+    @staticmethod
+    @abc.abstractmethod
+    def _grant_object_kind(table_type: DataObjectType) -> t.Optional[str]:
+        pass
+
+    @abc.abstractmethod
+    def _get_current_schema(self) -> str:
+        pass
+
+    def _dcl_grants_config_expr(
+        self,
+        dcl_cmd: t.Type[DCL],
+        table: exp.Table,
+        grants_config: GrantsConfig,
+        table_type: DataObjectType = DataObjectType.TABLE,
+    ) -> t.List[exp.Expression]:
+        expressions: t.List[exp.Expression] = []
+        if not grants_config:
+            return expressions
+
+        object_kind = self._grant_object_kind(table_type)
+        for privilege, principals in grants_config.items():
+            args: t.Dict[str, t.Any] = {
+                "privileges": [exp.GrantPrivilege(this=exp.Var(this=privilege))],
+                "securable": table.copy(),
+            }
+            if object_kind:
+                args["kind"] = exp.Var(this=object_kind)
+            if self.SUPPORTS_MULTIPLE_GRANT_PRINCIPALS:
+                args["principals"] = [
+                    normalize_identifiers(
+                        parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
+                        dialect=self.dialect,
+                    )
+                    for principal in principals
+                ]
+                expressions.append(dcl_cmd(**args))  # type: ignore[arg-type]
+            else:
+                for principal in principals:
+                    args["principals"] = [
+                        normalize_identifiers(
+                            parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
+                            dialect=self.dialect,
+                        )
+                    ]
+                    expressions.append(dcl_cmd(**args))  # type: ignore[arg-type]
+
+        return expressions
+
+    def _apply_grants_config_expr(
+        self,
+        table: exp.Table,
+        grants_config: GrantsConfig,
+        table_type: DataObjectType = DataObjectType.TABLE,
+    ) -> t.List[exp.Expression]:
+        return self._dcl_grants_config_expr(exp.Grant, table, grants_config, table_type)
+
+    def _revoke_grants_config_expr(
+        self,
+        table: exp.Table,
+        grants_config: GrantsConfig,
+        table_type: DataObjectType = DataObjectType.TABLE,
+    ) -> t.List[exp.Expression]:
+        return self._dcl_grants_config_expr(exp.Revoke, table, grants_config, table_type)
+
+    def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
+        schema_identifier = table.args.get("db") or normalize_identifiers(
+            exp.to_identifier(self._get_current_schema(), quoted=True), dialect=self.dialect
+        )
+        schema_name = schema_identifier.this
+        table_name = table.args.get("this").this  # type: ignore
+
+        grant_conditions = [
+            exp.column("table_schema").eq(exp.Literal.string(schema_name)),
+            exp.column("table_name").eq(exp.Literal.string(table_name)),
+            exp.column("grantor").eq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
+            exp.column("grantee").neq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
+        ]
+
+        info_schema_table = normalize_identifiers(
+            exp.table_(self.GRANT_INFORMATION_SCHEMA_TABLE_NAME, db="information_schema"),
+            dialect=self.dialect,
+        )
+        if self.USE_CATALOG_IN_GRANTS:
+            catalog_identifier = table.args.get("catalog")
+            if not catalog_identifier:
+                catalog_name = self.get_current_catalog()
+                if not catalog_name:
+                    raise SQLMeshError(
+                        "Current catalog could not be determined for fetching grants. This is unexpected."
+                    )
+                catalog_identifier = normalize_identifiers(
+                    exp.to_identifier(catalog_name, quoted=True), dialect=self.dialect
+                )
+            catalog_name = catalog_identifier.this
+            info_schema_table.set("catalog", catalog_identifier.copy())
+            grant_conditions.insert(
+                0, exp.column("table_catalog").eq(exp.Literal.string(catalog_name))
+            )
+
+        return (
+            exp.select("privilege_type", "grantee")
+            .from_(info_schema_table)
+            .where(exp.and_(*grant_conditions))
+        )
+
+    def _get_current_grants_config(self, table: exp.Table) -> GrantsConfig:
+        grant_expr = self._get_grant_expression(table)
+
+        results = self.fetchall(grant_expr)
+
+        grants_dict: GrantsConfig = {}
+        for privilege_raw, grantee_raw in results:
+            if privilege_raw is None or grantee_raw is None:
+                continue
+
+            privilege = str(privilege_raw)
+            grantee = str(grantee_raw)
+            if not privilege or not grantee:
+                continue
+
+            grantees = grants_dict.setdefault(privilege, [])
+            if grantee not in grantees:
+                grantees.append(grantee)
+
+        return grants_dict

sqlmesh/core/engine_adapter/mssql.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import typing as t
+import logging
 
 from sqlglot import exp
 
@@ -13,10 +14,10 @@ from sqlmesh.core.engine_adapter.base import (
     InsertOverwriteStrategy,
     MERGE_SOURCE_ALIAS,
     MERGE_TARGET_ALIAS,
+    _get_data_object_cache_key,
 )
 from sqlmesh.core.engine_adapter.mixins import (
     GetCurrentCatalogFromFunctionMixin,
-    InsertOverwriteWithMergeMixin,
     PandasNativeFetchDFSupportMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
@@ -37,11 +38,13 @@ if t.TYPE_CHECKING:
     from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF
 
 
+logger = logging.getLogger(__name__)
+
+
 @set_catalog()
 class MSSQLEngineAdapter(
     EngineAdapterWithIndexSupport,
     PandasNativeFetchDFSupportMixin,
-    InsertOverwriteWithMergeMixin,
     GetCurrentCatalogFromFunctionMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
@@ -53,6 +56,7 @@ class MSSQLEngineAdapter(
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    MAX_IDENTIFIER_LENGTH = 128
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
@@ -74,6 +78,7 @@ class MSSQLEngineAdapter(
         },
     }
     VARIABLE_LENGTH_DATA_TYPES = {"binary", "varbinary", "char", "varchar", "nchar", "nvarchar"}
+    INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.MERGE
 
     @property
     def catalog_support(self) -> CatalogSupport:
@@ -145,6 +150,10 @@ class MSSQLEngineAdapter(
     def table_exists(self, table_name: TableName) -> bool:
         """MsSql doesn't support describe so we query information_schema."""
         table = exp.to_table(table_name)
+        data_object_cache_key = _get_data_object_cache_key(table.catalog, table.db, table.name)
+        if data_object_cache_key in self._data_object_cache:
+            logger.debug("Table existence cache hit: %s", data_object_cache_key)
+            return self._data_object_cache[data_object_cache_key] is not None
 
         sql = (
             exp.select("1")
@@ -414,7 +423,9 @@ class MSSQLEngineAdapter(
         insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
         **kwargs: t.Any,
     ) -> None:
-        if not where or where == exp.true():
+        # note that this is passed as table_properties here rather than physical_properties
+        use_merge_strategy = kwargs.get("table_properties", {}).get("mssql_merge_exists")
+        if (not where or where == exp.true()) and not use_merge_strategy:
             # this is a full table replacement, call the base strategy to do DELETE+INSERT
             # which will result in TRUNCATE+INSERT due to how we have overridden self.delete_from()
             return EngineAdapter._insert_overwrite_by_condition(
@@ -427,7 +438,7 @@ class MSSQLEngineAdapter(
                 **kwargs,
             )
 
-        # For actual conditional overwrites, use MERGE from InsertOverwriteWithMergeMixin
+        # For conditional overwrites or when mssql_merge_exists is set use MERGE
         return super()._insert_overwrite_by_condition(
             table_name=table_name,
             source_queries=source_queries,

sqlmesh/core/engine_adapter/mysql.py

@@ -164,11 +164,11 @@ class MySQLEngineAdapter(
                         exc_info=True,
                     )
 
-    def create_table_like(
+    def _create_table_like(
         self,
         target_table_name: TableName,
         source_table_name: TableName,
-        exists: bool = True,
+        exists: bool,
         **kwargs: t.Any,
     ) -> None:
         self.execute(

sqlmesh/core/engine_adapter/postgres.py

@@ -12,6 +12,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     PandasNativeFetchDFSupportMixin,
     RowDiffMixin,
     logical_merge,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import set_catalog
 
@@ -28,14 +29,19 @@ class PostgresEngineAdapter(
     PandasNativeFetchDFSupportMixin,
     GetCurrentCatalogFromFunctionMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 ):
     DIALECT = "postgres"
+    SUPPORTS_GRANTS = True
     SUPPORTS_INDEXES = True
     HAS_VIEW_BINDING = True
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
-    MAX_IDENTIFIER_LENGTH = 63
+    MAX_IDENTIFIER_LENGTH: t.Optional[int] = 63
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
+    GRANT_INFORMATION_SCHEMA_TABLE_NAME = "role_table_grants"
+    CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.column("current_role")
+    SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"
@@ -79,11 +85,11 @@ class PostgresEngineAdapter(
             self._connection_pool.commit()
         return df
 
-    def create_table_like(
+    def _create_table_like(
         self,
         target_table_name: TableName,
         source_table_name: TableName,
-        exists: bool = True,
+        exists: bool,
         **kwargs: t.Any,
     ) -> None:
         self.execute(

sqlmesh/core/engine_adapter/redshift.py

@@ -14,6 +14,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
     logical_merge,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import (
     CommentCreationView,
@@ -40,12 +41,15 @@ class RedshiftEngineAdapter(
     NonTransactionalTruncateMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 ):
     DIALECT = "redshift"
     CURRENT_CATALOG_EXPRESSION = exp.func("current_database")
     # Redshift doesn't support comments for VIEWs WITH NO SCHEMA BINDING (which we always use)
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    SUPPORTS_GRANTS = True
+    SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
 
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {

sqlmesh/core/engine_adapter/risingwave.py

@@ -32,6 +32,7 @@ class RisingwaveEngineAdapter(PostgresEngineAdapter):
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_TRANSACTIONS = False
     MAX_IDENTIFIER_LENGTH = None
+    SUPPORTS_GRANTS = False
 
     def columns(
         self, table_name: TableName, include_pseudo_columns: bool = False

sqlmesh/core/engine_adapter/shared.py

@@ -243,6 +243,8 @@ class InsertOverwriteStrategy(Enum):
     # Issue a single INSERT query to replace a data range. The assumption is that the query engine will transparently match partition bounds
     # and replace data rather than append to it. Trino is an example of this when `hive.insert-existing-partitions-behavior=OVERWRITE` is configured
     INTO_IS_OVERWRITE = 4
+    # Do the INSERT OVERWRITE using merge since the engine doesn't support it natively
+    MERGE = 5
 
     @property
     def is_delete_insert(self) -> bool:
@@ -260,6 +262,10 @@ class InsertOverwriteStrategy(Enum):
     def is_into_is_overwrite(self) -> bool:
         return self == InsertOverwriteStrategy.INTO_IS_OVERWRITE
 
+    @property
+    def is_merge(self) -> bool:
+        return self == InsertOverwriteStrategy.MERGE
+
 
 class SourceQuery:
     def __init__(

sqlmesh/core/engine_adapter/snowflake.py

@@ -15,6 +15,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     GetCurrentCatalogFromFunctionMixin,
     ClusteredByMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import (
     CatalogSupport,
@@ -34,7 +35,12 @@ if t.TYPE_CHECKING:
     import pandas as pd
 
     from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
-    from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF, SnowparkSession
+    from sqlmesh.core.engine_adapter._typing import (
+        DF,
+        Query,
+        QueryOrDF,
+        SnowparkSession,
+    )
     from sqlmesh.core.node import IntervalUnit
 
 
@@ -46,7 +52,9 @@ if t.TYPE_CHECKING:
         "drop_catalog": CatalogSupport.REQUIRES_SET_CATALOG,  # needs a catalog to issue a query to information_schema.databases even though the result is global
     }
 )
-class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixin, RowDiffMixin):
+class SnowflakeEngineAdapter(
+    GetCurrentCatalogFromFunctionMixin, ClusteredByMixin, RowDiffMixin, GrantsFromInfoSchemaMixin
+):
     DIALECT = "snowflake"
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True
@@ -54,6 +62,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     SUPPORTS_MANAGED_MODELS = True
     CURRENT_CATALOG_EXPRESSION = exp.func("current_database")
     SUPPORTS_CREATE_DROP_CATALOG = True
+    SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["DATABASE", "SCHEMA", "TABLE"]
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
@@ -73,6 +82,9 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     MANAGED_TABLE_KIND = "DYNAMIC TABLE"
     SNOWPARK = "snowpark"
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
+    SUPPORTS_GRANTS = True
+    CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("CURRENT_ROLE")
+    USE_CATALOG_IN_GRANTS = True
 
     @contextlib.contextmanager
     def session(self, properties: SessionProperties) -> t.Iterator[None]:
@@ -127,6 +139,23 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     def catalog_support(self) -> CatalogSupport:
         return CatalogSupport.FULL_SUPPORT
 
+    @staticmethod
+    def _grant_object_kind(table_type: DataObjectType) -> str:
+        if table_type == DataObjectType.VIEW:
+            return "VIEW"
+        if table_type == DataObjectType.MATERIALIZED_VIEW:
+            return "MATERIALIZED VIEW"
+        if table_type == DataObjectType.MANAGED_TABLE:
+            return "DYNAMIC TABLE"
+        return "TABLE"
+
+    def _get_current_schema(self) -> str:
+        """Returns the current default schema for the connection."""
+        result = self.fetchone("SELECT CURRENT_SCHEMA()")
+        if not result or not result[0]:
+            raise SQLMeshError("Unable to determine current schema")
+        return str(result[0])
+
     def _create_catalog(self, catalog_name: exp.Identifier) -> None:
         props = exp.Properties(
             expressions=[exp.SchemaCommentProperty(this=exp.Literal.string(c.SQLMESH_MANAGED))]
@@ -378,6 +407,8 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             elif isinstance(df, pd.DataFrame):
                 from snowflake.connector.pandas_tools import write_pandas
 
+                ordered_df = df[list(source_columns_to_types)]
+
                 # Workaround for https://github.com/snowflakedb/snowflake-connector-python/issues/1034
                 # The above issue has already been fixed upstream, but we keep the following
                 # line anyway in order to support a wider range of Snowflake versions.
@@ -388,16 +419,16 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
                 # See: https://stackoverflow.com/a/75627721
                 for column, kind in source_columns_to_types.items():
-                    if is_datetime64_any_dtype(df.dtypes[column]):
+                    if is_datetime64_any_dtype(ordered_df.dtypes[column]):
                         if kind.is_type("date"):  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.date  # type: ignore
-                        elif getattr(df.dtypes[column], "tz", None) is not None:  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.strftime(
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.date  # type: ignore
+                        elif getattr(ordered_df.dtypes[column], "tz", None) is not None:  # type: ignore
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.strftime(
                                 "%Y-%m-%d %H:%M:%S.%f%z"
                             )  # type: ignore
                         # https://github.com/snowflakedb/snowflake-connector-python/issues/1677
                         else:  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.strftime(
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.strftime(
                                 "%Y-%m-%d %H:%M:%S.%f"
                             )  # type: ignore
 
@@ -407,7 +438,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
                 write_pandas(
                     self._connection_pool.get(),
-                    df,
+                    ordered_df,
                     temp_table.name,
                     schema=temp_table.db or None,
                     database=database.sql(dialect=self.dialect) if database else None,
@@ -526,16 +557,36 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
                 type=DataObjectType.from_str(row.type),  # type: ignore
                 clustering_key=row.clustering_key,  # type: ignore
             )
-            for row in df.itertuples()
+            # lowercase the column names for cases where Snowflake might return uppercase column names for certain catalogs
+            for row in df.rename(columns={col: col.lower() for col in df.columns}).itertuples()
         ]
 
+    def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
+        # Upon execute the catalog in table expressions are properly normalized to handle the case where a user provides
+        # the default catalog in their connection config. This doesn't though update catalogs in strings like when querying
+        # the information schema. So we need to manually replace those here.
+        expression = super()._get_grant_expression(table)
+        for col_exp in expression.find_all(exp.Column):
+            if col_exp.this.name == "table_catalog":
+                and_exp = col_exp.parent
+                assert and_exp is not None, "Expected column expression to have a parent"
+                assert and_exp.expression, "Expected AND expression to have an expression"
+                normalized_catalog = self._normalize_catalog(
+                    exp.table_("placeholder", db="placeholder", catalog=and_exp.expression.this)
+                )
+                and_exp.set(
+                    "expression",
+                    exp.Literal.string(normalized_catalog.args["catalog"].alias_or_name),
+                )
+        return expression
+
     def set_current_catalog(self, catalog: str) -> None:
         self.execute(exp.Use(this=exp.to_identifier(catalog)))
 
     def set_current_schema(self, schema: str) -> None:
         self.execute(exp.Use(kind="SCHEMA", this=to_schema(schema)))
 
-    def _to_sql(self, expression: exp.Expression, quote: bool = True, **kwargs: t.Any) -> str:
+    def _normalize_catalog(self, expression: exp.Expression) -> exp.Expression:
         # note: important to use self._default_catalog instead of the self.default_catalog property
         # otherwise we get RecursionError: maximum recursion depth exceeded
         # because it calls get_current_catalog(), which executes a query, which needs the default catalog, which calls get_current_catalog()... etc
@@ -568,8 +619,12 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             # Snowflake connection config. This is because the catalog present on the model gets normalized and quoted to match
             # the source dialect, which isnt always compatible with Snowflake
             expression = expression.transform(catalog_rewriter)
+        return expression
 
-        return super()._to_sql(expression=expression, quote=quote, **kwargs)
+    def _to_sql(self, expression: exp.Expression, quote: bool = True, **kwargs: t.Any) -> str:
+        return super()._to_sql(
+            expression=self._normalize_catalog(expression), quote=quote, **kwargs
+        )
 
     def _create_column_comments(
         self,
@@ -610,6 +665,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         target_table_name: TableName,
         source_table_name: TableName,
         replace: bool = False,
+        exists: bool = True,
         clone_kwargs: t.Optional[t.Dict[str, t.Any]] = None,
         **kwargs: t.Any,
     ) -> None:
@@ -665,3 +721,18 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             self._connection_pool.set_attribute(self.SNOWPARK, None)
 
         return super().close()
+
+    def get_table_last_modified_ts(self, table_names: t.List[TableName]) -> t.List[int]:
+        from sqlmesh.utils.date import to_timestamp
+
+        num_tables = len(table_names)
+
+        query = "SELECT LAST_ALTERED FROM INFORMATION_SCHEMA.TABLES WHERE"
+        for i, table_name in enumerate(table_names):
+            table = exp.to_table(table_name)
+            query += f"""(TABLE_NAME = '{table.name}' AND TABLE_SCHEMA = '{table.db}' AND TABLE_CATALOG = '{table.catalog}')"""
+            if i < num_tables - 1:
+                query += " OR "
+
+        result = self.fetchall(query)
+        return [to_timestamp(row[0]) for row in result]

sqlmesh/core/engine_adapter/spark.py

@@ -397,19 +397,21 @@ class SparkEngineAdapter(
     def set_current_catalog(self, catalog_name: str) -> None:
         self.connection.set_current_catalog(catalog_name)
 
-    def get_current_database(self) -> str:
+    def _get_current_schema(self) -> str:
         if self._use_spark_session:
             return self.spark.catalog.currentDatabase()
         return self.fetchone(exp.select(exp.func("current_database")))[0]  # type: ignore
 
-    def get_data_object(self, target_name: TableName) -> t.Optional[DataObject]:
+    def get_data_object(
+        self, target_name: TableName, safe_to_cache: bool = False
+    ) -> t.Optional[DataObject]:
         target_table = exp.to_table(target_name)
         if isinstance(target_table.this, exp.Dot) and target_table.this.expression.name.startswith(
             f"{self.BRANCH_PREFIX}{self.WAP_PREFIX}"
         ):
             # Exclude the branch name
             target_table.set("this", target_table.this.this)
-        return super().get_data_object(target_table)
+        return super().get_data_object(target_table, safe_to_cache=safe_to_cache)
 
     def create_state_table(
         self,
@@ -457,12 +459,14 @@ class SparkEngineAdapter(
             if wap_id.startswith(f"{self.BRANCH_PREFIX}{self.WAP_PREFIX}"):
                 table_name.set("this", table_name.this.this)
 
-        wap_supported = (
-            kwargs.get("storage_format") or ""
-        ).lower() == "iceberg" or self.wap_supported(table_name)
-        do_dummy_insert = (
-            False if not wap_supported or not exists else not self.table_exists(table_name)
-        )
+        do_dummy_insert = False
+        if self.wap_enabled:
+            wap_supported = (
+                kwargs.get("storage_format") or ""
+            ).lower() == "iceberg" or self.wap_supported(table_name)
+            do_dummy_insert = (
+                False if not wap_supported or not exists else not self.table_exists(table_name)
+            )
         super()._create_table(
             table_name_or_schema,
             expression,
@@ -535,7 +539,7 @@ class SparkEngineAdapter(
         if not table.catalog:
             table.set("catalog", self.get_current_catalog())
         if not table.db:
-            table.set("db", self.get_current_database())
+            table.set("db", self._get_current_schema())
         return table
 
     def _build_create_comment_column_exp(