sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/core/engine_adapter/mssql.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import typing as t
+import logging
 
 from sqlglot import exp
 
@@ -13,10 +14,10 @@ from sqlmesh.core.engine_adapter.base import (
     InsertOverwriteStrategy,
     MERGE_SOURCE_ALIAS,
     MERGE_TARGET_ALIAS,
+    _get_data_object_cache_key,
 )
 from sqlmesh.core.engine_adapter.mixins import (
     GetCurrentCatalogFromFunctionMixin,
-    InsertOverwriteWithMergeMixin,
     PandasNativeFetchDFSupportMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
@@ -37,11 +38,13 @@ if t.TYPE_CHECKING:
     from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF
 
 
+logger = logging.getLogger(__name__)
+
+
 @set_catalog()
 class MSSQLEngineAdapter(
     EngineAdapterWithIndexSupport,
     PandasNativeFetchDFSupportMixin,
-    InsertOverwriteWithMergeMixin,
     GetCurrentCatalogFromFunctionMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
@@ -53,6 +56,7 @@ class MSSQLEngineAdapter(
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    MAX_IDENTIFIER_LENGTH = 128
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
@@ -74,6 +78,7 @@ class MSSQLEngineAdapter(
         },
     }
     VARIABLE_LENGTH_DATA_TYPES = {"binary", "varbinary", "char", "varchar", "nchar", "nvarchar"}
+    INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.MERGE
 
     @property
     def catalog_support(self) -> CatalogSupport:
@@ -145,6 +150,10 @@ class MSSQLEngineAdapter(
     def table_exists(self, table_name: TableName) -> bool:
         """MsSql doesn't support describe so we query information_schema."""
         table = exp.to_table(table_name)
+        data_object_cache_key = _get_data_object_cache_key(table.catalog, table.db, table.name)
+        if data_object_cache_key in self._data_object_cache:
+            logger.debug("Table existence cache hit: %s", data_object_cache_key)
+            return self._data_object_cache[data_object_cache_key] is not None
 
         sql = (
             exp.select("1")
@@ -414,7 +423,9 @@ class MSSQLEngineAdapter(
         insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
         **kwargs: t.Any,
     ) -> None:
-        if not where or where == exp.true():
+        # note that this is passed as table_properties here rather than physical_properties
+        use_merge_strategy = kwargs.get("table_properties", {}).get("mssql_merge_exists")
+        if (not where or where == exp.true()) and not use_merge_strategy:
             # this is a full table replacement, call the base strategy to do DELETE+INSERT
             # which will result in TRUNCATE+INSERT due to how we have overridden self.delete_from()
             return EngineAdapter._insert_overwrite_by_condition(
@@ -427,7 +438,7 @@ class MSSQLEngineAdapter(
                 **kwargs,
             )
 
-        # For actual conditional overwrites, use MERGE from InsertOverwriteWithMergeMixin
+        # For conditional overwrites or when mssql_merge_exists is set use MERGE
         return super()._insert_overwrite_by_condition(
             table_name=table_name,
             source_queries=source_queries,

sqlmesh/core/engine_adapter/mysql.py

@@ -164,11 +164,11 @@ class MySQLEngineAdapter(
                         exc_info=True,
                     )
 
-    def create_table_like(
+    def _create_table_like(
         self,
         target_table_name: TableName,
         source_table_name: TableName,
-        exists: bool = True,
+        exists: bool,
         **kwargs: t.Any,
     ) -> None:
         self.execute(

sqlmesh/core/engine_adapter/postgres.py

@@ -12,6 +12,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     PandasNativeFetchDFSupportMixin,
     RowDiffMixin,
     logical_merge,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import set_catalog
 
@@ -28,14 +29,19 @@ class PostgresEngineAdapter(
     PandasNativeFetchDFSupportMixin,
     GetCurrentCatalogFromFunctionMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 ):
     DIALECT = "postgres"
+    SUPPORTS_GRANTS = True
     SUPPORTS_INDEXES = True
     HAS_VIEW_BINDING = True
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
-    MAX_IDENTIFIER_LENGTH = 63
+    MAX_IDENTIFIER_LENGTH: t.Optional[int] = 63
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
+    GRANT_INFORMATION_SCHEMA_TABLE_NAME = "role_table_grants"
+    CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.column("current_role")
+    SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"
@@ -79,11 +85,11 @@ class PostgresEngineAdapter(
             self._connection_pool.commit()
         return df
 
-    def create_table_like(
+    def _create_table_like(
         self,
         target_table_name: TableName,
         source_table_name: TableName,
-        exists: bool = True,
+        exists: bool,
         **kwargs: t.Any,
     ) -> None:
         self.execute(

sqlmesh/core/engine_adapter/redshift.py

@@ -14,6 +14,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
     logical_merge,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import (
     CommentCreationView,
@@ -40,12 +41,15 @@ class RedshiftEngineAdapter(
     NonTransactionalTruncateMixin,
     VarcharSizeWorkaroundMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 ):
     DIALECT = "redshift"
     CURRENT_CATALOG_EXPRESSION = exp.func("current_database")
     # Redshift doesn't support comments for VIEWs WITH NO SCHEMA BINDING (which we always use)
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    SUPPORTS_GRANTS = True
+    SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
 
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {

sqlmesh/core/engine_adapter/risingwave.py

@@ -32,6 +32,7 @@ class RisingwaveEngineAdapter(PostgresEngineAdapter):
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_TRANSACTIONS = False
     MAX_IDENTIFIER_LENGTH = None
+    SUPPORTS_GRANTS = False
 
     def columns(
         self, table_name: TableName, include_pseudo_columns: bool = False

sqlmesh/core/engine_adapter/shared.py

@@ -243,6 +243,8 @@ class InsertOverwriteStrategy(Enum):
     # Issue a single INSERT query to replace a data range. The assumption is that the query engine will transparently match partition bounds
     # and replace data rather than append to it. Trino is an example of this when `hive.insert-existing-partitions-behavior=OVERWRITE` is configured
     INTO_IS_OVERWRITE = 4
+    # Do the INSERT OVERWRITE using merge since the engine doesn't support it natively
+    MERGE = 5
 
     @property
     def is_delete_insert(self) -> bool:
@@ -260,6 +262,10 @@ class InsertOverwriteStrategy(Enum):
     def is_into_is_overwrite(self) -> bool:
         return self == InsertOverwriteStrategy.INTO_IS_OVERWRITE
 
+    @property
+    def is_merge(self) -> bool:
+        return self == InsertOverwriteStrategy.MERGE
+
 
 class SourceQuery:
     def __init__(

sqlmesh/core/engine_adapter/snowflake.py

@@ -15,6 +15,7 @@ from sqlmesh.core.engine_adapter.mixins import (
     GetCurrentCatalogFromFunctionMixin,
     ClusteredByMixin,
     RowDiffMixin,
+    GrantsFromInfoSchemaMixin,
 )
 from sqlmesh.core.engine_adapter.shared import (
     CatalogSupport,
@@ -34,7 +35,12 @@ if t.TYPE_CHECKING:
     import pandas as pd
 
     from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
-    from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF, SnowparkSession
+    from sqlmesh.core.engine_adapter._typing import (
+        DF,
+        Query,
+        QueryOrDF,
+        SnowparkSession,
+    )
     from sqlmesh.core.node import IntervalUnit
 
 
@@ -46,7 +52,9 @@ if t.TYPE_CHECKING:
         "drop_catalog": CatalogSupport.REQUIRES_SET_CATALOG,  # needs a catalog to issue a query to information_schema.databases even though the result is global
     }
 )
-class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixin, RowDiffMixin):
+class SnowflakeEngineAdapter(
+    GetCurrentCatalogFromFunctionMixin, ClusteredByMixin, RowDiffMixin, GrantsFromInfoSchemaMixin
+):
     DIALECT = "snowflake"
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True
@@ -54,6 +62,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     SUPPORTS_MANAGED_MODELS = True
     CURRENT_CATALOG_EXPRESSION = exp.func("current_database")
     SUPPORTS_CREATE_DROP_CATALOG = True
+    SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["DATABASE", "SCHEMA", "TABLE"]
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
@@ -73,6 +82,9 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     MANAGED_TABLE_KIND = "DYNAMIC TABLE"
     SNOWPARK = "snowpark"
     SUPPORTS_QUERY_EXECUTION_TRACKING = True
+    SUPPORTS_GRANTS = True
+    CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("CURRENT_ROLE")
+    USE_CATALOG_IN_GRANTS = True
 
     @contextlib.contextmanager
     def session(self, properties: SessionProperties) -> t.Iterator[None]:
@@ -127,6 +139,23 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     def catalog_support(self) -> CatalogSupport:
         return CatalogSupport.FULL_SUPPORT
 
+    @staticmethod
+    def _grant_object_kind(table_type: DataObjectType) -> str:
+        if table_type == DataObjectType.VIEW:
+            return "VIEW"
+        if table_type == DataObjectType.MATERIALIZED_VIEW:
+            return "MATERIALIZED VIEW"
+        if table_type == DataObjectType.MANAGED_TABLE:
+            return "DYNAMIC TABLE"
+        return "TABLE"
+
+    def _get_current_schema(self) -> str:
+        """Returns the current default schema for the connection."""
+        result = self.fetchone("SELECT CURRENT_SCHEMA()")
+        if not result or not result[0]:
+            raise SQLMeshError("Unable to determine current schema")
+        return str(result[0])
+
     def _create_catalog(self, catalog_name: exp.Identifier) -> None:
         props = exp.Properties(
             expressions=[exp.SchemaCommentProperty(this=exp.Literal.string(c.SQLMESH_MANAGED))]
@@ -378,6 +407,8 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             elif isinstance(df, pd.DataFrame):
                 from snowflake.connector.pandas_tools import write_pandas
 
+                ordered_df = df[list(source_columns_to_types)]
+
                 # Workaround for https://github.com/snowflakedb/snowflake-connector-python/issues/1034
                 # The above issue has already been fixed upstream, but we keep the following
                 # line anyway in order to support a wider range of Snowflake versions.
@@ -388,16 +419,16 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
                 # See: https://stackoverflow.com/a/75627721
                 for column, kind in source_columns_to_types.items():
-                    if is_datetime64_any_dtype(df.dtypes[column]):
+                    if is_datetime64_any_dtype(ordered_df.dtypes[column]):
                         if kind.is_type("date"):  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.date  # type: ignore
-                        elif getattr(df.dtypes[column], "tz", None) is not None:  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.strftime(
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.date  # type: ignore
+                        elif getattr(ordered_df.dtypes[column], "tz", None) is not None:  # type: ignore
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.strftime(
                                 "%Y-%m-%d %H:%M:%S.%f%z"
                             )  # type: ignore
                         # https://github.com/snowflakedb/snowflake-connector-python/issues/1677
                         else:  # type: ignore
-                            df[column] = pd.to_datetime(df[column]).dt.strftime(
+                            ordered_df[column] = pd.to_datetime(ordered_df[column]).dt.strftime(
                                 "%Y-%m-%d %H:%M:%S.%f"
                             )  # type: ignore
 
@@ -407,7 +438,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
                 write_pandas(
                     self._connection_pool.get(),
-                    df,
+                    ordered_df,
                     temp_table.name,
                     schema=temp_table.db or None,
                     database=database.sql(dialect=self.dialect) if database else None,
@@ -526,16 +557,36 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
                 type=DataObjectType.from_str(row.type),  # type: ignore
                 clustering_key=row.clustering_key,  # type: ignore
             )
-            for row in df.itertuples()
+            # lowercase the column names for cases where Snowflake might return uppercase column names for certain catalogs
+            for row in df.rename(columns={col: col.lower() for col in df.columns}).itertuples()
         ]
 
+    def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
+        # Upon execute the catalog in table expressions are properly normalized to handle the case where a user provides
+        # the default catalog in their connection config. This doesn't though update catalogs in strings like when querying
+        # the information schema. So we need to manually replace those here.
+        expression = super()._get_grant_expression(table)
+        for col_exp in expression.find_all(exp.Column):
+            if col_exp.this.name == "table_catalog":
+                and_exp = col_exp.parent
+                assert and_exp is not None, "Expected column expression to have a parent"
+                assert and_exp.expression, "Expected AND expression to have an expression"
+                normalized_catalog = self._normalize_catalog(
+                    exp.table_("placeholder", db="placeholder", catalog=and_exp.expression.this)
+                )
+                and_exp.set(
+                    "expression",
+                    exp.Literal.string(normalized_catalog.args["catalog"].alias_or_name),
+                )
+        return expression
+
     def set_current_catalog(self, catalog: str) -> None:
         self.execute(exp.Use(this=exp.to_identifier(catalog)))
 
     def set_current_schema(self, schema: str) -> None:
         self.execute(exp.Use(kind="SCHEMA", this=to_schema(schema)))
 
-    def _to_sql(self, expression: exp.Expression, quote: bool = True, **kwargs: t.Any) -> str:
+    def _normalize_catalog(self, expression: exp.Expression) -> exp.Expression:
         # note: important to use self._default_catalog instead of the self.default_catalog property
         # otherwise we get RecursionError: maximum recursion depth exceeded
         # because it calls get_current_catalog(), which executes a query, which needs the default catalog, which calls get_current_catalog()... etc
@@ -568,8 +619,12 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             # Snowflake connection config. This is because the catalog present on the model gets normalized and quoted to match
             # the source dialect, which isnt always compatible with Snowflake
             expression = expression.transform(catalog_rewriter)
+        return expression
 
-        return super()._to_sql(expression=expression, quote=quote, **kwargs)
+    def _to_sql(self, expression: exp.Expression, quote: bool = True, **kwargs: t.Any) -> str:
+        return super()._to_sql(
+            expression=self._normalize_catalog(expression), quote=quote, **kwargs
+        )
 
     def _create_column_comments(
         self,
@@ -610,6 +665,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         target_table_name: TableName,
         source_table_name: TableName,
         replace: bool = False,
+        exists: bool = True,
         clone_kwargs: t.Optional[t.Dict[str, t.Any]] = None,
         **kwargs: t.Any,
     ) -> None:
@@ -665,3 +721,18 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             self._connection_pool.set_attribute(self.SNOWPARK, None)
 
         return super().close()
+
+    def get_table_last_modified_ts(self, table_names: t.List[TableName]) -> t.List[int]:
+        from sqlmesh.utils.date import to_timestamp
+
+        num_tables = len(table_names)
+
+        query = "SELECT LAST_ALTERED FROM INFORMATION_SCHEMA.TABLES WHERE"
+        for i, table_name in enumerate(table_names):
+            table = exp.to_table(table_name)
+            query += f"""(TABLE_NAME = '{table.name}' AND TABLE_SCHEMA = '{table.db}' AND TABLE_CATALOG = '{table.catalog}')"""
+            if i < num_tables - 1:
+                query += " OR "
+
+        result = self.fetchall(query)
+        return [to_timestamp(row[0]) for row in result]

sqlmesh/core/engine_adapter/spark.py

@@ -397,19 +397,21 @@ class SparkEngineAdapter(
     def set_current_catalog(self, catalog_name: str) -> None:
         self.connection.set_current_catalog(catalog_name)
 
-    def get_current_database(self) -> str:
+    def _get_current_schema(self) -> str:
         if self._use_spark_session:
             return self.spark.catalog.currentDatabase()
         return self.fetchone(exp.select(exp.func("current_database")))[0]  # type: ignore
 
-    def get_data_object(self, target_name: TableName) -> t.Optional[DataObject]:
+    def get_data_object(
+        self, target_name: TableName, safe_to_cache: bool = False
+    ) -> t.Optional[DataObject]:
         target_table = exp.to_table(target_name)
         if isinstance(target_table.this, exp.Dot) and target_table.this.expression.name.startswith(
             f"{self.BRANCH_PREFIX}{self.WAP_PREFIX}"
         ):
             # Exclude the branch name
             target_table.set("this", target_table.this.this)
-        return super().get_data_object(target_table)
+        return super().get_data_object(target_table, safe_to_cache=safe_to_cache)
 
     def create_state_table(
         self,
@@ -457,12 +459,14 @@ class SparkEngineAdapter(
             if wap_id.startswith(f"{self.BRANCH_PREFIX}{self.WAP_PREFIX}"):
                 table_name.set("this", table_name.this.this)
 
-        wap_supported = (
-            kwargs.get("storage_format") or ""
-        ).lower() == "iceberg" or self.wap_supported(table_name)
-        do_dummy_insert = (
-            False if not wap_supported or not exists else not self.table_exists(table_name)
-        )
+        do_dummy_insert = False
+        if self.wap_enabled:
+            wap_supported = (
+                kwargs.get("storage_format") or ""
+            ).lower() == "iceberg" or self.wap_supported(table_name)
+            do_dummy_insert = (
+                False if not wap_supported or not exists else not self.table_exists(table_name)
+            )
         super()._create_table(
             table_name_or_schema,
             expression,
@@ -535,7 +539,7 @@ class SparkEngineAdapter(
         if not table.catalog:
             table.set("catalog", self.get_current_catalog())
         if not table.db:
-            table.set("db", self.get_current_database())
+            table.set("db", self._get_current_schema())
         return table
 
     def _build_create_comment_column_exp(

sqlmesh/core/engine_adapter/trino.py

@@ -71,7 +71,7 @@ class TrinoEngineAdapter(
     MAX_TIMESTAMP_PRECISION = 3
 
     @property
-    def schema_location_mapping(self) -> t.Optional[dict[re.Pattern, str]]:
+    def schema_location_mapping(self) -> t.Optional[t.Dict[re.Pattern, str]]:
         return self._extra_config.get("schema_location_mapping")
 
     @property
@@ -86,6 +86,8 @@ class TrinoEngineAdapter(
     def get_catalog_type(self, catalog: t.Optional[str]) -> str:
         row: t.Tuple = tuple()
         if catalog:
+            if catalog_type_override := self._catalog_type_overrides.get(catalog):
+                return catalog_type_override
             row = (
                 self.fetchone(
                     f"select connector_name from system.metadata.catalogs where catalog_name='{catalog}'"
@@ -300,7 +302,7 @@ class TrinoEngineAdapter(
         execution_time: t.Union[TimeLike, exp.Column],
         invalidate_hard_deletes: bool = True,
         updated_at_col: t.Optional[exp.Column] = None,
-        check_columns: t.Optional[t.Union[exp.Star, t.Sequence[exp.Column]]] = None,
+        check_columns: t.Optional[t.Union[exp.Star, t.Sequence[exp.Expression]]] = None,
         updated_at_as_valid_from: bool = False,
         execution_time_as_valid_from: bool = False,
         target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,

sqlmesh/core/janitor.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import typing as t
+
+from sqlglot import exp
+
+from sqlmesh.core.engine_adapter import EngineAdapter
+from sqlmesh.core.console import Console
+from sqlmesh.core.dialect import schema_
+from sqlmesh.core.environment import Environment
+from sqlmesh.core.snapshot import SnapshotEvaluator
+from sqlmesh.core.state_sync import StateSync
+from sqlmesh.core.state_sync.common import (
+    logger,
+    iter_expired_snapshot_batches,
+    RowBoundary,
+    ExpiredBatchRange,
+)
+from sqlmesh.utils.errors import SQLMeshError
+
+
+def cleanup_expired_views(
+    default_adapter: EngineAdapter,
+    engine_adapters: t.Dict[str, EngineAdapter],
+    environments: t.List[Environment],
+    warn_on_delete_failure: bool = False,
+    console: t.Optional[Console] = None,
+) -> None:
+    expired_schema_or_catalog_environments = [
+        environment
+        for environment in environments
+        if environment.suffix_target.is_schema or environment.suffix_target.is_catalog
+    ]
+    expired_table_environments = [
+        environment for environment in environments if environment.suffix_target.is_table
+    ]
+
+    # We have to use the corresponding adapter if the virtual layer is gateway managed
+    def get_adapter(gateway_managed: bool, gateway: t.Optional[str] = None) -> EngineAdapter:
+        if gateway_managed and gateway:
+            return engine_adapters.get(gateway, default_adapter)
+        return default_adapter
+
+    catalogs_to_drop: t.Set[t.Tuple[EngineAdapter, str]] = set()
+    schemas_to_drop: t.Set[t.Tuple[EngineAdapter, exp.Table]] = set()
+
+    # Collect schemas and catalogs to drop
+    for engine_adapter, expired_catalog, expired_schema, suffix_target in {
+        (
+            (engine_adapter := get_adapter(environment.gateway_managed, snapshot.model_gateway)),
+            snapshot.qualified_view_name.catalog_for_environment(
+                environment.naming_info, dialect=engine_adapter.dialect
+            ),
+            snapshot.qualified_view_name.schema_for_environment(
+                environment.naming_info, dialect=engine_adapter.dialect
+            ),
+            environment.suffix_target,
+        )
+        for environment in expired_schema_or_catalog_environments
+        for snapshot in environment.snapshots
+        if snapshot.is_model and not snapshot.is_symbolic
+    }:
+        if suffix_target.is_catalog:
+            if expired_catalog:
+                catalogs_to_drop.add((engine_adapter, expired_catalog))
+        else:
+            schema = schema_(expired_schema, expired_catalog)
+            schemas_to_drop.add((engine_adapter, schema))
+
+    # Drop the views for the expired environments
+    for engine_adapter, expired_view in {
+        (
+            (engine_adapter := get_adapter(environment.gateway_managed, snapshot.model_gateway)),
+            snapshot.qualified_view_name.for_environment(
+                environment.naming_info, dialect=engine_adapter.dialect
+            ),
+        )
+        for environment in expired_table_environments
+        for snapshot in environment.snapshots
+        if snapshot.is_model and not snapshot.is_symbolic
+    }:
+        try:
+            engine_adapter.drop_view(expired_view, ignore_if_not_exists=True)
+            if console:
+                console.update_cleanup_progress(expired_view)
+        except Exception as e:
+            message = f"Failed to drop the expired environment view '{expired_view}': {e}"
+            if warn_on_delete_failure:
+                logger.warning(message)
+            else:
+                raise SQLMeshError(message) from e
+
+    # Drop the schemas for the expired environments
+    for engine_adapter, schema in schemas_to_drop:
+        try:
+            engine_adapter.drop_schema(
+                schema,
+                ignore_if_not_exists=True,
+                cascade=True,
+            )
+            if console:
+                console.update_cleanup_progress(schema.sql(dialect=engine_adapter.dialect))
+        except Exception as e:
+            message = f"Failed to drop the expired environment schema '{schema}': {e}"
+            if warn_on_delete_failure:
+                logger.warning(message)
+            else:
+                raise SQLMeshError(message) from e
+
+    # Drop any catalogs that were associated with a snapshot where the engine adapter supports dropping catalogs
+    # catalogs_to_drop is only populated when environment_suffix_target is set to 'catalog'
+    for engine_adapter, catalog in catalogs_to_drop:
+        if engine_adapter.SUPPORTS_CREATE_DROP_CATALOG:
+            try:
+                engine_adapter.drop_catalog(catalog)
+                if console:
+                    console.update_cleanup_progress(catalog)
+            except Exception as e:
+                message = f"Failed to drop the expired environment catalog '{catalog}': {e}"
+                if warn_on_delete_failure:
+                    logger.warning(message)
+                else:
+                    raise SQLMeshError(message) from e
+
+
+def delete_expired_snapshots(
+    state_sync: StateSync,
+    snapshot_evaluator: SnapshotEvaluator,
+    *,
+    current_ts: int,
+    ignore_ttl: bool = False,
+    batch_size: t.Optional[int] = None,
+    console: t.Optional[Console] = None,
+) -> None:
+    """Delete all expired snapshots in batches.
+
+    This helper function encapsulates the logic for deleting expired snapshots in batches,
+    eliminating code duplication across different use cases.
+
+    Args:
+        state_sync: StateSync instance to query and delete expired snapshots from.
+        snapshot_evaluator: SnapshotEvaluator instance to clean up tables associated with snapshots.
+        current_ts: Timestamp used to evaluate expiration.
+        ignore_ttl: If True, include snapshots regardless of TTL (only checks if unreferenced).
+        batch_size: Maximum number of snapshots to fetch per batch.
+        console: Optional console for reporting progress.
+
+    Returns:
+        The total number of deleted expired snapshots.
+    """
+    num_expired_snapshots = 0
+    for batch in iter_expired_snapshot_batches(
+        state_reader=state_sync,
+        current_ts=current_ts,
+        ignore_ttl=ignore_ttl,
+        batch_size=batch_size,
+    ):
+        end_info = (
+            f"updated_ts={batch.batch_range.end.updated_ts}"
+            if isinstance(batch.batch_range.end, RowBoundary)
+            else f"limit={batch.batch_range.end.batch_size}"
+        )
+        logger.info(
+            "Processing batch of size %s with end %s",
+            len(batch.expired_snapshot_ids),
+            end_info,
+        )
+        snapshot_evaluator.cleanup(
+            target_snapshots=batch.cleanup_tasks,
+            on_complete=console.update_cleanup_progress if console else None,
+        )
+        state_sync.delete_expired_snapshots(
+            batch_range=ExpiredBatchRange(
+                start=RowBoundary.lowest_boundary(),
+                end=batch.batch_range.end,
+            ),
+            ignore_ttl=ignore_ttl,
+        )
+        logger.info("Cleaned up expired snapshots batch")
+        num_expired_snapshots += len(batch.expired_snapshot_ids)
+    logger.info("Cleaned up %s expired snapshots", num_expired_snapshots)

sqlmesh/core/lineage.py

@@ -66,6 +66,7 @@ def lineage(
         scope=scope,
         trim_selects=trim_selects,
         dialect=model.dialect,
+        copy=False,
     )