sqlmesh 0.210.1.dev2__tar.gz → 0.210.1.dev4__tar.gz

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.210.1.dev2
+Version: 0.210.1.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.210.1.dev2'
-__version_tuple__ = version_tuple = (0, 210, 1, 'dev2')
+__version__ = version = '0.210.1.dev4'
+__version_tuple__ = version_tuple = (0, 210, 1, 'dev4')
 
-__commit_id__ = commit_id = 'g672806a16'
+__commit_id__ = commit_id = 'g4d8e83152'

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/config/connection.py

@@ -100,6 +100,7 @@ class ConnectionConfig(abc.ABC, BaseConfig):
     register_comments: bool
     pre_ping: bool
     pretty_sql: bool = False
+    schema_differ_overrides: t.Optional[t.Dict[str, t.Any]] = None
 
     # Whether to share a  single connection across threads or create a new connection per thread.
     shared_connection: t.ClassVar[bool] = False
@@ -174,6 +175,7 @@ class ConnectionConfig(abc.ABC, BaseConfig):
             pre_ping=self.pre_ping,
             pretty_sql=self.pretty_sql,
             shared_connection=self.shared_connection,
+            schema_differ_overrides=self.schema_differ_overrides,
             **self._extra_engine_config,
         )
 

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/athena.py

@@ -39,7 +39,7 @@ class AthenaEngineAdapter(PandasNativeFetchDFSupportMixin, RowDiffMixin):
     # CTAS, Views: No comment support at all
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
-    SCHEMA_DIFFER = TrinoEngineAdapter.SCHEMA_DIFFER
+    SCHEMA_DIFFER_KWARGS = TrinoEngineAdapter.SCHEMA_DIFFER_KWARGS
     MAX_TIMESTAMP_PRECISION = 3  # copied from Trino
     # Athena does not deal with comments well, e.g:
     # >>> self._execute('/* test */ DESCRIBE foo')

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/base.py

@@ -14,7 +14,7 @@ import itertools
 import logging
 import sys
 import typing as t
-from functools import partial
+from functools import cached_property, partial
 
 from sqlglot import Dialect, exp
 from sqlglot.errors import ErrorLevel
@@ -109,7 +109,7 @@ class EngineAdapter:
     SUPPORTS_MANAGED_MODELS = False
     SUPPORTS_CREATE_DROP_CATALOG = False
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS: t.List[str] = []
-    SCHEMA_DIFFER = SchemaDiffer()
+    SCHEMA_DIFFER_KWARGS: t.Dict[str, t.Any] = {}
     SUPPORTS_TUPLE_IN = True
     HAS_VIEW_BINDING = False
     SUPPORTS_REPLACE_TABLE = True
@@ -132,6 +132,7 @@ class EngineAdapter:
         pretty_sql: bool = False,
         shared_connection: bool = False,
         correlation_id: t.Optional[CorrelationId] = None,
+        schema_differ_overrides: t.Optional[t.Dict[str, t.Any]] = None,
         **kwargs: t.Any,
     ):
         self.dialect = dialect.lower() or self.DIALECT
@@ -154,6 +155,7 @@ class EngineAdapter:
         self._pretty_sql = pretty_sql
         self._multithreaded = multithreaded
         self.correlation_id = correlation_id
+        self._schema_differ_overrides = schema_differ_overrides
 
     def with_settings(self, **kwargs: t.Any) -> EngineAdapter:
         extra_kwargs = {
@@ -204,6 +206,15 @@ class EngineAdapter:
     def catalog_support(self) -> CatalogSupport:
         return CatalogSupport.UNSUPPORTED
 
+    @cached_property
+    def schema_differ(self) -> SchemaDiffer:
+        return SchemaDiffer(
+            **{
+                **self.SCHEMA_DIFFER_KWARGS,
+                **(self._schema_differ_overrides or {}),
+            }
+        )
+
     @classmethod
     def _casted_columns(
         cls,
@@ -1101,7 +1112,7 @@ class EngineAdapter:
         """
         return t.cast(
             t.List[TableAlterOperation],
-            self.SCHEMA_DIFFER.compare_columns(
+            self.schema_differ.compare_columns(
                 current_table_name,
                 self.columns(current_table_name),
                 self.columns(target_table_name),

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/bigquery.py

@@ -22,7 +22,7 @@ from sqlmesh.core.engine_adapter.shared import (
     set_catalog,
 )
 from sqlmesh.core.node import IntervalUnit
-from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation
+from sqlmesh.core.schema_diff import TableAlterOperation, NestedSupport
 from sqlmesh.utils import optional_import, get_source_columns_to_types
 from sqlmesh.utils.date import to_datetime
 from sqlmesh.utils.errors import SQLMeshError
@@ -68,8 +68,8 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     MAX_COLUMN_COMMENT_LENGTH = 1024
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
 
-    SCHEMA_DIFFER = SchemaDiffer(
-        compatible_types={
+    SCHEMA_DIFFER_KWARGS = {
+        "compatible_types": {
             exp.DataType.build("INT64", dialect=DIALECT): {
                 exp.DataType.build("NUMERIC", dialect=DIALECT),
                 exp.DataType.build("FLOAT64", dialect=DIALECT),
@@ -83,17 +83,17 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
                 exp.DataType.build("DATETIME", dialect=DIALECT),
             },
         },
-        coerceable_types={
+        "coerceable_types": {
             exp.DataType.build("FLOAT64", dialect=DIALECT): {
                 exp.DataType.build("BIGNUMERIC", dialect=DIALECT),
             },
         },
-        support_coercing_compatible_types=True,
-        parameterized_type_defaults={
+        "support_coercing_compatible_types": True,
+        "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(38, 9), (0,)],
             exp.DataType.build("BIGDECIMAL", dialect=DIALECT).this: [(76.76, 38), (0,)],
         },
-        types_with_unlimited_length={
+        "types_with_unlimited_length": {
             # parameterized `STRING(n)` can ALTER to unparameterized `STRING`
             exp.DataType.build("STRING", dialect=DIALECT).this: {
                 exp.DataType.build("STRING", dialect=DIALECT).this,
@@ -103,9 +103,8 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
                 exp.DataType.build("BYTES", dialect=DIALECT).this,
             },
         },
-        support_nested_operations=True,
-        support_nested_drop=False,
-    )
+        "nested_support": NestedSupport.ALL_BUT_DROP,
+    }
 
     @property
     def client(self) -> BigQueryClient:

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/clickhouse.py

@@ -15,7 +15,7 @@ from sqlmesh.core.engine_adapter.shared import (
     CommentCreationView,
     InsertOverwriteStrategy,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation
+from sqlmesh.core.schema_diff import TableAlterOperation
 from sqlmesh.utils import get_source_columns_to_types
 
 if t.TYPE_CHECKING:
@@ -37,7 +37,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
     SUPPORTS_REPLACE_TABLE = False
     COMMENT_CREATION_VIEW = CommentCreationView.COMMENT_COMMAND_ONLY
 
-    SCHEMA_DIFFER = SchemaDiffer()
+    SCHEMA_DIFFER_KWARGS = {}
 
     DEFAULT_TABLE_ENGINE = "MergeTree"
     ORDER_BY_TABLE_ENGINE_REGEX = "^.*?MergeTree.*$"

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/databricks.py

@@ -15,7 +15,7 @@ from sqlmesh.core.engine_adapter.shared import (
 )
 from sqlmesh.core.engine_adapter.spark import SparkEngineAdapter
 from sqlmesh.core.node import IntervalUnit
-from sqlmesh.core.schema_diff import SchemaDiffer
+from sqlmesh.core.schema_diff import NestedSupport
 from sqlmesh.engines.spark.db_api.spark_session import connection, SparkSessionConnection
 from sqlmesh.utils.errors import SQLMeshError, MissingDefaultCatalogError
 
@@ -34,15 +34,14 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
     SUPPORTS_CLONING = True
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True
-    SCHEMA_DIFFER = SchemaDiffer(
-        support_positional_add=True,
-        support_nested_operations=True,
-        support_nested_drop=True,
-        array_element_selector="element",
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "support_positional_add": True,
+        "nested_support": NestedSupport.ALL,
+        "array_element_selector": "element",
+        "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(10, 0), (0,)],
         },
-    )
+    }
 
     def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
         super().__init__(*args, **kwargs)

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/duckdb.py

@@ -18,7 +18,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import SchemaName, TableName
@@ -29,11 +28,11 @@ if t.TYPE_CHECKING:
 class DuckDBEngineAdapter(LogicalMergeMixin, GetCurrentCatalogFromFunctionMixin, RowDiffMixin):
     DIALECT = "duckdb"
     SUPPORTS_TRANSACTIONS = False
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(18, 3), (0,)],
         },
-    )
+    }
     COMMENT_CREATION_TABLE = CommentCreationTable.COMMENT_COMMAND_ONLY
     COMMENT_CREATION_VIEW = CommentCreationView.COMMENT_COMMAND_ONLY
     SUPPORTS_CREATE_DROP_CATALOG = True

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/mixins.py

@@ -259,9 +259,9 @@ class VarcharSizeWorkaroundMixin(EngineAdapter):
     ) -> t.Dict[str, exp.DataType]:
         # get default lengths for types that support "max" length
         types_with_max_default_param = {
-            k: [self.SCHEMA_DIFFER.parameterized_type_defaults[k][0][0]]
-            for k in self.SCHEMA_DIFFER.max_parameter_length
-            if k in self.SCHEMA_DIFFER.parameterized_type_defaults
+            k: [self.schema_differ.parameterized_type_defaults[k][0][0]]
+            for k in self.schema_differ.max_parameter_length
+            if k in self.schema_differ.parameterized_type_defaults
         }
 
         # Redshift and MSSQL have a bug where CTAS statements have non-deterministic types. If a LIMIT
@@ -270,7 +270,7 @@ class VarcharSizeWorkaroundMixin(EngineAdapter):
         # and supports "max" length, we convert it to "max" length to prevent inadvertent data truncation.
         for col_name, col_type in columns_to_types.items():
             if col_type.this in types_with_max_default_param and col_type.expressions:
-                parameter = self.SCHEMA_DIFFER.get_type_parameters(col_type)
+                parameter = self.schema_differ.get_type_parameters(col_type)
                 type_default = types_with_max_default_param[col_type.this]
                 if parameter == type_default:
                     col_type.set("expressions", [exp.DataTypeParam(this=exp.var("max"))])

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/mssql.py

@@ -30,7 +30,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import get_source_columns_to_types
 
 if t.TYPE_CHECKING:
@@ -54,8 +53,8 @@ class MSSQLEngineAdapter(
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(18, 0), (0,)],
             exp.DataType.build("BINARY", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("VARBINARY", dialect=DIALECT).this: [(1,)],
@@ -67,12 +66,12 @@ class MSSQLEngineAdapter(
             exp.DataType.build("DATETIME2", dialect=DIALECT).this: [(7,)],
             exp.DataType.build("DATETIMEOFFSET", dialect=DIALECT).this: [(7,)],
         },
-        max_parameter_length={
+        "max_parameter_length": {
             exp.DataType.build("VARBINARY", dialect=DIALECT).this: 2147483647,  # 2 GB
             exp.DataType.build("VARCHAR", dialect=DIALECT).this: 2147483647,
             exp.DataType.build("NVARCHAR", dialect=DIALECT).this: 2147483647,
         },
-    )
+    }
     VARIABLE_LENGTH_DATA_TYPES = {"binary", "varbinary", "char", "varchar", "nchar", "nvarchar"}
 
     @property

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/mysql.py

@@ -19,7 +19,6 @@ from sqlmesh.core.engine_adapter.shared import (
     DataObjectType,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import SchemaName, TableName
@@ -40,8 +39,8 @@ class MySQLEngineAdapter(
     MAX_COLUMN_COMMENT_LENGTH = 1024
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 64
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             exp.DataType.build("BIT", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("BINARY", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(10, 0), (0,)],
@@ -52,7 +51,7 @@ class MySQLEngineAdapter(
             exp.DataType.build("DATETIME", dialect=DIALECT).this: [(0,)],
             exp.DataType.build("TIMESTAMP", dialect=DIALECT).this: [(0,)],
         },
-    )
+    }
 
     def get_current_catalog(self) -> t.Optional[str]:
         """Returns the catalog name of the current connection."""

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/postgres.py

@@ -14,7 +14,6 @@ from sqlmesh.core.engine_adapter.mixins import (
     logical_merge,
 )
 from sqlmesh.core.engine_adapter.shared import set_catalog
-from sqlmesh.core.schema_diff import SchemaDiffer
 
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import TableName
@@ -36,15 +35,15 @@ class PostgresEngineAdapter(
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 63
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(131072 + 16383, 16383), (0,)],
             exp.DataType.build("CHAR", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("TIME", dialect=DIALECT).this: [(6,)],
             exp.DataType.build("TIMESTAMP", dialect=DIALECT).this: [(6,)],
         },
-        types_with_unlimited_length={
+        "types_with_unlimited_length": {
             # all can ALTER to `TEXT`
             exp.DataType.build("TEXT", dialect=DIALECT).this: {
                 exp.DataType.build("VARCHAR", dialect=DIALECT).this,
@@ -63,8 +62,8 @@ class PostgresEngineAdapter(
                 exp.DataType.build("BPCHAR", dialect=DIALECT).this
             },
         },
-        drop_cascade=True,
-    )
+        "drop_cascade": True,
+    }
 
     def _fetch_native_df(
         self, query: t.Union[exp.Expression, str], quote_identifiers: bool = False

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/redshift.py

@@ -22,7 +22,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils.errors import SQLMeshError
 
 if t.TYPE_CHECKING:
@@ -48,8 +47,8 @@ class RedshiftEngineAdapter(
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
 
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             exp.DataType.build("VARBYTE", dialect=DIALECT).this: [(64000,)],
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(18, 0), (0,)],
             exp.DataType.build("CHAR", dialect=DIALECT).this: [(1,)],
@@ -57,13 +56,13 @@ class RedshiftEngineAdapter(
             exp.DataType.build("NCHAR", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("NVARCHAR", dialect=DIALECT).this: [(256,)],
         },
-        max_parameter_length={
+        "max_parameter_length": {
             exp.DataType.build("CHAR", dialect=DIALECT).this: 4096,
             exp.DataType.build("VARCHAR", dialect=DIALECT).this: 65535,
         },
-        precision_increase_allowed_types={exp.DataType.build("VARCHAR", dialect=DIALECT).this},
-        drop_cascade=True,
-    )
+        "precision_increase_allowed_types": {exp.DataType.build("VARCHAR", dialect=DIALECT).this},
+        "drop_cascade": True,
+    }
     VARIABLE_LENGTH_DATA_TYPES = {
         "char",
         "character",

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/snowflake.py

@@ -23,7 +23,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import optional_import, get_source_columns_to_types
 from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.utils.pandas import columns_to_types_from_dtypes
@@ -56,8 +55,8 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     CURRENT_CATALOG_EXPRESSION = exp.func("current_database")
     SUPPORTS_CREATE_DROP_CATALOG = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["DATABASE", "SCHEMA", "TABLE"]
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             exp.DataType.build("BINARY", dialect=DIALECT).this: [(8388608,)],
             exp.DataType.build("VARBINARY", dialect=DIALECT).this: [(8388608,)],
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(38, 0), (0,)],
@@ -70,7 +69,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             exp.DataType.build("TIMESTAMP_NTZ", dialect=DIALECT).this: [(9,)],
             exp.DataType.build("TIMESTAMP_TZ", dialect=DIALECT).this: [(9,)],
         },
-    )
+    }
     MANAGED_TABLE_KIND = "DYNAMIC TABLE"
     SNOWPARK = "snowpark"
 

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/spark.py

@@ -22,7 +22,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import classproperty, get_source_columns_to_types
 from sqlmesh.utils.errors import SQLMeshError
 
@@ -61,12 +60,12 @@ class SparkEngineAdapter(
 
     WAP_PREFIX = "wap_"
     BRANCH_PREFIX = "branch_"
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             # default decimal precision varies across backends
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(), (0,)],
         },
-    )
+    }
 
     @property
     def connection(self) -> SparkSessionConnection:

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/engine_adapter/trino.py

@@ -26,7 +26,6 @@ from sqlmesh.core.engine_adapter.shared import (
     SourceQuery,
     set_catalog,
 )
-from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import get_source_columns_to_types
 from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.utils.date import TimeLike
@@ -56,14 +55,14 @@ class TrinoEngineAdapter(
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
     DEFAULT_CATALOG_TYPE = "hive"
     QUOTE_IDENTIFIERS_IN_VIEWS = False
-    SCHEMA_DIFFER = SchemaDiffer(
-        parameterized_type_defaults={
+    SCHEMA_DIFFER_KWARGS = {
+        "parameterized_type_defaults": {
             # default decimal precision varies across backends
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(), (0,)],
             exp.DataType.build("CHAR", dialect=DIALECT).this: [(1,)],
             exp.DataType.build("TIMESTAMP", dialect=DIALECT).this: [(3,)],
         },
-    )
+    }
     # some catalogs support microsecond (precision 6) but it has to be specifically enabled (Hive) or just isnt available (Delta / TIMESTAMP WITH TIME ZONE)
     # and even if you have a TIMESTAMP(6) the date formatting functions still only support millisecond precision
     MAX_TIMESTAMP_PRECISION = 3

{sqlmesh-0.210.1.dev2 → sqlmesh-0.210.1.dev4}/sqlmesh/core/schema_diff.py

@@ -5,6 +5,8 @@ import logging
 import typing as t
 from dataclasses import dataclass
 from collections import defaultdict
+from enum import Enum
+
 from pydantic import Field
 from sqlglot import exp
 from sqlglot.helper import ensure_list, seq_get
@@ -132,14 +134,15 @@ class TableAlterDropColumnOperation(TableAlterColumnOperation):
 @dataclass(frozen=True)
 class TableAlterChangeColumnTypeOperation(TableAlterTypedColumnOperation):
     current_type: exp.DataType
+    is_part_of_destructive_change: bool = False
 
     @property
     def is_additive(self) -> bool:
-        return True
+        return not self.is_part_of_destructive_change
 
     @property
     def is_destructive(self) -> bool:
-        return False
+        return self.is_part_of_destructive_change
 
     @property
     def _alter_actions(self) -> t.List[exp.Expression]:
@@ -278,6 +281,33 @@ class TableAlterColumnPosition:
         return exp.ColumnPosition(this=column, position=position)
 
 
+class NestedSupport(str, Enum):
+    # Supports all nested data type operations
+    ALL = "ALL"
+    # Does not support any nested data type operations
+    NONE = "NONE"
+    # Supports nested data type operations except for those that require dropping a nested field
+    ALL_BUT_DROP = "ALL_BUT_DROP"
+    # Ignores all nested data type operations
+    IGNORE = "IGNORE"
+
+    @property
+    def is_all(self) -> bool:
+        return self == NestedSupport.ALL
+
+    @property
+    def is_none(self) -> bool:
+        return self == NestedSupport.NONE
+
+    @property
+    def is_all_but_drop(self) -> bool:
+        return self == NestedSupport.ALL_BUT_DROP
+
+    @property
+    def is_ignore(self) -> bool:
+        return self == NestedSupport.IGNORE
+
+
 class SchemaDiffer(PydanticModel):
     """
     Compares a source schema against a target schema and returns a list of alter statements to have the source
@@ -297,10 +327,7 @@ class SchemaDiffer(PydanticModel):
     Args:
         support_positional_add: Whether the engine for which the diff is being computed supports adding columns in a
             specific position in the set of existing columns.
-        support_nested_operations: Whether the engine for which the diff is being computed supports modifications to
-            nested data types like STRUCTs and ARRAYs.
-        support_nested_drop: Whether the engine for which the diff is being computed supports removing individual
-            columns of nested STRUCTs.
+        nested_support: How the engine for which the diff is being computed supports nested types.
         compatible_types: Types that are compatible and automatically coerced in actions like UNION ALL. Dict key is data
             type, and value is the set of types that are compatible with it.
         coerceable_types: The mapping from a current type to all types that can be safely coerced to the current one without
@@ -323,11 +350,14 @@ class SchemaDiffer(PydanticModel):
         max_parameter_length: Numeric parameter values corresponding to "max". Example: `VARCHAR(max)` -> `VARCHAR(65535)`.
         types_with_unlimited_length: Data types that accept values of any length up to system limits. Any explicitly
             parameterized type can ALTER to its unlimited length version, along with different types in some engines.
+        treat_alter_data_type_as_destructive: The SchemaDiffer will only output change data type operations if it
+            concludes the change is compatible and won't result in data loss. If this flag is set to True, it will
+            flag these data type changes as destructive. This was added for dbt adapter support and likely shouldn't
+            be set outside of that context.
     """
 
     support_positional_add: bool = False
-    support_nested_operations: bool = False
-    support_nested_drop: bool = False
+    nested_support: NestedSupport = NestedSupport.NONE
     array_element_selector: str = ""
     compatible_types: t.Dict[exp.DataType, t.Set[exp.DataType]] = {}
     coerceable_types_: t.Dict[exp.DataType, t.Set[exp.DataType]] = Field(
@@ -341,6 +371,7 @@ class SchemaDiffer(PydanticModel):
     ] = {}
     max_parameter_length: t.Dict[exp.DataType.Type, t.Union[int, float]] = {}
     types_with_unlimited_length: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
+    treat_alter_data_type_as_destructive: bool = False
 
     _coerceable_types: t.Dict[exp.DataType, t.Set[exp.DataType]] = {}
 
@@ -575,9 +606,11 @@ class SchemaDiffer(PydanticModel):
         # We don't copy on purpose here because current_type may need to be mutated inside
         # _get_operations (struct.expressions.pop and struct.expressions.insert)
         current_type = exp.DataType.build(current_type, copy=False)
-        if self.support_nested_operations:
+        if not self.nested_support.is_none:
             if new_type.this == current_type.this == exp.DataType.Type.STRUCT:
-                if self.support_nested_drop or not self._requires_drop_alteration(
+                if self.nested_support.is_ignore:
+                    return []
+                if self.nested_support.is_all or not self._requires_drop_alteration(
                     current_type, new_type
                 ):
                     return self._get_operations(
@@ -597,7 +630,9 @@ class SchemaDiffer(PydanticModel):
                 new_array_type = new_type.expressions[0]
                 current_array_type = current_type.expressions[0]
                 if new_array_type.this == current_array_type.this == exp.DataType.Type.STRUCT:
-                    if self.support_nested_drop or not self._requires_drop_alteration(
+                    if self.nested_support.is_ignore:
+                        return []
+                    if self.nested_support.is_all or not self._requires_drop_alteration(
                         current_array_type, new_array_type
                     ):
                         return self._get_operations(
@@ -624,6 +659,7 @@ class SchemaDiffer(PydanticModel):
                     current_type=current_type,
                     expected_table_struct=root_struct.copy(),
                     array_element_selector=self.array_element_selector,
+                    is_part_of_destructive_change=self.treat_alter_data_type_as_destructive,
                 )
             ]
         if ignore_destructive:
@@ -806,12 +842,15 @@ def get_additive_column_names(alter_expressions: t.List[TableAlterOperation]) ->
     ]
 
 
-def get_schema_differ(dialect: str) -> SchemaDiffer:
+def get_schema_differ(
+    dialect: str, overrides: t.Optional[t.Dict[str, t.Any]] = None
+) -> SchemaDiffer:
     """
     Returns the appropriate SchemaDiffer for a given dialect without initializing the engine adapter.
 
     Args:
         dialect: The dialect for which to get the schema differ.
+        overrides: Optional dictionary of overrides to apply to the SchemaDiffer instance.
 
     Returns:
         The SchemaDiffer instance configured for the given dialect.
@@ -825,7 +864,12 @@ def get_schema_differ(dialect: str) -> SchemaDiffer:
     dialect = dialect.lower()
     dialect = DIALECT_ALIASES.get(dialect, dialect)
     engine_adapter_class = DIALECT_TO_ENGINE_ADAPTER.get(dialect, EngineAdapter)
-    return getattr(engine_adapter_class, "SCHEMA_DIFFER", SchemaDiffer())
+    return SchemaDiffer(
+        **{
+            **getattr(engine_adapter_class, "SCHEMA_DIFFER_KWARGS"),
+            **(overrides or {}),
+        }
+    )
 
 
 def _get_name_and_type(struct: exp.ColumnDef) -> t.Tuple[exp.Identifier, exp.DataType]: