sqlmesh 0.212.1.dev2__tar.gz → 0.212.1.dev4__tar.gz

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.212.1.dev2
+Version: 0.212.1.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.212.1.dev2'
-__version_tuple__ = version_tuple = (0, 212, 1, 'dev2')
+__version__ = version = '0.212.1.dev4'
+__version_tuple__ = version_tuple = (0, 212, 1, 'dev4')
 
-__commit_id__ = commit_id = 'g70c8deb48'
+__commit_id__ = commit_id = 'g75f825e9f'

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/audit/definition.py

@@ -15,11 +15,10 @@ from sqlmesh.core.model.common import (
     bool_validator,
     default_catalog_validator,
     depends_on_validator,
-    expression_validator,
     sort_python_env,
     sorted_python_env_payloads,
 )
-from sqlmesh.core.model.common import make_python_env, single_value_or_tuple
+from sqlmesh.core.model.common import make_python_env, single_value_or_tuple, ParsableSql
 from sqlmesh.core.node import _Node
 from sqlmesh.core.renderer import QueryRenderer
 from sqlmesh.utils.date import TimeLike
@@ -67,15 +66,26 @@ class AuditMixin(AuditCommonMetaMixin):
         jinja_macros: A registry of jinja macros to use when rendering the audit query.
     """
 
-    query: t.Union[exp.Query, d.JinjaQuery]
+    query_: ParsableSql
     defaults: t.Dict[str, exp.Expression]
-    expressions_: t.Optional[t.List[exp.Expression]]
+    expressions_: t.Optional[t.List[ParsableSql]]
     jinja_macros: JinjaMacroRegistry
     formatting: t.Optional[bool]
 
+    @property
+    def query(self) -> t.Union[exp.Query, d.JinjaQuery]:
+        return t.cast(t.Union[exp.Query, d.JinjaQuery], self.query_.parse(self.dialect))
+
     @property
     def expressions(self) -> t.List[exp.Expression]:
-        return self.expressions_ or []
+        if not self.expressions_:
+            return []
+        result = []
+        for e in self.expressions_:
+            parsed = e.parse(self.dialect)
+            if not isinstance(parsed, exp.Semicolon):
+                result.append(parsed)
+        return result
 
     @property
     def macro_definitions(self) -> t.List[d.MacroDef]:
@@ -122,16 +132,16 @@ class ModelAudit(PydanticModel, AuditMixin, frozen=True):
     skip: bool = False
     blocking: bool = True
     standalone: t.Literal[False] = False
-    query: t.Union[exp.Query, d.JinjaQuery]
+    query_: ParsableSql = Field(alias="query")
     defaults: t.Dict[str, exp.Expression] = {}
-    expressions_: t.Optional[t.List[exp.Expression]] = Field(default=None, alias="expressions")
+    expressions_: t.Optional[t.List[ParsableSql]] = Field(default=None, alias="expressions")
     jinja_macros: JinjaMacroRegistry = JinjaMacroRegistry()
     formatting: t.Optional[bool] = Field(default=None, exclude=True)
 
     _path: t.Optional[Path] = None
 
     # Validators
-    _query_validator = expression_validator
+    _query_validator = ParsableSql.validator()
     _bool_validator = bool_validator
     _string_validator = audit_string_validator
     _map_validator = audit_map_validator
@@ -153,9 +163,9 @@ class StandaloneAudit(_Node, AuditMixin):
     skip: bool = False
     blocking: bool = False
     standalone: t.Literal[True] = True
-    query: t.Union[exp.Query, d.JinjaQuery]
+    query_: ParsableSql = Field(alias="query")
     defaults: t.Dict[str, exp.Expression] = {}
-    expressions_: t.Optional[t.List[exp.Expression]] = Field(default=None, alias="expressions")
+    expressions_: t.Optional[t.List[ParsableSql]] = Field(default=None, alias="expressions")
     jinja_macros: JinjaMacroRegistry = JinjaMacroRegistry()
     default_catalog: t.Optional[str] = None
     depends_on_: t.Optional[t.Set[str]] = Field(default=None, alias="depends_on")
@@ -165,7 +175,7 @@ class StandaloneAudit(_Node, AuditMixin):
     source_type: t.Literal["audit"] = "audit"
 
     # Validators
-    _query_validator = expression_validator
+    _query_validator = ParsableSql.validator()
     _bool_validator = bool_validator
     _string_validator = audit_string_validator
     _map_validator = audit_map_validator
@@ -276,8 +286,8 @@ class StandaloneAudit(_Node, AuditMixin):
                 self.cron_tz.key if self.cron_tz else None,
             ]
 
-            query = self.render_audit_query() or self.query
-            data.append(gen(query))
+            data.append(self.query_.sql)
+            data.extend([e.sql for e in self.expressions_ or []])
             self._metadata_hash = hash_data(data)
         return self._metadata_hash
 
@@ -461,11 +471,17 @@ def load_audit(
         if project is not None:
             extra_kwargs["project"] = project
 
-    dialect = meta_fields.pop("dialect", dialect)
+    dialect = meta_fields.pop("dialect", dialect) or ""
+
+    parsable_query = ParsableSql.from_parsed_expression(query, dialect, use_meta_sql=True)
+    parsable_statements = [
+        ParsableSql.from_parsed_expression(s, dialect, use_meta_sql=True) for s in statements
+    ]
+
     try:
         audit = audit_class(
-            query=query,
-            expressions=statements,
+            query=parsable_query,
+            expressions=parsable_statements,
             dialect=dialect,
             **extra_kwargs,
             **meta_fields,

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/context_diff.py

@@ -435,7 +435,7 @@ class ContextDiff(PydanticModel):
             return False
 
         current, previous = self.modified_snapshots[name]
-        return current.fingerprint.data_hash != previous.fingerprint.data_hash
+        return current.is_directly_modified(previous)
 
     def indirectly_modified(self, name: str) -> bool:
         """Returns whether or not a node was indirectly modified in this context.
@@ -451,10 +451,7 @@ class ContextDiff(PydanticModel):
             return False
 
         current, previous = self.modified_snapshots[name]
-        return (
-            current.fingerprint.data_hash == previous.fingerprint.data_hash
-            and current.fingerprint.parent_data_hash != previous.fingerprint.parent_data_hash
-        )
+        return current.is_indirectly_modified(previous)
 
     def metadata_updated(self, name: str) -> bool:
         """Returns whether or not the given node's metadata has been updated.
@@ -470,7 +467,7 @@ class ContextDiff(PydanticModel):
             return False
 
         current, previous = self.modified_snapshots[name]
-        return current.fingerprint.metadata_hash != previous.fingerprint.metadata_hash
+        return current.is_metadata_updated(previous)
 
     def text_diff(self, name: str) -> str:
         """Finds the difference of a node between the current and remote environment.

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/engine_adapter/athena.py

@@ -437,6 +437,7 @@ class AthenaEngineAdapter(PandasNativeFetchDFSupportMixin, RowDiffMixin):
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         source_columns: t.Optional[t.List[str]] = None,
+        supports_replace_table_override: t.Optional[bool] = None,
         **kwargs: t.Any,
     ) -> None:
         table = exp.to_table(table_name)

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/engine_adapter/base.py

@@ -432,8 +432,16 @@ class EngineAdapter:
             )
         return self.DEFAULT_CATALOG_TYPE
 
+    def get_catalog_type_from_table(self, table: TableName) -> str:
+        """Get the catalog type from a table name if it has a catalog specified, otherwise return the current catalog type"""
+        catalog = exp.to_table(table).catalog or self.get_current_catalog()
+        return self.get_catalog_type(catalog)
+
     @property
     def current_catalog_type(self) -> str:
+        # `get_catalog_type_from_table` should be used over this property. Reason is that the table that is the target
+        # of the operation is what matters and not the catalog type of the connection.
+        # This still remains for legacy reasons and should be refactored out.
         return self.get_catalog_type(self.get_current_catalog())
 
     def replace_query(
@@ -444,6 +452,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         source_columns: t.Optional[t.List[str]] = None,
+        supports_replace_table_override: t.Optional[bool] = None,
         **kwargs: t.Any,
     ) -> None:
         """Replaces an existing table with a query.
@@ -494,12 +503,17 @@ class EngineAdapter:
             )
         # All engines support `CREATE TABLE AS` so we use that if the table doesn't already exist and we
         # use `CREATE OR REPLACE TABLE AS` if the engine supports it
-        if self.SUPPORTS_REPLACE_TABLE or not table_exists:
+        supports_replace_table = (
+            self.SUPPORTS_REPLACE_TABLE
+            if supports_replace_table_override is None
+            else supports_replace_table_override
+        )
+        if supports_replace_table or not table_exists:
             return self._create_table_from_source_queries(
                 target_table,
                 source_queries,
                 target_columns_to_types,
-                replace=self.SUPPORTS_REPLACE_TABLE,
+                replace=supports_replace_table,
                 table_description=table_description,
                 column_descriptions=column_descriptions,
                 **kwargs,

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/engine_adapter/mixins.py

@@ -228,7 +228,7 @@ class HiveMetastoreTablePropertiesMixin(EngineAdapter):
 
     def _truncate_comment(self, comment: str, length: t.Optional[int]) -> str:
         # iceberg and delta do not have a comment length limit
-        if self.current_catalog_type in ("iceberg", "delta"):
+        if self.current_catalog_type in ("iceberg", "delta_lake"):
             return comment
         return super()._truncate_comment(comment, length)
 

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/engine_adapter/redshift.py

@@ -253,6 +253,7 @@ class RedshiftEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         source_columns: t.Optional[t.List[str]] = None,
+        supports_replace_table_override: t.Optional[bool] = None,
         **kwargs: t.Any,
     ) -> None:
         """

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/engine_adapter/trino.py

@@ -34,6 +34,8 @@ if t.TYPE_CHECKING:
     from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
     from sqlmesh.core.engine_adapter._typing import DF, QueryOrDF
 
+CATALOG_TYPES_SUPPORTING_REPLACE_TABLE = {"iceberg", "delta_lake"}
+
 
 @set_catalog()
 class TrinoEngineAdapter(
@@ -115,6 +117,37 @@ class TrinoEngineAdapter(
         finally:
             self.execute(f"RESET SESSION AUTHORIZATION")
 
+    def replace_query(
+        self,
+        table_name: TableName,
+        query_or_df: QueryOrDF,
+        target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        table_description: t.Optional[str] = None,
+        column_descriptions: t.Optional[t.Dict[str, str]] = None,
+        source_columns: t.Optional[t.List[str]] = None,
+        supports_replace_table_override: t.Optional[bool] = None,
+        **kwargs: t.Any,
+    ) -> None:
+        catalog_type = self.get_catalog_type(self.get_catalog_type_from_table(table_name))
+        # User may have a custom catalog type name so we are assuming they keep the catalog type still in the name
+        # Ex: `acme_iceberg` would be identified as an iceberg catalog and therefore supports replace table
+        supports_replace_table_override = None
+        for replace_table_catalog_type in CATALOG_TYPES_SUPPORTING_REPLACE_TABLE:
+            if replace_table_catalog_type in catalog_type:
+                supports_replace_table_override = True
+                break
+
+        super().replace_query(
+            table_name=table_name,
+            query_or_df=query_or_df,
+            target_columns_to_types=target_columns_to_types,
+            table_description=table_description,
+            column_descriptions=column_descriptions,
+            source_columns=source_columns,
+            supports_replace_table_override=supports_replace_table_override,
+            **kwargs,
+        )
+
     def _insert_overwrite_by_condition(
         self,
         table_name: TableName,
@@ -250,7 +283,7 @@ class TrinoEngineAdapter(
         expressions: t.Optional[t.List[exp.PrimaryKey]] = None,
         is_view: bool = False,
     ) -> exp.Schema:
-        if self.current_catalog_type == "delta_lake":
+        if "delta_lake" in self.get_catalog_type_from_table(table):
             target_columns_to_types = self._to_delta_ts(target_columns_to_types)
 
         return super()._build_schema_exp(
@@ -277,7 +310,9 @@ class TrinoEngineAdapter(
         source_columns: t.Optional[t.List[str]] = None,
         **kwargs: t.Any,
     ) -> None:
-        if target_columns_to_types and self.current_catalog_type == "delta_lake":
+        if target_columns_to_types and "delta_lake" in self.get_catalog_type_from_table(
+            target_table
+        ):
             target_columns_to_types = self._to_delta_ts(target_columns_to_types)
 
         return super()._scd_type_2(
@@ -381,7 +416,7 @@ class TrinoEngineAdapter(
         else:
             table_name = table_name_or_schema
 
-        if self.current_catalog_type == "hive":
+        if "hive" in self.get_catalog_type_from_table(table_name):
             # the Trino Hive connector can take a few seconds for metadata changes to propagate to all internal threads
             # (even if metadata TTL is set to 0s)
             # Blocking until the table shows up means that subsequent code expecting it to exist immediately will not fail

{sqlmesh-0.212.1.dev2 → sqlmesh-0.212.1.dev4}/sqlmesh/core/model/common.py

@@ -21,7 +21,7 @@ from sqlmesh.utils.metaprogramming import (
     prepare_env,
     serialize_env,
 )
-from sqlmesh.utils.pydantic import PydanticModel, ValidationInfo, field_validator
+from sqlmesh.utils.pydantic import PydanticModel, ValidationInfo, field_validator, get_dialect
 
 if t.TYPE_CHECKING:
     from sqlglot.dialects.dialect import DialectType
@@ -616,11 +616,6 @@ def parse_strings_with_macro_refs(value: t.Any, dialect: DialectType) -> t.Any:
 
 
 expression_validator: t.Callable = field_validator(
-    "query",
-    "expressions_",
-    "pre_statements_",
-    "post_statements_",
-    "on_virtual_update_",
     "unique_key",
     mode="before",
     check_fields=False,
@@ -663,3 +658,65 @@ depends_on_validator: t.Callable = field_validator(
     mode="before",
     check_fields=False,
 )(depends_on)
+
+
+class ParsableSql(PydanticModel):
+    sql: str
+
+    _parsed: t.Optional[exp.Expression] = None
+    _parsed_dialect: t.Optional[str] = None
+
+    def parse(self, dialect: str) -> exp.Expression:
+        if self._parsed is None or self._parsed_dialect != dialect:
+            self._parsed = d.parse_one(self.sql, dialect=dialect)
+            self._parsed_dialect = dialect
+        return self._parsed
+
+    @classmethod
+    def from_parsed_expression(
+        cls, parsed_expression: exp.Expression, dialect: str, use_meta_sql: bool = False
+    ) -> ParsableSql:
+        sql = (
+            parsed_expression.meta.get("sql") or parsed_expression.sql(dialect=dialect)
+            if use_meta_sql
+            else parsed_expression.sql(dialect=dialect)
+        )
+        result = cls(sql=sql)
+        result._parsed = parsed_expression
+        result._parsed_dialect = dialect
+        return result
+
+    @classmethod
+    def validator(cls) -> classmethod:
+        def _validate_parsable_sql(
+            v: t.Any, info: ValidationInfo
+        ) -> t.Optional[t.Union[ParsableSql, t.List[ParsableSql]]]:
+            if v is None:
+                return v
+            if isinstance(v, str):
+                return ParsableSql(sql=v)
+            if isinstance(v, exp.Expression):
+                return ParsableSql.from_parsed_expression(
+                    v, get_dialect(info.data), use_meta_sql=False
+                )
+            if isinstance(v, list):
+                dialect = get_dialect(info.data)
+                return [
+                    ParsableSql(sql=s)
+                    if isinstance(s, str)
+                    else ParsableSql.from_parsed_expression(s, dialect, use_meta_sql=False)
+                    if isinstance(s, exp.Expression)
+                    else ParsableSql.parse_obj(s)
+                    for s in v
+                ]
+            return ParsableSql.parse_obj(v)
+
+        return field_validator(
+            "query_",
+            "expressions_",
+            "pre_statements_",
+            "post_statements_",
+            "on_virtual_update_",
+            mode="before",
+            check_fields=False,
+        )(_validate_parsable_sql)