sqlmesh 0.211.1.dev4__tar.gz → 0.211.2.dev2__tar.gz

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/.circleci/continue_config.yml

@@ -239,7 +239,7 @@ jobs:
       - checkout
       - run:
           name: Install OS-level dependencies
-          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"      
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
       - run:
           name: Generate database name
           command: |
@@ -307,7 +307,7 @@ workflows:
                 - redshift
                 - bigquery
                 - clickhouse-cloud
-                - athena                
+                - athena
                 - fabric
                 - gcp-postgres
           filters:

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.211.1.dev4
+Version: 0.211.2.dev2
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -223,6 +223,7 @@ Requires-Dist: click
 Requires-Dist: croniter
 Requires-Dist: duckdb!=0.10.3,>=0.10.0
 Requires-Dist: dateparser<=1.2.1
+Requires-Dist: humanize
 Requires-Dist: hyperscript>=0.1.0
 Requires-Dist: importlib-metadata; python_version < "3.12"
 Requires-Dist: ipywidgets

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "croniter",
     "duckdb>=0.10.0,!=0.10.3",
     "dateparser<=1.2.1",
+    "humanize",
     "hyperscript>=0.1.0",
     "importlib-metadata; python_version<'3.12'",
     "ipywidgets",

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.211.1.dev4'
-__version_tuple__ = version_tuple = (0, 211, 1, 'dev4')
+__version__ = version = '0.211.2.dev2'
+__version_tuple__ = version_tuple = (0, 211, 2, 'dev2')
 
-__commit_id__ = commit_id = 'g66cd077fe'
+__commit_id__ = commit_id = 'g91428c779'

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/console.py

@@ -7,6 +7,7 @@ import unittest
 import uuid
 import logging
 import textwrap
+from humanize import metric, naturalsize
 from itertools import zip_longest
 from pathlib import Path
 from hyperscript import h
@@ -39,6 +40,7 @@ from sqlmesh.core.snapshot import (
     SnapshotInfoLike,
 )
 from sqlmesh.core.snapshot.definition import Interval, Intervals, SnapshotTableInfo
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionStats
 from sqlmesh.core.test import ModelTest
 from sqlmesh.utils import rich as srich
 from sqlmesh.utils import Verbosity
@@ -439,6 +441,7 @@ class Console(
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Updates the snapshot evaluation progress."""
@@ -587,6 +590,7 @@ class NoopConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         pass
@@ -1032,7 +1036,9 @@ class TerminalConsole(Console):
 
             # determine column widths
             self.evaluation_column_widths["annotation"] = (
-                _calculate_annotation_str_len(batched_intervals, self.AUDIT_PADDING)
+                _calculate_annotation_str_len(
+                    batched_intervals, self.AUDIT_PADDING, len(" (123.4m rows, 123.4 KiB)")
+                )
                 + 3  # brackets and opening escape backslash
             )
             self.evaluation_column_widths["name"] = max(
@@ -1077,6 +1083,7 @@ class TerminalConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Update the snapshot evaluation progress."""
@@ -1097,7 +1104,7 @@ class TerminalConsole(Console):
                 ).ljust(self.evaluation_column_widths["name"])
 
                 annotation = _create_evaluation_model_annotation(
-                    snapshot, _format_evaluation_model_interval(snapshot, interval)
+                    snapshot, _format_evaluation_model_interval(snapshot, interval), execution_stats
                 )
                 audits_str = ""
                 if num_audits_passed:
@@ -3668,6 +3675,7 @@ class DatabricksMagicConsole(CaptureTerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         view_name, loaded_batches = self.evaluation_batch_progress[snapshot.snapshot_id]
@@ -3838,6 +3846,7 @@ class DebuggerTerminalConsole(TerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         message = f"Evaluated {snapshot.name} | batch={batch_idx} | duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
@@ -4169,33 +4178,62 @@ def _format_evaluation_model_interval(snapshot: Snapshot, interval: Interval) ->
     return ""
 
 
-def _create_evaluation_model_annotation(snapshot: Snapshot, interval_info: t.Optional[str]) -> str:
+def _create_evaluation_model_annotation(
+    snapshot: Snapshot,
+    interval_info: t.Optional[str],
+    execution_stats: t.Optional[QueryExecutionStats],
+) -> str:
+    annotation = None
+    execution_stats_str = ""
+    if execution_stats:
+        rows_processed = execution_stats.total_rows_processed
+        if rows_processed:
+            # 1.00 and 1.0 to 1
+            rows_processed_str = metric(rows_processed).replace(".00", "").replace(".0", "")
+            execution_stats_str += f"{rows_processed_str} row{'s' if rows_processed > 1 else ''}"
+
+        bytes_processed = execution_stats.total_bytes_processed
+        execution_stats_str += (
+            f"{', ' if execution_stats_str else ''}{naturalsize(bytes_processed, binary=True)}"
+            if bytes_processed
+            else ""
+        )
+    execution_stats_str = f" ({execution_stats_str})" if execution_stats_str else ""
+
     if snapshot.is_audit:
-        return "run standalone audit"
-    if snapshot.is_model and snapshot.model.kind.is_external:
-        return "run external audits"
-    if snapshot.model.kind.is_seed:
-        return "insert seed file"
-    if snapshot.model.kind.is_full:
-        return "full refresh"
-    if snapshot.model.kind.is_view:
-        return "recreate view"
-    if snapshot.model.kind.is_incremental_by_unique_key:
-        return "insert/update rows"
-    if snapshot.model.kind.is_incremental_by_partition:
-        return "insert partitions"
-
-    return interval_info if interval_info else ""
-
-
-def _calculate_interval_str_len(snapshot: Snapshot, intervals: t.List[Interval]) -> int:
+        annotation = "run standalone audit"
+    if snapshot.is_model:
+        if snapshot.model.kind.is_external:
+            annotation = "run external audits"
+        if snapshot.model.kind.is_view:
+            annotation = "recreate view"
+        if snapshot.model.kind.is_seed:
+            annotation = f"insert seed file{execution_stats_str}"
+        if snapshot.model.kind.is_full:
+            annotation = f"full refresh{execution_stats_str}"
+        if snapshot.model.kind.is_incremental_by_unique_key:
+            annotation = f"insert/update rows{execution_stats_str}"
+        if snapshot.model.kind.is_incremental_by_partition:
+            annotation = f"insert partitions{execution_stats_str}"
+
+    if annotation:
+        return annotation
+
+    return f"{interval_info}{execution_stats_str}" if interval_info else ""
+
+
+def _calculate_interval_str_len(
+    snapshot: Snapshot,
+    intervals: t.List[Interval],
+    execution_stats: t.Optional[QueryExecutionStats] = None,
+) -> int:
     interval_str_len = 0
     for interval in intervals:
         interval_str_len = max(
             interval_str_len,
             len(
                 _create_evaluation_model_annotation(
-                    snapshot, _format_evaluation_model_interval(snapshot, interval)
+                    snapshot, _format_evaluation_model_interval(snapshot, interval), execution_stats
                 )
             ),
         )
@@ -4248,13 +4286,16 @@ def _calculate_audit_str_len(snapshot: Snapshot, audit_padding: int = 0) -> int:
 
 
 def _calculate_annotation_str_len(
-    batched_intervals: t.Dict[Snapshot, t.List[Interval]], audit_padding: int = 0
+    batched_intervals: t.Dict[Snapshot, t.List[Interval]],
+    audit_padding: int = 0,
+    execution_stats_len: int = 0,
 ) -> int:
     annotation_str_len = 0
     for snapshot, intervals in batched_intervals.items():
         annotation_str_len = max(
             annotation_str_len,
             _calculate_interval_str_len(snapshot, intervals)
-            + _calculate_audit_str_len(snapshot, audit_padding),
+            + _calculate_audit_str_len(snapshot, audit_padding)
+            + execution_stats_len,
         )
     return annotation_str_len

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/athena.py

@@ -45,6 +45,7 @@ class AthenaEngineAdapter(PandasNativeFetchDFSupportMixin, RowDiffMixin):
     # >>> self._execute('/* test */ DESCRIBE foo')
     #     pyathena.error.OperationalError: FAILED: ParseException line 1:0 cannot recognize input near '/' '*' 'test'
     ATTACH_CORRELATION_ID = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["DATABASE", "SCHEMA"]
 
     def __init__(

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/base.py

@@ -40,6 +40,7 @@ from sqlmesh.core.engine_adapter.shared import (
 )
 from sqlmesh.core.model.kind import TimeColumn
 from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionTracker
 from sqlmesh.utils import (
     CorrelationId,
     columns_to_types_all_known,
@@ -117,6 +118,7 @@ class EngineAdapter:
     QUOTE_IDENTIFIERS_IN_VIEWS = True
     MAX_IDENTIFIER_LENGTH: t.Optional[int] = None
     ATTACH_CORRELATION_ID = True
+    SUPPORTS_QUERY_EXECUTION_TRACKING = False
 
     def __init__(
         self,
@@ -133,6 +135,7 @@ class EngineAdapter:
         shared_connection: bool = False,
         correlation_id: t.Optional[CorrelationId] = None,
         schema_differ_overrides: t.Optional[t.Dict[str, t.Any]] = None,
+        query_execution_tracker: t.Optional[QueryExecutionTracker] = None,
         **kwargs: t.Any,
     ):
         self.dialect = dialect.lower() or self.DIALECT
@@ -156,11 +159,16 @@ class EngineAdapter:
         self._multithreaded = multithreaded
         self.correlation_id = correlation_id
         self._schema_differ_overrides = schema_differ_overrides
+        self._query_execution_tracker = query_execution_tracker
 
     def with_settings(self, **kwargs: t.Any) -> EngineAdapter:
         extra_kwargs = {
             "null_connection": True,
             "execute_log_level": kwargs.pop("execute_log_level", self._execute_log_level),
+            "correlation_id": kwargs.pop("correlation_id", self.correlation_id),
+            "query_execution_tracker": kwargs.pop(
+                "query_execution_tracker", self._query_execution_tracker
+            ),
             **self._extra_config,
             **kwargs,
         }
@@ -854,6 +862,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table = exp.to_table(table_name)
@@ -899,11 +908,15 @@ class EngineAdapter:
                             replace=replace,
                             table_description=table_description,
                             table_kind=table_kind,
+                            track_rows_processed=track_rows_processed,
                             **kwargs,
                         )
                     else:
                         self._insert_append_query(
-                            table_name, query, target_columns_to_types or self.columns(table)
+                            table_name,
+                            query,
+                            target_columns_to_types or self.columns(table),
+                            track_rows_processed=track_rows_processed,
                         )
 
         # Register comments with commands if the engine supports comments and we weren't able to
@@ -927,6 +940,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         self.execute(
@@ -943,7 +957,8 @@ class EngineAdapter:
                 ),
                 table_kind=table_kind,
                 **kwargs,
-            )
+            ),
+            track_rows_processed=track_rows_processed,
         )
 
     def _build_create_table_exp(
@@ -1431,6 +1446,7 @@ class EngineAdapter:
         table_name: TableName,
         query_or_df: QueryOrDF,
         target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        track_rows_processed: bool = True,
         source_columns: t.Optional[t.List[str]] = None,
     ) -> None:
         source_queries, target_columns_to_types = self._get_source_queries_and_columns_to_types(
@@ -1439,19 +1455,27 @@ class EngineAdapter:
             target_table=table_name,
             source_columns=source_columns,
         )
-        self._insert_append_source_queries(table_name, source_queries, target_columns_to_types)
+        self._insert_append_source_queries(
+            table_name, source_queries, target_columns_to_types, track_rows_processed
+        )
 
     def _insert_append_source_queries(
         self,
         table_name: TableName,
         source_queries: t.List[SourceQuery],
         target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        track_rows_processed: bool = True,
     ) -> None:
         with self.transaction(condition=len(source_queries) > 0):
             target_columns_to_types = target_columns_to_types or self.columns(table_name)
             for source_query in source_queries:
                 with source_query as query:
-                    self._insert_append_query(table_name, query, target_columns_to_types)
+                    self._insert_append_query(
+                        table_name,
+                        query,
+                        target_columns_to_types,
+                        track_rows_processed=track_rows_processed,
+                    )
 
     def _insert_append_query(
         self,
@@ -1459,10 +1483,14 @@ class EngineAdapter:
         query: Query,
         target_columns_to_types: t.Dict[str, exp.DataType],
         order_projections: bool = True,
+        track_rows_processed: bool = True,
     ) -> None:
         if order_projections:
             query = self._order_projections_and_filter(query, target_columns_to_types)
-        self.execute(exp.insert(query, table_name, columns=list(target_columns_to_types)))
+        self.execute(
+            exp.insert(query, table_name, columns=list(target_columns_to_types)),
+            track_rows_processed=track_rows_processed,
+        )
 
     def insert_overwrite_by_partition(
         self,
@@ -1604,7 +1632,7 @@ class EngineAdapter:
                         )
                         if insert_overwrite_strategy.is_replace_where:
                             insert_exp.set("where", where or exp.true())
-                        self.execute(insert_exp)
+                        self.execute(insert_exp, track_rows_processed=True)
 
     def update_table(
         self,
@@ -1625,7 +1653,9 @@ class EngineAdapter:
         using = exp.alias_(
             exp.Subquery(this=query), alias=MERGE_SOURCE_ALIAS, copy=False, table=True
         )
-        self.execute(exp.Merge(this=this, using=using, on=on, whens=whens))
+        self.execute(
+            exp.Merge(this=this, using=using, on=on, whens=whens), track_rows_processed=True
+        )
 
     def scd_type_2_by_time(
         self,
@@ -2374,6 +2404,7 @@ class EngineAdapter:
         expressions: t.Union[str, exp.Expression, t.Sequence[exp.Expression]],
         ignore_unsupported_errors: bool = False,
         quote_identifiers: bool = True,
+        track_rows_processed: bool = False,
         **kwargs: t.Any,
     ) -> None:
         """Execute a sql query."""
@@ -2395,7 +2426,7 @@ class EngineAdapter:
                     expression=e if isinstance(e, exp.Expression) else None,
                     quote_identifiers=quote_identifiers,
                 )
-                self._execute(sql, **kwargs)
+                self._execute(sql, track_rows_processed, **kwargs)
 
     def _attach_correlation_id(self, sql: str) -> str:
         if self.ATTACH_CORRELATION_ID and self.correlation_id:
@@ -2420,9 +2451,29 @@ class EngineAdapter:
 
         logger.log(self._execute_log_level, "Executing SQL: %s", sql_to_log)
 
-    def _execute(self, sql: str, **kwargs: t.Any) -> None:
+    def _record_execution_stats(
+        self, sql: str, rowcount: t.Optional[int] = None, bytes_processed: t.Optional[int] = None
+    ) -> None:
+        if self._query_execution_tracker:
+            self._query_execution_tracker.record_execution(sql, rowcount, bytes_processed)
+
+    def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any) -> None:
         self.cursor.execute(sql, **kwargs)
 
+        if (
+            self.SUPPORTS_QUERY_EXECUTION_TRACKING
+            and track_rows_processed
+            and self._query_execution_tracker
+            and self._query_execution_tracker.is_tracking()
+        ):
+            if (
+                rowcount := getattr(self.cursor, "rowcount", None)
+            ) is not None and rowcount is not None:
+                try:
+                    self._record_execution_stats(sql, int(rowcount))
+                except (TypeError, ValueError):
+                    return
+
     @contextlib.contextmanager
     def temp_table(
         self,
@@ -2467,6 +2518,7 @@ class EngineAdapter:
                 exists=True,
                 table_description=None,
                 column_descriptions=None,
+                track_rows_processed=False,
                 **kwargs,
             )
 
@@ -2718,7 +2770,7 @@ class EngineAdapter:
                     insert_statement.set("where", delete_filter)
                     insert_statement.set("this", exp.to_table(target_table))
 
-                self.execute(insert_statement)
+                self.execute(insert_statement, track_rows_processed=True)
             finally:
                 self.drop_table(temp_table)
 

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/base_postgres.py

@@ -24,6 +24,7 @@ class BasePostgresEngineAdapter(EngineAdapter):
     DEFAULT_BATCH_SIZE = 400
     COMMENT_CREATION_TABLE = CommentCreationTable.COMMENT_COMMAND_ONLY
     COMMENT_CREATION_VIEW = CommentCreationView.COMMENT_COMMAND_ONLY
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA", "TABLE", "VIEW"]
 
     def columns(

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/bigquery.py

@@ -66,6 +66,7 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     SUPPORTS_CLONING = True
     MAX_TABLE_COMMENT_LENGTH = 1024
     MAX_COLUMN_COMMENT_LENGTH = 1024
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
 
     SCHEMA_DIFFER_KWARGS = {
@@ -1049,6 +1050,7 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     def _execute(
         self,
         sql: str,
+        track_rows_processed: bool = False,
         **kwargs: t.Any,
     ) -> None:
         """Execute a sql query."""
@@ -1094,6 +1096,23 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
         self.cursor._set_rowcount(query_results)
         self.cursor._set_description(query_results.schema)
 
+        if (
+            track_rows_processed
+            and self._query_execution_tracker
+            and self._query_execution_tracker.is_tracking()
+        ):
+            num_rows = None
+            if query_job.statement_type == "CREATE_TABLE_AS_SELECT":
+                # since table was just created, number rows in table == number rows processed
+                query_table = self.client.get_table(query_job.destination)
+                num_rows = query_table.num_rows
+            elif query_job.statement_type in ["INSERT", "DELETE", "MERGE", "UPDATE"]:
+                num_rows = query_job.num_dml_affected_rows
+
+            self._query_execution_tracker.record_execution(
+                sql, num_rows, query_job.total_bytes_processed
+            )
+
     def _get_data_objects(
         self, schema_name: SchemaName, object_names: t.Optional[t.Set[str]] = None
     ) -> t.List[DataObject]:

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/clickhouse.py

@@ -294,7 +294,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
                     )
 
                 try:
-                    self.execute(existing_records_insert_exp)
+                    self.execute(existing_records_insert_exp, track_rows_processed=True)
                 finally:
                     if table_partition_exp:
                         self.drop_table(partitions_temp_table_name)
@@ -489,6 +489,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         """Creates a table in the database.
@@ -525,6 +526,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
             column_descriptions,
             table_kind,
             empty_ctas=(self.engine_run_mode.is_cloud and expression is not None),
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
 

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/duckdb.py

@@ -170,6 +170,7 @@ class DuckDBEngineAdapter(LogicalMergeMixin, GetCurrentCatalogFromFunctionMixin,
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         catalog = self.get_current_catalog()
@@ -193,6 +194,7 @@ class DuckDBEngineAdapter(LogicalMergeMixin, GetCurrentCatalogFromFunctionMixin,
             table_description,
             column_descriptions,
             table_kind,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
 

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/mssql.py

@@ -53,6 +53,7 @@ class MSSQLEngineAdapter(
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(18, 0), (0,)],

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/mysql.py

@@ -39,6 +39,7 @@ class MySQLEngineAdapter(
     MAX_COLUMN_COMMENT_LENGTH = 1024
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 64
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             exp.DataType.build("BIT", dialect=DIALECT).this: [(1,)],

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/postgres.py

@@ -35,6 +35,7 @@ class PostgresEngineAdapter(
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 63
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/redshift.py

@@ -173,6 +173,7 @@ class RedshiftEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         """
@@ -426,7 +427,8 @@ class RedshiftEngineAdapter(
                 using=using,
                 on=on.transform(resolve_target_table),
                 whens=whens.transform(resolve_target_table),
-            )
+            ),
+            track_rows_processed=True,
         )
 
     def _normalize_decimal_value(self, expr: exp.Expression, precision: int) -> exp.Expression:

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/snowflake.py

@@ -72,6 +72,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     }
     MANAGED_TABLE_KIND = "DYNAMIC TABLE"
     SNOWPARK = "snowpark"
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
 
     @contextlib.contextmanager
     def session(self, properties: SessionProperties) -> t.Iterator[None]:
@@ -166,6 +167,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table_format = kwargs.get("table_format")
@@ -185,6 +187,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             table_description=table_description,
             column_descriptions=column_descriptions,
             table_kind=table_kind,
+            track_rows_processed=False,  # snowflake tracks CTAS row counts incorrectly
             **kwargs,
         )
 

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/spark.py

@@ -433,6 +433,7 @@ class SparkEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table_name = (
@@ -461,6 +462,7 @@ class SparkEngineAdapter(
             target_columns_to_types=target_columns_to_types,
             table_description=table_description,
             column_descriptions=column_descriptions,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
         table_name = (

{sqlmesh-0.211.1.dev4 → sqlmesh-0.211.2.dev2}/sqlmesh/core/engine_adapter/trino.py

@@ -55,6 +55,7 @@ class TrinoEngineAdapter(
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
     DEFAULT_CATALOG_TYPE = "hive"
     QUOTE_IDENTIFIERS_IN_VIEWS = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # default decimal precision varies across backends
@@ -357,6 +358,7 @@ class TrinoEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         super()._create_table(
@@ -368,6 +370,7 @@ class TrinoEngineAdapter(
             table_description=table_description,
             column_descriptions=column_descriptions,
             table_kind=table_kind,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )