sqlmesh 0.211.1.dev4__py3-none-any.whl → 0.211.2.dev2__py3-none-any.whl

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.211.1.dev4'
-__version_tuple__ = version_tuple = (0, 211, 1, 'dev4')
+__version__ = version = '0.211.2.dev2'
+__version_tuple__ = version_tuple = (0, 211, 2, 'dev2')
 
 __commit_id__ = commit_id = None

sqlmesh/core/console.py

@@ -7,6 +7,7 @@ import unittest
 import uuid
 import logging
 import textwrap
+from humanize import metric, naturalsize
 from itertools import zip_longest
 from pathlib import Path
 from hyperscript import h
@@ -39,6 +40,7 @@ from sqlmesh.core.snapshot import (
     SnapshotInfoLike,
 )
 from sqlmesh.core.snapshot.definition import Interval, Intervals, SnapshotTableInfo
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionStats
 from sqlmesh.core.test import ModelTest
 from sqlmesh.utils import rich as srich
 from sqlmesh.utils import Verbosity
@@ -439,6 +441,7 @@ class Console(
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Updates the snapshot evaluation progress."""
@@ -587,6 +590,7 @@ class NoopConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         pass
@@ -1032,7 +1036,9 @@ class TerminalConsole(Console):
 
             # determine column widths
             self.evaluation_column_widths["annotation"] = (
-                _calculate_annotation_str_len(batched_intervals, self.AUDIT_PADDING)
+                _calculate_annotation_str_len(
+                    batched_intervals, self.AUDIT_PADDING, len(" (123.4m rows, 123.4 KiB)")
+                )
                 + 3  # brackets and opening escape backslash
             )
             self.evaluation_column_widths["name"] = max(
@@ -1077,6 +1083,7 @@ class TerminalConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Update the snapshot evaluation progress."""
@@ -1097,7 +1104,7 @@ class TerminalConsole(Console):
                 ).ljust(self.evaluation_column_widths["name"])
 
                 annotation = _create_evaluation_model_annotation(
-                    snapshot, _format_evaluation_model_interval(snapshot, interval)
+                    snapshot, _format_evaluation_model_interval(snapshot, interval), execution_stats
                 )
                 audits_str = ""
                 if num_audits_passed:
@@ -3668,6 +3675,7 @@ class DatabricksMagicConsole(CaptureTerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         view_name, loaded_batches = self.evaluation_batch_progress[snapshot.snapshot_id]
@@ -3838,6 +3846,7 @@ class DebuggerTerminalConsole(TerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         message = f"Evaluated {snapshot.name} | batch={batch_idx} | duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
@@ -4169,33 +4178,62 @@ def _format_evaluation_model_interval(snapshot: Snapshot, interval: Interval) ->
     return ""
 
 
-def _create_evaluation_model_annotation(snapshot: Snapshot, interval_info: t.Optional[str]) -> str:
+def _create_evaluation_model_annotation(
+    snapshot: Snapshot,
+    interval_info: t.Optional[str],
+    execution_stats: t.Optional[QueryExecutionStats],
+) -> str:
+    annotation = None
+    execution_stats_str = ""
+    if execution_stats:
+        rows_processed = execution_stats.total_rows_processed
+        if rows_processed:
+            # 1.00 and 1.0 to 1
+            rows_processed_str = metric(rows_processed).replace(".00", "").replace(".0", "")
+            execution_stats_str += f"{rows_processed_str} row{'s' if rows_processed > 1 else ''}"
+
+        bytes_processed = execution_stats.total_bytes_processed
+        execution_stats_str += (
+            f"{', ' if execution_stats_str else ''}{naturalsize(bytes_processed, binary=True)}"
+            if bytes_processed
+            else ""
+        )
+    execution_stats_str = f" ({execution_stats_str})" if execution_stats_str else ""
+
     if snapshot.is_audit:
-        return "run standalone audit"
-    if snapshot.is_model and snapshot.model.kind.is_external:
-        return "run external audits"
-    if snapshot.model.kind.is_seed:
-        return "insert seed file"
-    if snapshot.model.kind.is_full:
-        return "full refresh"
-    if snapshot.model.kind.is_view:
-        return "recreate view"
-    if snapshot.model.kind.is_incremental_by_unique_key:
-        return "insert/update rows"
-    if snapshot.model.kind.is_incremental_by_partition:
-        return "insert partitions"
-
-    return interval_info if interval_info else ""
-
-
-def _calculate_interval_str_len(snapshot: Snapshot, intervals: t.List[Interval]) -> int:
+        annotation = "run standalone audit"
+    if snapshot.is_model:
+        if snapshot.model.kind.is_external:
+            annotation = "run external audits"
+        if snapshot.model.kind.is_view:
+            annotation = "recreate view"
+        if snapshot.model.kind.is_seed:
+            annotation = f"insert seed file{execution_stats_str}"
+        if snapshot.model.kind.is_full:
+            annotation = f"full refresh{execution_stats_str}"
+        if snapshot.model.kind.is_incremental_by_unique_key:
+            annotation = f"insert/update rows{execution_stats_str}"
+        if snapshot.model.kind.is_incremental_by_partition:
+            annotation = f"insert partitions{execution_stats_str}"
+
+    if annotation:
+        return annotation
+
+    return f"{interval_info}{execution_stats_str}" if interval_info else ""
+
+
+def _calculate_interval_str_len(
+    snapshot: Snapshot,
+    intervals: t.List[Interval],
+    execution_stats: t.Optional[QueryExecutionStats] = None,
+) -> int:
     interval_str_len = 0
     for interval in intervals:
         interval_str_len = max(
             interval_str_len,
             len(
                 _create_evaluation_model_annotation(
-                    snapshot, _format_evaluation_model_interval(snapshot, interval)
+                    snapshot, _format_evaluation_model_interval(snapshot, interval), execution_stats
                 )
             ),
         )
@@ -4248,13 +4286,16 @@ def _calculate_audit_str_len(snapshot: Snapshot, audit_padding: int = 0) -> int:
 
 
 def _calculate_annotation_str_len(
-    batched_intervals: t.Dict[Snapshot, t.List[Interval]], audit_padding: int = 0
+    batched_intervals: t.Dict[Snapshot, t.List[Interval]],
+    audit_padding: int = 0,
+    execution_stats_len: int = 0,
 ) -> int:
     annotation_str_len = 0
     for snapshot, intervals in batched_intervals.items():
         annotation_str_len = max(
             annotation_str_len,
             _calculate_interval_str_len(snapshot, intervals)
-            + _calculate_audit_str_len(snapshot, audit_padding),
+            + _calculate_audit_str_len(snapshot, audit_padding)
+            + execution_stats_len,
         )
     return annotation_str_len

sqlmesh/core/engine_adapter/athena.py

@@ -45,6 +45,7 @@ class AthenaEngineAdapter(PandasNativeFetchDFSupportMixin, RowDiffMixin):
     # >>> self._execute('/* test */ DESCRIBE foo')
     #     pyathena.error.OperationalError: FAILED: ParseException line 1:0 cannot recognize input near '/' '*' 'test'
     ATTACH_CORRELATION_ID = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["DATABASE", "SCHEMA"]
 
     def __init__(

sqlmesh/core/engine_adapter/base.py

@@ -40,6 +40,7 @@ from sqlmesh.core.engine_adapter.shared import (
 )
 from sqlmesh.core.model.kind import TimeColumn
 from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionTracker
 from sqlmesh.utils import (
     CorrelationId,
     columns_to_types_all_known,
@@ -117,6 +118,7 @@ class EngineAdapter:
     QUOTE_IDENTIFIERS_IN_VIEWS = True
     MAX_IDENTIFIER_LENGTH: t.Optional[int] = None
     ATTACH_CORRELATION_ID = True
+    SUPPORTS_QUERY_EXECUTION_TRACKING = False
 
     def __init__(
         self,
@@ -133,6 +135,7 @@ class EngineAdapter:
         shared_connection: bool = False,
         correlation_id: t.Optional[CorrelationId] = None,
         schema_differ_overrides: t.Optional[t.Dict[str, t.Any]] = None,
+        query_execution_tracker: t.Optional[QueryExecutionTracker] = None,
         **kwargs: t.Any,
     ):
         self.dialect = dialect.lower() or self.DIALECT
@@ -156,11 +159,16 @@ class EngineAdapter:
         self._multithreaded = multithreaded
         self.correlation_id = correlation_id
         self._schema_differ_overrides = schema_differ_overrides
+        self._query_execution_tracker = query_execution_tracker
 
     def with_settings(self, **kwargs: t.Any) -> EngineAdapter:
         extra_kwargs = {
             "null_connection": True,
             "execute_log_level": kwargs.pop("execute_log_level", self._execute_log_level),
+            "correlation_id": kwargs.pop("correlation_id", self.correlation_id),
+            "query_execution_tracker": kwargs.pop(
+                "query_execution_tracker", self._query_execution_tracker
+            ),
             **self._extra_config,
             **kwargs,
         }
@@ -854,6 +862,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table = exp.to_table(table_name)
@@ -899,11 +908,15 @@ class EngineAdapter:
                             replace=replace,
                             table_description=table_description,
                             table_kind=table_kind,
+                            track_rows_processed=track_rows_processed,
                             **kwargs,
                         )
                     else:
                         self._insert_append_query(
-                            table_name, query, target_columns_to_types or self.columns(table)
+                            table_name,
+                            query,
+                            target_columns_to_types or self.columns(table),
+                            track_rows_processed=track_rows_processed,
                         )
 
         # Register comments with commands if the engine supports comments and we weren't able to
@@ -927,6 +940,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         self.execute(
@@ -943,7 +957,8 @@ class EngineAdapter:
                 ),
                 table_kind=table_kind,
                 **kwargs,
-            )
+            ),
+            track_rows_processed=track_rows_processed,
         )
 
     def _build_create_table_exp(
@@ -1431,6 +1446,7 @@ class EngineAdapter:
         table_name: TableName,
         query_or_df: QueryOrDF,
         target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        track_rows_processed: bool = True,
         source_columns: t.Optional[t.List[str]] = None,
     ) -> None:
         source_queries, target_columns_to_types = self._get_source_queries_and_columns_to_types(
@@ -1439,19 +1455,27 @@ class EngineAdapter:
             target_table=table_name,
             source_columns=source_columns,
         )
-        self._insert_append_source_queries(table_name, source_queries, target_columns_to_types)
+        self._insert_append_source_queries(
+            table_name, source_queries, target_columns_to_types, track_rows_processed
+        )
 
     def _insert_append_source_queries(
         self,
         table_name: TableName,
         source_queries: t.List[SourceQuery],
         target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        track_rows_processed: bool = True,
     ) -> None:
         with self.transaction(condition=len(source_queries) > 0):
             target_columns_to_types = target_columns_to_types or self.columns(table_name)
             for source_query in source_queries:
                 with source_query as query:
-                    self._insert_append_query(table_name, query, target_columns_to_types)
+                    self._insert_append_query(
+                        table_name,
+                        query,
+                        target_columns_to_types,
+                        track_rows_processed=track_rows_processed,
+                    )
 
     def _insert_append_query(
         self,
@@ -1459,10 +1483,14 @@ class EngineAdapter:
         query: Query,
         target_columns_to_types: t.Dict[str, exp.DataType],
         order_projections: bool = True,
+        track_rows_processed: bool = True,
     ) -> None:
         if order_projections:
             query = self._order_projections_and_filter(query, target_columns_to_types)
-        self.execute(exp.insert(query, table_name, columns=list(target_columns_to_types)))
+        self.execute(
+            exp.insert(query, table_name, columns=list(target_columns_to_types)),
+            track_rows_processed=track_rows_processed,
+        )
 
     def insert_overwrite_by_partition(
         self,
@@ -1604,7 +1632,7 @@ class EngineAdapter:
                         )
                         if insert_overwrite_strategy.is_replace_where:
                             insert_exp.set("where", where or exp.true())
-                        self.execute(insert_exp)
+                        self.execute(insert_exp, track_rows_processed=True)
 
     def update_table(
         self,
@@ -1625,7 +1653,9 @@ class EngineAdapter:
         using = exp.alias_(
             exp.Subquery(this=query), alias=MERGE_SOURCE_ALIAS, copy=False, table=True
         )
-        self.execute(exp.Merge(this=this, using=using, on=on, whens=whens))
+        self.execute(
+            exp.Merge(this=this, using=using, on=on, whens=whens), track_rows_processed=True
+        )
 
     def scd_type_2_by_time(
         self,
@@ -2374,6 +2404,7 @@ class EngineAdapter:
         expressions: t.Union[str, exp.Expression, t.Sequence[exp.Expression]],
         ignore_unsupported_errors: bool = False,
         quote_identifiers: bool = True,
+        track_rows_processed: bool = False,
         **kwargs: t.Any,
     ) -> None:
         """Execute a sql query."""
@@ -2395,7 +2426,7 @@ class EngineAdapter:
                     expression=e if isinstance(e, exp.Expression) else None,
                     quote_identifiers=quote_identifiers,
                 )
-                self._execute(sql, **kwargs)
+                self._execute(sql, track_rows_processed, **kwargs)
 
     def _attach_correlation_id(self, sql: str) -> str:
         if self.ATTACH_CORRELATION_ID and self.correlation_id:
@@ -2420,9 +2451,29 @@ class EngineAdapter:
 
         logger.log(self._execute_log_level, "Executing SQL: %s", sql_to_log)
 
-    def _execute(self, sql: str, **kwargs: t.Any) -> None:
+    def _record_execution_stats(
+        self, sql: str, rowcount: t.Optional[int] = None, bytes_processed: t.Optional[int] = None
+    ) -> None:
+        if self._query_execution_tracker:
+            self._query_execution_tracker.record_execution(sql, rowcount, bytes_processed)
+
+    def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any) -> None:
         self.cursor.execute(sql, **kwargs)
 
+        if (
+            self.SUPPORTS_QUERY_EXECUTION_TRACKING
+            and track_rows_processed
+            and self._query_execution_tracker
+            and self._query_execution_tracker.is_tracking()
+        ):
+            if (
+                rowcount := getattr(self.cursor, "rowcount", None)
+            ) is not None and rowcount is not None:
+                try:
+                    self._record_execution_stats(sql, int(rowcount))
+                except (TypeError, ValueError):
+                    return
+
     @contextlib.contextmanager
     def temp_table(
         self,
@@ -2467,6 +2518,7 @@ class EngineAdapter:
                 exists=True,
                 table_description=None,
                 column_descriptions=None,
+                track_rows_processed=False,
                 **kwargs,
             )
 
@@ -2718,7 +2770,7 @@ class EngineAdapter:
                     insert_statement.set("where", delete_filter)
                     insert_statement.set("this", exp.to_table(target_table))
 
-                self.execute(insert_statement)
+                self.execute(insert_statement, track_rows_processed=True)
             finally:
                 self.drop_table(temp_table)
 

sqlmesh/core/engine_adapter/base_postgres.py

@@ -24,6 +24,7 @@ class BasePostgresEngineAdapter(EngineAdapter):
     DEFAULT_BATCH_SIZE = 400
     COMMENT_CREATION_TABLE = CommentCreationTable.COMMENT_COMMAND_ONLY
     COMMENT_CREATION_VIEW = CommentCreationView.COMMENT_COMMAND_ONLY
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA", "TABLE", "VIEW"]
 
     def columns(

sqlmesh/core/engine_adapter/bigquery.py

@@ -66,6 +66,7 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     SUPPORTS_CLONING = True
     MAX_TABLE_COMMENT_LENGTH = 1024
     MAX_COLUMN_COMMENT_LENGTH = 1024
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
 
     SCHEMA_DIFFER_KWARGS = {
@@ -1049,6 +1050,7 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     def _execute(
         self,
         sql: str,
+        track_rows_processed: bool = False,
         **kwargs: t.Any,
     ) -> None:
         """Execute a sql query."""
@@ -1094,6 +1096,23 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
         self.cursor._set_rowcount(query_results)
         self.cursor._set_description(query_results.schema)
 
+        if (
+            track_rows_processed
+            and self._query_execution_tracker
+            and self._query_execution_tracker.is_tracking()
+        ):
+            num_rows = None
+            if query_job.statement_type == "CREATE_TABLE_AS_SELECT":
+                # since table was just created, number rows in table == number rows processed
+                query_table = self.client.get_table(query_job.destination)
+                num_rows = query_table.num_rows
+            elif query_job.statement_type in ["INSERT", "DELETE", "MERGE", "UPDATE"]:
+                num_rows = query_job.num_dml_affected_rows
+
+            self._query_execution_tracker.record_execution(
+                sql, num_rows, query_job.total_bytes_processed
+            )
+
     def _get_data_objects(
         self, schema_name: SchemaName, object_names: t.Optional[t.Set[str]] = None
     ) -> t.List[DataObject]:

sqlmesh/core/engine_adapter/clickhouse.py

@@ -294,7 +294,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
                     )
 
                 try:
-                    self.execute(existing_records_insert_exp)
+                    self.execute(existing_records_insert_exp, track_rows_processed=True)
                 finally:
                     if table_partition_exp:
                         self.drop_table(partitions_temp_table_name)
@@ -489,6 +489,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         """Creates a table in the database.
@@ -525,6 +526,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
             column_descriptions,
             table_kind,
             empty_ctas=(self.engine_run_mode.is_cloud and expression is not None),
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
 

sqlmesh/core/engine_adapter/duckdb.py

@@ -170,6 +170,7 @@ class DuckDBEngineAdapter(LogicalMergeMixin, GetCurrentCatalogFromFunctionMixin,
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         catalog = self.get_current_catalog()
@@ -193,6 +194,7 @@ class DuckDBEngineAdapter(LogicalMergeMixin, GetCurrentCatalogFromFunctionMixin,
             table_description,
             column_descriptions,
             table_kind,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
 

sqlmesh/core/engine_adapter/mssql.py

@@ -53,6 +53,7 @@ class MSSQLEngineAdapter(
     COMMENT_CREATION_TABLE = CommentCreationTable.UNSUPPORTED
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_REPLACE_TABLE = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(18, 0), (0,)],

sqlmesh/core/engine_adapter/mysql.py

@@ -39,6 +39,7 @@ class MySQLEngineAdapter(
     MAX_COLUMN_COMMENT_LENGTH = 1024
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 64
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             exp.DataType.build("BIT", dialect=DIALECT).this: [(1,)],

sqlmesh/core/engine_adapter/postgres.py

@@ -35,6 +35,7 @@ class PostgresEngineAdapter(
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
     MAX_IDENTIFIER_LENGTH = 63
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"

sqlmesh/core/engine_adapter/redshift.py

@@ -173,6 +173,7 @@ class RedshiftEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         """
@@ -426,7 +427,8 @@ class RedshiftEngineAdapter(
                 using=using,
                 on=on.transform(resolve_target_table),
                 whens=whens.transform(resolve_target_table),
-            )
+            ),
+            track_rows_processed=True,
         )
 
     def _normalize_decimal_value(self, expr: exp.Expression, precision: int) -> exp.Expression:

sqlmesh/core/engine_adapter/snowflake.py

@@ -72,6 +72,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     }
     MANAGED_TABLE_KIND = "DYNAMIC TABLE"
     SNOWPARK = "snowpark"
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
 
     @contextlib.contextmanager
     def session(self, properties: SessionProperties) -> t.Iterator[None]:
@@ -166,6 +167,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table_format = kwargs.get("table_format")
@@ -185,6 +187,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             table_description=table_description,
             column_descriptions=column_descriptions,
             table_kind=table_kind,
+            track_rows_processed=False,  # snowflake tracks CTAS row counts incorrectly
             **kwargs,
         )
 

sqlmesh/core/engine_adapter/spark.py

@@ -433,6 +433,7 @@ class SparkEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         table_name = (
@@ -461,6 +462,7 @@ class SparkEngineAdapter(
             target_columns_to_types=target_columns_to_types,
             table_description=table_description,
             column_descriptions=column_descriptions,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
         table_name = (

sqlmesh/core/engine_adapter/trino.py

@@ -55,6 +55,7 @@ class TrinoEngineAdapter(
     SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
     DEFAULT_CATALOG_TYPE = "hive"
     QUOTE_IDENTIFIERS_IN_VIEWS = False
+    SUPPORTS_QUERY_EXECUTION_TRACKING = True
     SCHEMA_DIFFER_KWARGS = {
         "parameterized_type_defaults": {
             # default decimal precision varies across backends
@@ -357,6 +358,7 @@ class TrinoEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         table_kind: t.Optional[str] = None,
+        track_rows_processed: bool = True,
         **kwargs: t.Any,
     ) -> None:
         super()._create_table(
@@ -368,6 +370,7 @@ class TrinoEngineAdapter(
             table_description=table_description,
             column_descriptions=column_descriptions,
             table_kind=table_kind,
+            track_rows_processed=track_rows_processed,
             **kwargs,
         )
 

sqlmesh/core/scheduler.py

@@ -20,6 +20,7 @@ from sqlmesh.core.snapshot import (
     DeployabilityIndex,
     Snapshot,
     SnapshotId,
+    SnapshotIdBatch,
     SnapshotEvaluator,
     apply_auto_restatements,
     earliest_start_date,
@@ -531,6 +532,11 @@ class Scheduler:
                 finally:
                     num_audits = len(audit_results)
                     num_audits_failed = sum(1 for result in audit_results if result.count)
+
+                    execution_stats = self.snapshot_evaluator.execution_tracker.get_execution_stats(
+                        SnapshotIdBatch(snapshot_id=snapshot.snapshot_id, batch_id=node.batch_index)
+                    )
+
                     self.console.update_snapshot_evaluation_progress(
                         snapshot,
                         batched_intervals[snapshot][node.batch_index],
@@ -538,6 +544,7 @@ class Scheduler:
                         evaluation_duration_ms,
                         num_audits - num_audits_failed,
                         num_audits_failed,
+                        execution_stats=execution_stats,
                         auto_restatement_triggers=auto_restatement_triggers.get(
                             snapshot.snapshot_id
                         ),

sqlmesh/core/snapshot/__init__.py

@@ -8,6 +8,7 @@ from sqlmesh.core.snapshot.definition import (
     SnapshotDataVersion as SnapshotDataVersion,
     SnapshotFingerprint as SnapshotFingerprint,
     SnapshotId as SnapshotId,
+    SnapshotIdBatch as SnapshotIdBatch,
     SnapshotIdLike as SnapshotIdLike,
     SnapshotInfoLike as SnapshotInfoLike,
     SnapshotIntervals as SnapshotIntervals,

sqlmesh/core/snapshot/definition.py

@@ -13,10 +13,13 @@ from pydantic import Field
 from sqlglot import exp
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 
-from sqlmesh.core.config.common import TableNamingConvention, VirtualEnvironmentMode
+from sqlmesh.core.config.common import (
+    TableNamingConvention,
+    VirtualEnvironmentMode,
+    EnvironmentSuffixTarget,
+)
 from sqlmesh.core import constants as c
 from sqlmesh.core.audit import StandaloneAudit
-from sqlmesh.core.environment import EnvironmentSuffixTarget
 from sqlmesh.core.macros import call_macro
 from sqlmesh.core.model import Model, ModelKindMixin, ModelKindName, ViewKind, CustomKind
 from sqlmesh.core.model.definition import _Model
@@ -159,6 +162,11 @@ class SnapshotId(PydanticModel, frozen=True):
         return f"SnapshotId<{self.name}: {self.identifier}>"
 
 
+class SnapshotIdBatch(PydanticModel, frozen=True):
+    snapshot_id: SnapshotId
+    batch_id: int
+
+
 class SnapshotNameVersion(PydanticModel, frozen=True):
     name: str
     version: str

sqlmesh/core/snapshot/evaluator.py

@@ -37,7 +37,6 @@ from sqlmesh.core import constants as c
 from sqlmesh.core import dialect as d
 from sqlmesh.core.audit import Audit, StandaloneAudit
 from sqlmesh.core.dialect import schema_
-from sqlmesh.core.engine_adapter import EngineAdapter
 from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType
 from sqlmesh.core.macros import RuntimeStage
 from sqlmesh.core.model import (
@@ -62,9 +61,11 @@ from sqlmesh.core.snapshot import (
     Intervals,
     Snapshot,
     SnapshotId,
+    SnapshotIdBatch,
     SnapshotInfoLike,
     SnapshotTableCleanupTask,
 )
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionTracker
 from sqlmesh.utils import random_id, CorrelationId
 from sqlmesh.utils.concurrency import (
     concurrent_apply_to_snapshots,
@@ -88,6 +89,7 @@ else:
 
 if t.TYPE_CHECKING:
     from sqlmesh.core.engine_adapter._typing import DF, QueryOrDF
+    from sqlmesh.core.engine_adapter.base import EngineAdapter
     from sqlmesh.core.environment import EnvironmentNamingInfo
 
 logger = logging.getLogger(__name__)
@@ -128,6 +130,11 @@ class SnapshotEvaluator:
         self.adapters = (
             adapters if isinstance(adapters, t.Dict) else {selected_gateway or "": adapters}
         )
+        self.execution_tracker = QueryExecutionTracker()
+        self.adapters = {
+            gateway: adapter.with_settings(query_execution_tracker=self.execution_tracker)
+            for gateway, adapter in self.adapters.items()
+        }
         self.adapter = (
             next(iter(self.adapters.values()))
             if not selected_gateway
@@ -169,19 +176,22 @@ class SnapshotEvaluator:
         Returns:
             The WAP ID of this evaluation if supported, None otherwise.
         """
-        result = self._evaluate_snapshot(
-            start=start,
-            end=end,
-            execution_time=execution_time,
-            snapshot=snapshot,
-            snapshots=snapshots,
-            allow_destructive_snapshots=allow_destructive_snapshots or set(),
-            allow_additive_snapshots=allow_additive_snapshots or set(),
-            deployability_index=deployability_index,
-            batch_index=batch_index,
-            target_table_exists=target_table_exists,
-            **kwargs,
-        )
+        with self.execution_tracker.track_execution(
+            SnapshotIdBatch(snapshot_id=snapshot.snapshot_id, batch_id=batch_index)
+        ):
+            result = self._evaluate_snapshot(
+                start=start,
+                end=end,
+                execution_time=execution_time,
+                snapshot=snapshot,
+                snapshots=snapshots,
+                allow_destructive_snapshots=allow_destructive_snapshots or set(),
+                allow_additive_snapshots=allow_additive_snapshots or set(),
+                deployability_index=deployability_index,
+                batch_index=batch_index,
+                target_table_exists=target_table_exists,
+                **kwargs,
+            )
         if result is None or isinstance(result, str):
             return result
         raise SQLMeshError(

sqlmesh/core/snapshot/execution_tracker.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import typing as t
+from contextlib import contextmanager
+from threading import local
+from dataclasses import dataclass, field
+from sqlmesh.core.snapshot import SnapshotIdBatch
+
+
+@dataclass
+class QueryExecutionStats:
+    snapshot_id_batch: SnapshotIdBatch
+    total_rows_processed: t.Optional[int] = None
+    total_bytes_processed: t.Optional[int] = None
+
+
+@dataclass
+class QueryExecutionContext:
+    """
+    Container for tracking rows processed or other execution information during snapshot evaluation.
+
+    It accumulates statistics from multiple cursor.execute() calls during a single snapshot evaluation.
+
+    Attributes:
+        snapshot_id_batch: Identifier linking this context to a specific snapshot evaluation
+        stats: Running sum of cursor.rowcount and possibly bytes processed from all executed queries during evaluation
+    """
+
+    snapshot_id_batch: SnapshotIdBatch
+    stats: QueryExecutionStats = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.stats = QueryExecutionStats(snapshot_id_batch=self.snapshot_id_batch)
+
+    def add_execution(
+        self, sql: str, row_count: t.Optional[int], bytes_processed: t.Optional[int]
+    ) -> None:
+        if row_count is not None and row_count >= 0:
+            if self.stats.total_rows_processed is None:
+                self.stats.total_rows_processed = row_count
+            else:
+                self.stats.total_rows_processed += row_count
+
+            # conditional on row_count because we should only count bytes corresponding to
+            # DML actions whose rows were captured
+            if bytes_processed is not None:
+                if self.stats.total_bytes_processed is None:
+                    self.stats.total_bytes_processed = bytes_processed
+                else:
+                    self.stats.total_bytes_processed += bytes_processed
+
+    def get_execution_stats(self) -> QueryExecutionStats:
+        return self.stats
+
+
+class QueryExecutionTracker:
+    """Thread-local context manager for snapshot execution statistics, such as rows processed."""
+
+    def __init__(self) -> None:
+        self._thread_local = local()
+        self._contexts: t.Dict[SnapshotIdBatch, QueryExecutionContext] = {}
+
+    def get_execution_context(
+        self, snapshot_id_batch: SnapshotIdBatch
+    ) -> t.Optional[QueryExecutionContext]:
+        return self._contexts.get(snapshot_id_batch)
+
+    def is_tracking(self) -> bool:
+        return getattr(self._thread_local, "context", None) is not None
+
+    @contextmanager
+    def track_execution(
+        self, snapshot_id_batch: SnapshotIdBatch
+    ) -> t.Iterator[t.Optional[QueryExecutionContext]]:
+        """Context manager for tracking snapshot execution statistics such as row counts and bytes processed."""
+        context = QueryExecutionContext(snapshot_id_batch=snapshot_id_batch)
+        self._thread_local.context = context
+        self._contexts[snapshot_id_batch] = context
+
+        try:
+            yield context
+        finally:
+            self._thread_local.context = None
+
+    def record_execution(
+        self, sql: str, row_count: t.Optional[int], bytes_processed: t.Optional[int]
+    ) -> None:
+        context = getattr(self._thread_local, "context", None)
+        if context is not None:
+            context.add_execution(sql, row_count, bytes_processed)
+
+    def get_execution_stats(
+        self, snapshot_id_batch: SnapshotIdBatch
+    ) -> t.Optional[QueryExecutionStats]:
+        context = self._contexts.get(snapshot_id_batch)
+        self._contexts.pop(snapshot_id_batch, None)
+        return context.get_execution_stats() if context else None

sqlmesh/core/state_sync/db/environment.py

@@ -78,6 +78,7 @@ class EnvironmentState:
             self.environments_table,
             _environment_to_df(environment),
             target_columns_to_types=self._environment_columns_to_types,
+            track_rows_processed=False,
         )
 
     def update_environment_statements(
@@ -108,6 +109,7 @@ class EnvironmentState:
                 self.environment_statements_table,
                 _environment_statements_to_df(environment_name, plan_id, environment_statements),
                 target_columns_to_types=self._environment_statements_columns_to_types,
+                track_rows_processed=False,
             )
 
     def invalidate_environment(self, name: str, protect_prod: bool = True) -> None:

sqlmesh/core/state_sync/db/interval.py

@@ -115,6 +115,7 @@ class IntervalState:
             self.intervals_table,
             _intervals_to_df(intervals_to_remove, is_dev=False, is_removed=True),
             target_columns_to_types=self._interval_columns_to_types,
+            track_rows_processed=False,
         )
 
     def get_snapshot_intervals(
@@ -243,6 +244,7 @@ class IntervalState:
                 self.intervals_table,
                 pd.DataFrame(new_intervals),
                 target_columns_to_types=self._interval_columns_to_types,
+                track_rows_processed=False,
             )
 
     def _get_snapshot_intervals(

sqlmesh/core/state_sync/db/migrator.py

@@ -413,7 +413,9 @@ class StateMigrator:
                     backup_name = _backup_table_name(table)
                     self.engine_adapter.drop_table(backup_name)
                     self.engine_adapter.create_table_like(backup_name, table)
-                    self.engine_adapter.insert_append(backup_name, exp.select("*").from_(table))
+                    self.engine_adapter.insert_append(
+                        backup_name, exp.select("*").from_(table), track_rows_processed=False
+                    )
 
     def _restore_table(
         self,

sqlmesh/core/state_sync/db/snapshot.py

@@ -103,6 +103,7 @@ class SnapshotState:
             self.snapshots_table,
             _snapshots_to_df(snapshots_to_store),
             target_columns_to_types=self._snapshot_columns_to_types,
+            track_rows_processed=False,
         )
 
         for snapshot in snapshots:
@@ -406,6 +407,7 @@ class SnapshotState:
             self.snapshots_table,
             _snapshots_to_df(snapshots_to_store),
             target_columns_to_types=self._snapshot_columns_to_types,
+            track_rows_processed=False,
         )
 
     def _get_snapshots(

sqlmesh/core/state_sync/db/version.py

@@ -55,6 +55,7 @@ class VersionState:
                 ]
             ),
             target_columns_to_types=self._version_columns_to_types,
+            track_rows_processed=False,
         )
 
     def get_versions(self) -> Versions:

{sqlmesh-0.211.1.dev4.dist-info → sqlmesh-0.211.2.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.211.1.dev4
+Version: 0.211.2.dev2
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -223,6 +223,7 @@ Requires-Dist: click
 Requires-Dist: croniter
 Requires-Dist: duckdb!=0.10.3,>=0.10.0
 Requires-Dist: dateparser<=1.2.1
+Requires-Dist: humanize
 Requires-Dist: hyperscript>=0.1.0
 Requires-Dist: importlib-metadata; python_version < "3.12"
 Requires-Dist: ipywidgets

{sqlmesh-0.211.1.dev4.dist-info → sqlmesh-0.211.2.dev2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=qYxwtlsmdtANLFoBZXS_Mq7mAhgSXOczqQ3GgApnbwU,721
+sqlmesh/_version.py,sha256=_SX7vdymMYLUIT2uNWUZp-U7rXKSg-_tOmix0J3KAwA,721
 sqlmesh/magics.py,sha256=xLh3u4eqpVrKRVN5KF3X84RPRqjygAB9AJP1TXwH8hg,42086
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -11,7 +11,7 @@ sqlmesh/cli/options.py,sha256=8qEGWivk4urGikpEmaU9T8WsyaZtjCpmW4Ed25iQ73I,2543
 sqlmesh/cli/project_init.py,sha256=-Fhc-wAbKmjo9Ktog2i96nvITFwD4qDqaaiuqONkM5c,17204
 sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
-sqlmesh/core/console.py,sha256=TclHdPBzufeuM546TcLs24FGupIuJBUN3vqjQFu0sJ0,167042
+sqlmesh/core/console.py,sha256=mgUrgbwjuR0HEZsXkPBtfpjeKzmFjgWVqJOcROyI74E,168778
 sqlmesh/core/constants.py,sha256=Wg6qApCRhENA1me7ELUS5GnArKlTBH-8192gQYKQnE8,2759
 sqlmesh/core/context.py,sha256=q9n9e8JSpgnrkrJCLzKyipK5BNzJ8cPQuDsAyt3MPwc,128717
 sqlmesh/core/context_diff.py,sha256=pfMcdcV-8fdxkD68dv-c1EYuN-rfSvy3wriEMYSsa3o,21848
@@ -24,7 +24,7 @@ sqlmesh/core/node.py,sha256=3ChBz1PIMJhp-jzed_GLBrtmEF2v_BVDEM5WNe0ssPk,14658
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
 sqlmesh/core/renderer.py,sha256=PMUmmtwqk3FA5lN-16OWl6buCbrVa1M8NIgRXCYEB8k,26805
-sqlmesh/core/scheduler.py,sha256=TyRPooWRDIMxZrE44msckQSwUFaPElsXKzF5fURtj1g,46806
+sqlmesh/core/scheduler.py,sha256=-RDcvwmU2VhvpnnFYT7KcQMe3VK_thsNzJ1i35c5Kfc,47110
 sqlmesh/core/schema_diff.py,sha256=QRQfiVuptTfDAVf7zkuGo7beqXKRno3MUvRjYB6q0nk,33702
 sqlmesh/core/schema_loader.py,sha256=_Pq2RSw91uthqv1vNi_eHmLlzhtGz_APMJ0wAJZYuvk,3677
 sqlmesh/core/selector.py,sha256=9bqtBx7Onl8zZb-G4OsGVYVwkdm24j_qnp5HgXLFhR4,12837
@@ -57,24 +57,24 @@ sqlmesh/core/config/scheduler.py,sha256=LVEHIln7dqv8ygo4uniji4j9-gfuJlk-WpQU9FAq
 sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=y9jZAFdMBkkkRrf0ymfsJJn6s_7Ya6OpDgR4Bf1OG_U,2383
 sqlmesh/core/engine_adapter/_typing.py,sha256=QSXXdxtTHXTwJyiv-9ArAJHvwmJPfa7tShuD_3vGwfg,1010
-sqlmesh/core/engine_adapter/athena.py,sha256=UlZTHy1bGq0Wqzp9hmbphCpL-P6duOALNGNBSNEHYnk,26756
-sqlmesh/core/engine_adapter/base.py,sha256=Nzd6Rg--W0BGuEgWVxgAfarabc-HPJcR9gUQXZjj16g,113607
-sqlmesh/core/engine_adapter/base_postgres.py,sha256=U4Y_0-M6_0_RcKbljX57DIUSwIa70w02k-UoDOnWPxQ,7016
-sqlmesh/core/engine_adapter/bigquery.py,sha256=dtzRk6BlCLJxnwXeoz1ANAXbIrhtiJEAruasHzJYj0M,53935
-sqlmesh/core/engine_adapter/clickhouse.py,sha256=yBVEorfoaHp-O0qi3UXOnGlDwM7_Yo6lw6pOUMEjwB4,35854
+sqlmesh/core/engine_adapter/athena.py,sha256=-HysZHXI9vETmqw6s88sYTUD7_z7LiAiMmi6nulqri0,26801
+sqlmesh/core/engine_adapter/base.py,sha256=xBIBu4S1bhiRLR4Bj2KWETWiZzy5wk82P9tqbiBOCNo,115889
+sqlmesh/core/engine_adapter/base_postgres.py,sha256=V4NwGs1IxOpRDjrDeh16A8TZdkHYfJEoJyVWW96xYQU,7061
+sqlmesh/core/engine_adapter/bigquery.py,sha256=e66jX3T885y5ADytYVNfqH8pAPWe4_j8CJxeQXaswb8,54785
+sqlmesh/core/engine_adapter/clickhouse.py,sha256=Lxtsz-8EoVGPEZpmGbfoU5VXQromkM-6h3QHALdAQ9k,35979
 sqlmesh/core/engine_adapter/databricks.py,sha256=B6OkHelGrxxehZlYno5cZ8MB_bHirQS_imznSZdQGrY,14589
-sqlmesh/core/engine_adapter/duckdb.py,sha256=AyG8mpL1f2cfv4a-_mO_XqJEjMpQZlnLTw4hKhijcvU,7895
+sqlmesh/core/engine_adapter/duckdb.py,sha256=9AXeRhaYXBcYSmIavyFY9LUzfgh94qkTO98v0-suQ8I,7993
 sqlmesh/core/engine_adapter/fabric.py,sha256=nXDYpKtzTN8WgBqOEmSLct7oyXNR3HwMZgOL3OzMvyo,14009
 sqlmesh/core/engine_adapter/mixins.py,sha256=JoITl_xbY5io5LtiTenqEhjWE9VCv2cK4Ue025Bkm7Y,23607
-sqlmesh/core/engine_adapter/mssql.py,sha256=xLX15SrL-2ukv_YLy9a_4g9pxPga-1WcwxWUBIFY-f4,18225
-sqlmesh/core/engine_adapter/mysql.py,sha256=071qYHeJ_7gDdC7YWJoRCorjhxGAVkhtqWL-KetfPCg,6914
-sqlmesh/core/engine_adapter/postgres.py,sha256=v6fD7ci22-DrZBtry_03Lfq5m7SNgSPq5X_BZ1dWLhk,5083
-sqlmesh/core/engine_adapter/redshift.py,sha256=bfrgxY3Fqcbz5inAF4cxOQCmRdfuC--aPu5U3ks29LA,17494
+sqlmesh/core/engine_adapter/mssql.py,sha256=oUKTUvXUzLxqtrtDsnyOJjV6p6R6pkiVIhL2WmKMs9k,18270
+sqlmesh/core/engine_adapter/mysql.py,sha256=cvWN0HXHxhxxIUHZ6syV8mfpNc3MjQget0SR26qVwSY,6959
+sqlmesh/core/engine_adapter/postgres.py,sha256=prSV9trLqzmFGnquoqfmR20ZtKmAtVvfP8rX6E8ofHA,5128
+sqlmesh/core/engine_adapter/redshift.py,sha256=lWPX1_u6HNUhPgq0WcWlURcgKqidJpoasBV1baPmp3E,17577
 sqlmesh/core/engine_adapter/risingwave.py,sha256=GY0u085TWerl8kTRNJpEutdD5FBsoPijQ0CcOkWOJEE,2359
 sqlmesh/core/engine_adapter/shared.py,sha256=pvYU-sC2WEIOENm4oG4UrOCWXF56h-FEmlAqAR8TYoA,13500
-sqlmesh/core/engine_adapter/snowflake.py,sha256=5vlEaDRROI4pUHCkkRUNMCl0-R7p24Q5goJZ-FmJ9EQ,30004
-sqlmesh/core/engine_adapter/spark.py,sha256=YTFLBBF7Io6iZAhfsyzqtaqxddCCHxt0J9XiclUalf0,22368
-sqlmesh/core/engine_adapter/trino.py,sha256=euroG7zMlQyv8hlo7SBo-deZuj1gti5jRy-9zToNTGE,16289
+sqlmesh/core/engine_adapter/snowflake.py,sha256=mg1vMwGtXB5fmO_HSm290RsYR0o8zT-QLbNKNCjuTfM,30180
+sqlmesh/core/engine_adapter/spark.py,sha256=EqqFIZmFabdha_Zba2UW1NY77pfn9_bqTP0wvN7Y7Mw,22466
+sqlmesh/core/engine_adapter/trino.py,sha256=_aFxm6GVsd9pZY2CBh_JcZW1gULMndnr91O_EiZHyEE,16432
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=JDJTkhByUw4Tr3nF-BOHKFAx_eASQqEqzX6AUVQUz8I,5659
 sqlmesh/core/linter/helpers.py,sha256=cwKXP4sL6azRtNVGbMfJ5_6Hqq5Xx2M2rRLCgH3Y3ag,10743
@@ -100,24 +100,25 @@ sqlmesh/core/plan/definition.py,sha256=NWosaN3wYPwgQU0wLxv___Gx-obvnAdmYzkQN7RGK
 sqlmesh/core/plan/evaluator.py,sha256=QvDk2NAuWSGEffOGCPAdp3kN6-mNGR-_GpWTJJEx7wk,22601
 sqlmesh/core/plan/explainer.py,sha256=aBkvtM9eog_2NIglazSDdG_i9TTCdFSVOkCw5N8O-1k,12028
 sqlmesh/core/plan/stages.py,sha256=aAA29iNSZC3CvWh_4zx9NNmS8OcFsfGSXtAadISfE_0,25329
-sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
+sqlmesh/core/snapshot/__init__.py,sha256=LU-0TGyLTpz6SIPEJr4s0Zvu1d6vuN2YH7m3mR70080,1357
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
-sqlmesh/core/snapshot/definition.py,sha256=LPHpFYY4gKaJnQQzOoToeIbrDWvVVObqaLfyRjB2Wz4,91637
-sqlmesh/core/snapshot/evaluator.py,sha256=cWAOXiB9Yd8hZpfv4m45Ht-0ix0wJRVF710t4eO8XeA,109534
+sqlmesh/core/snapshot/definition.py,sha256=4WuHZsaET6A_aXRLc2Sam7bUPslR1JKBDEOMCATeVTI,91717
+sqlmesh/core/snapshot/evaluator.py,sha256=aE-zOHFuEh6DTd-DZz5Be8IIx58JVodftRAUIjlOzKQ,110080
+sqlmesh/core/snapshot/execution_tracker.py,sha256=Ss1oYgH28Fy1mQ4HriX-luE9MG0eLdecrE1SssUveQI,3651
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=AdqWmbhnvRFqVcFK6L01uCvpBVAYGuMrm4P0D3u_NaU,18334
 sqlmesh/core/state_sync/cache.py,sha256=0AsYdQyBqbLdJvkBQ-hDAipsvnn_jSps-nya-Q4XeL0,5337
 sqlmesh/core/state_sync/common.py,sha256=scuisdmaO3iliuUNFZi16CLDwiSKpW_zkMA0_MCNGLk,7902
 sqlmesh/core/state_sync/export_import.py,sha256=WJ7sFX2b_RPAFE4yspJRi_4JQnaO23xpJgRxwXD0agA,8709
 sqlmesh/core/state_sync/db/__init__.py,sha256=jGCqKYB7npq4eh6H2LyfBAJWdOP-IFu0DC4CMiQgYvw,107
-sqlmesh/core/state_sync/db/environment.py,sha256=SSH4NzqKxUGZ6g0qg0_vLry5T3UyNNyaWA_ePdE7gFI,13689
+sqlmesh/core/state_sync/db/environment.py,sha256=Pn7T79JH0WxdWApci75sjfhNPJ0slHF4TkvqsLcCKQo,13773
 sqlmesh/core/state_sync/db/facade.py,sha256=7OMZt_T3TvavxeC95uwBWe7uD_PtgRyf-s7oYrTkVPg,25245
-sqlmesh/core/state_sync/db/interval.py,sha256=v2NpzKD3Ff6uQekbXjlZZ_tXjzZfFws2gGCmy1XOkZo,17999
-sqlmesh/core/state_sync/db/migrator.py,sha256=av5-dw_0T-u171L8Aepv6nDEg14GFVibdM_hYWq3Hz0,18118
-sqlmesh/core/state_sync/db/snapshot.py,sha256=3QpaVGoqCn81Rlk5oKZrTcaj4pbOFrpUHHT5P-6dXJc,28220
+sqlmesh/core/state_sync/db/interval.py,sha256=gPHDuFU4VJIAP7HfZjZ4_gZo5E3az8Jch43gNO3lh4M,18083
+sqlmesh/core/state_sync/db/migrator.py,sha256=rRwWTETCJ9Mfua-W3fdV_TOqUp1q1Y7i80Oqrd3rO58,18192
+sqlmesh/core/state_sync/db/snapshot.py,sha256=zK-FFADihpK1xgUWcEmC2gabAHQryD5EDOCzy3J_5TY,28300
 sqlmesh/core/state_sync/db/utils.py,sha256=8KjRmOjP5CLuSRkYBUE2k34V-UYB0iSyuO0rWBQ_5pY,4218
-sqlmesh/core/state_sync/db/version.py,sha256=CcmtJyNOOYsACrncxxyc1WGWlx1vPczm_vC5Ii4uVw4,2217
+sqlmesh/core/state_sync/db/version.py,sha256=q5VDIIvY-585vTbvqPalU0N4qjG6RKs4gr8a51R-_UE,2257
 sqlmesh/core/test/__init__.py,sha256=e83TJPwPRR_rAG29Y0OVbZb-5oWVBzz-_wrcd22Qk10,418
 sqlmesh/core/test/context.py,sha256=9mmskKXpzsIkW_I31uN1n3Ioj8ngvyGAqyL_AZvVHpc,2253
 sqlmesh/core/test/definition.py,sha256=VvjwvBoyd-y6Ucepf_vLuDm2DxqDLJh0X--D8BzFUng,41868
@@ -293,14 +294,14 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.211.1.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.211.2.dev2.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
-sqlmesh_dbt/cli.py,sha256=6CqnLKx3DJmacvKtKHmade3RPNBfao2YSi5dh0SI9j4,3853
+sqlmesh_dbt/cli.py,sha256=wSShIx02GCYz0Ciq808GbbO9tjXYkaM6U6SFoT7rmpA,4117
 sqlmesh_dbt/console.py,sha256=80sCEbrKfR5f3SvxIyI7hw_SuiDiiUwuJYxXbQO2014,830
 sqlmesh_dbt/error.py,sha256=khi7UM_nqH4T47QL0NQbiXUvpr15Amh0F-Rtt4Xov20,885
-sqlmesh_dbt/operations.py,sha256=1mA9MqVQB2bqk9JnZCNZ1hxIhp1lau05ANUqEHok8V4,5290
+sqlmesh_dbt/operations.py,sha256=EON-uSJaLkjh_uyhHAiRWdT1yuD2SqZq5CnepmJmZZU,5584
 sqlmesh_dbt/options.py,sha256=noB_qK4uGGi7Erqk1XkkMaFz5aUc6lp44wwn1Nv_LI4,737
-sqlmesh_dbt/selectors.py,sha256=RJITA13d2hILvQYSiWFi_4aZeu0YzxREERb23GVm5P8,4846
+sqlmesh_dbt/selectors.py,sha256=4w8eE0qAgoYC2K8afk3whfju-u3e5muXGwgkfXxQR3Y,5228
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=9Xh9IRDTAsC-XXYlaB1dLXv3dv15cDXnW2qz1z-Dj98,5472
 web/client/dist/assets/Audits-Ucsx1GzF.js,sha256=fZhFbeOvI1RlQt7dBwG-GLDt5SBbCdbTY0i4zfXFGwE,1041
@@ -397,7 +398,7 @@ web/client/dist/fonts/Sohne/sohne-mono-kraftig.woff2,sha256=DGsWMljiFHztzyb0Gg_T
 web/client/src/workers/sqlglot/sqlglot.py,sha256=XAMrvqcRBa7vy8bHVayYbZNyWsf1lNSjVk_odtif-YY,1566
 web/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/server/app.py,sha256=b6qEJ4btjaSm2kWwQUGV--S3VE8QW0Um7f4rBdaoB_M,59
-web/server/console.py,sha256=GsxIYu-oAlXhsSpWohNHYLAO9lOz617wOW1oJI3fR8I,14669
+web/server/console.py,sha256=s9caTcfLd-0Zhj6cQgkJBYxsl9w_ZDNv3gN6MaBNj0o,14806
 web/server/exceptions.py,sha256=I2PPpChwhDWawQENr5kHg6DjyRGeiJ6OWZsCzEZhVzs,1533
 web/server/main.py,sha256=9PaDa8sigo1z0AxtI3sQ3fHmDqs6uXwP-_IvoMwkznw,3199
 web/server/models.py,sha256=SfuknVLSRzAAiTzUQgGyQbF5aZBt8yZg-sBnQ_8lWJQ,16552
@@ -418,8 +419,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=8XTwgOh6QBbNy_hTM1JuHgRjbnie-pGPrphiW-FNLjQ,6058
-sqlmesh-0.211.1.dev4.dist-info/METADATA,sha256=RYC-W7lOS9dtSnU_Mov0UyR6Z3YcmL8DuGEVOCFM190,26749
-sqlmesh-0.211.1.dev4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.211.1.dev4.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
-sqlmesh-0.211.1.dev4.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
-sqlmesh-0.211.1.dev4.dist-info/RECORD,,
+sqlmesh-0.211.2.dev2.dist-info/METADATA,sha256=eRMMo1P2DMyeqqYL3g5V9rjB_JN40Tcx0F70pYBKnhU,26773
+sqlmesh-0.211.2.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.211.2.dev2.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
+sqlmesh-0.211.2.dev2.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
+sqlmesh-0.211.2.dev2.dist-info/RECORD,,

sqlmesh_dbt/cli.py

@@ -46,10 +46,19 @@ exclude_option = click.option("--exclude", multiple=True, help="Specify the node
 @click.group(invoke_without_command=True)
 @click.option("--profile", help="Which existing profile to load. Overrides output.profile")
 @click.option("-t", "--target", help="Which target to load for the given profile")
+@click.option(
+    "-d",
+    "--debug/--no-debug",
+    default=False,
+    help="Display debug logging during dbt execution. Useful for debugging and making bug reports events to help when debugging.",
+)
 @click.pass_context
 @cli_global_error_handler
 def dbt(
-    ctx: click.Context, profile: t.Optional[str] = None, target: t.Optional[str] = None
+    ctx: click.Context,
+    profile: t.Optional[str] = None,
+    target: t.Optional[str] = None,
+    debug: bool = False,
 ) -> None:
     """
     An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine.
@@ -61,7 +70,9 @@ def dbt(
 
     # we have a partially applied function here because subcommands might set extra options like --vars
     # that need to be known before we attempt to load the project
-    ctx.obj = functools.partial(create, project_dir=Path.cwd(), profile=profile, target=target)
+    ctx.obj = functools.partial(
+        create, project_dir=Path.cwd(), profile=profile, target=target, debug=debug
+    )
 
     if not ctx.invoked_subcommand:
         if profile or target:

sqlmesh_dbt/operations.py

@@ -16,9 +16,10 @@ logger = logging.getLogger(__name__)
 
 
 class DbtOperations:
-    def __init__(self, sqlmesh_context: Context, dbt_project: Project):
+    def __init__(self, sqlmesh_context: Context, dbt_project: Project, debug: bool = False):
         self.context = sqlmesh_context
         self.project = dbt_project
+        self.debug = debug
 
     def list_(
         self,
@@ -55,6 +56,10 @@ class DbtOperations:
         self, select: t.Optional[t.List[str]] = None, exclude: t.Optional[t.List[str]] = None
     ) -> t.Dict[str, Model]:
         if sqlmesh_selector := selectors.to_sqlmesh(select or [], exclude or []):
+            if self.debug:
+                self.console.print(f"dbt --select: {select}")
+                self.console.print(f"dbt --exclude: {exclude}")
+                self.console.print(f"sqlmesh equivalent: '{sqlmesh_selector}'")
             model_selector = self.context._new_selector()
             selected_models = {
                 fqn: model
@@ -119,7 +124,7 @@ def create(
         # so that DbtOperations can query information from the DBT project files in order to invoke SQLMesh correctly
         dbt_project = dbt_loader._projects[0]
 
-        return DbtOperations(sqlmesh_context, dbt_project)
+        return DbtOperations(sqlmesh_context, dbt_project, debug=debug)
 
 
 def init_project_if_required(project_dir: Path) -> None:

sqlmesh_dbt/selectors.py

@@ -37,6 +37,10 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) ->
             -> "+main.model_a & ^(raw.src_data)"
         --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data"
             -> "(+main.model_a | main.*b+) & ^(raw.src_data)"
+        --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data" --exclude "main.model_c"
+            -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)"
+        --select "+main.model_a main.*b+" --exclude "raw.src_data main.model_c"
+            -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)"
     """
     if not dbt_select and not dbt_exclude:
         return None
@@ -44,8 +48,13 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) ->
     select_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_select)
     select_expr = _wrap(select_expr) if dbt_exclude and len(dbt_select) > 1 else select_expr
 
-    exclude_expr = " | ".join(_to_sqlmesh(expr, negate=True) for expr in dbt_exclude)
-    exclude_expr = _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr
+    exclude_expr = ""
+
+    if dbt_exclude:
+        exclude_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_exclude)
+        exclude_expr = _negate(
+            _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr
+        )
 
     main_expr = " & ".join([expr for expr in [select_expr, exclude_expr] if expr])
 
@@ -56,13 +65,9 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) ->
     return main_expr
 
 
-def _to_sqlmesh(selector_str: str, negate: bool = False) -> str:
+def _to_sqlmesh(selector_str: str) -> str:
     unions, intersections = _split_unions_and_intersections(selector_str)
 
-    if negate:
-        unions = [_negate(u) for u in unions]
-        intersections = [_negate(i) for i in intersections]
-
     union_expr = " | ".join(unions)
     intersection_expr = " & ".join(intersections)
 
@@ -79,6 +84,7 @@ def _split_unions_and_intersections(selector_str: str) -> t.Tuple[t.List[str], t
     # break space-separated items like: "my_first_model my_second_model" into a list of selectors to union
     # and comma-separated items like: "my_first_model,my_second_model" into a list of selectors to intersect
     # but, take into account brackets, eg "(my_first_model & my_second_model)" should not be split
+    # also take into account both types in the same string, eg "my_first_model my_second_model model_3,model_4,model_5"
 
     def _split_by(input: str, delimiter: str) -> t.Iterator[str]:
         buf = ""

web/server/console.py

@@ -10,6 +10,7 @@ from sqlmesh.core.console import TerminalConsole
 from sqlmesh.core.environment import EnvironmentNamingInfo
 from sqlmesh.core.plan.definition import EvaluatablePlan
 from sqlmesh.core.snapshot import Snapshot, SnapshotInfoLike, SnapshotTableInfo, SnapshotId
+from sqlmesh.core.snapshot.execution_tracker import QueryExecutionStats
 from sqlmesh.core.test import ModelTest
 from sqlmesh.core.test.result import ModelTextTestResult
 from sqlmesh.utils.date import now_timestamp
@@ -142,6 +143,7 @@ class ApiConsole(TerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        execution_stats: t.Optional[QueryExecutionStats] = None,
         auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         if audit_only: