sqlmesh 0.201.1.dev1__py3-none-any.whl → 0.201.1.dev3__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.201.1.dev1'
-__version_tuple__ = version_tuple = (0, 201, 1, 'dev1')
+__version__ = version = '0.201.1.dev3'
+__version_tuple__ = version_tuple = (0, 201, 1, 'dev3')

sqlmesh/core/engine_adapter/base.py

@@ -1462,6 +1462,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         self._scd_type_2(
@@ -1478,6 +1479,7 @@ class EngineAdapter:
             table_description=table_description,
             column_descriptions=column_descriptions,
             truncate=truncate,
+            is_restatement=is_restatement,
             **kwargs,
         )
 
@@ -1496,6 +1498,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         self._scd_type_2(
@@ -1512,6 +1515,7 @@ class EngineAdapter:
             table_description=table_description,
             column_descriptions=column_descriptions,
             truncate=truncate,
+            is_restatement=is_restatement,
             **kwargs,
         )
 
@@ -1533,6 +1537,7 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         def remove_managed_columns(
@@ -1718,13 +1723,15 @@ class EngineAdapter:
             target_table
         )
 
-        cleanup_ts = None
         if truncate:
             existing_rows_query = existing_rows_query.limit(0)
-        else:
-            # If truncate is false it is not the first insert
-            # Determine the cleanup timestamp for restatement or a regular incremental run
-            cleanup_ts = to_time_column(start, time_data_type, self.dialect, nullable=True)
+
+        # Only set cleanup_ts if is_restatement is True and truncate is False (this to enable full restatement)
+        cleanup_ts = (
+            to_time_column(start, time_data_type, self.dialect, nullable=True)
+            if is_restatement and not truncate
+            else None
+        )
 
         with source_queries[0] as source_query:
             prefixed_columns_to_types = []
@@ -1763,7 +1770,7 @@ class EngineAdapter:
                 .with_(
                     "static",
                     existing_rows_query.where(valid_to_col.is_(exp.Null()).not_())
-                    if truncate
+                    if cleanup_ts is None
                     else existing_rows_query.where(
                         exp.and_(
                             valid_to_col.is_(exp.Null().not_()),
@@ -1775,7 +1782,7 @@ class EngineAdapter:
                 .with_(
                     "latest",
                     existing_rows_query.where(valid_to_col.is_(exp.Null()))
-                    if truncate
+                    if cleanup_ts is None
                     else exp.select(
                         *(
                             to_time_column(

sqlmesh/core/engine_adapter/bigquery.py

@@ -134,6 +134,10 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
         }
         if self._extra_config.get("maximum_bytes_billed"):
             params["maximum_bytes_billed"] = self._extra_config.get("maximum_bytes_billed")
+        if self.correlation_id:
+            # BigQuery label keys must be lowercase
+            key = self.correlation_id.job_type.value.lower()
+            params["labels"] = {key: self.correlation_id.job_id}
         return params
 
     @property
@@ -204,6 +208,11 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
                 "Invalid value for `session_properties.query_label`. Must be an array or tuple."
             )
 
+        if self.correlation_id:
+            parsed_query_label.append(
+                (self.correlation_id.job_type.value.lower(), self.correlation_id.job_id)
+            )
+
         if parsed_query_label:
             query_label_str = ",".join([":".join(label) for label in parsed_query_label])
             query = f'SET @@query_label = "{query_label_str}";SELECT 1;'

sqlmesh/core/engine_adapter/trino.py

@@ -267,6 +267,7 @@ class TrinoEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         if columns_to_types and self.current_catalog_type == "delta_lake":
@@ -289,6 +290,7 @@ class TrinoEngineAdapter(
             table_description,
             column_descriptions,
             truncate,
+            is_restatement,
             **kwargs,
         )
 

sqlmesh/core/plan/evaluator.py

@@ -234,6 +234,11 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             return
 
         scheduler = self.create_scheduler(stage.all_snapshots.values(), self.snapshot_evaluator)
+        # Convert model name restatements to snapshot ID restatements
+        restatements_by_snapshot_id = {
+            stage.all_snapshots[name].snapshot_id: interval
+            for name, interval in plan.restatements.items()
+        }
         errors, _ = scheduler.run_merged_intervals(
             merged_intervals=stage.snapshot_to_intervals,
             deployability_index=stage.deployability_index,
@@ -242,6 +247,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             circuit_breaker=self._circuit_breaker,
             start=plan.start,
             end=plan.end,
+            restatements=restatements_by_snapshot_id,
         )
         if errors:
             raise PlanError("Plan application failed.")

sqlmesh/core/scheduler.py

@@ -161,6 +161,7 @@ class Scheduler:
         deployability_index: DeployabilityIndex,
         batch_index: int,
         environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> t.List[AuditResult]:
         """Evaluate a snapshot and add the processed interval to the state sync.
@@ -192,6 +193,7 @@ class Scheduler:
             snapshots=snapshots,
             deployability_index=deployability_index,
             batch_index=batch_index,
+            is_restatement=is_restatement,
             **kwargs,
         )
         audit_results = self._audit_snapshot(
@@ -371,6 +373,7 @@ class Scheduler:
         end: t.Optional[TimeLike] = None,
         run_environment_statements: bool = False,
         audit_only: bool = False,
+        restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
     ) -> t.Tuple[t.List[NodeExecutionFailedError[SchedulingUnit]], t.List[SchedulingUnit]]:
         """Runs precomputed batches of missing intervals.
 
@@ -447,6 +450,10 @@ class Scheduler:
                         execution_time=execution_time,
                     )
                 else:
+                    # Determine if this snapshot and interval is a restatement (for SCD type 2)
+                    is_restatement = (
+                        restatements is not None and snapshot.snapshot_id in restatements
+                    )
                     audit_results = self.evaluate(
                         snapshot=snapshot,
                         environment_naming_info=environment_naming_info,
@@ -455,6 +462,7 @@ class Scheduler:
                         execution_time=execution_time,
                         deployability_index=deployability_index,
                         batch_index=batch_idx,
+                        is_restatement=is_restatement,
                     )
 
                 evaluation_duration_ms = now_timestamp() - execution_start_ts
@@ -663,6 +671,7 @@ class Scheduler:
             end=end,
             run_environment_statements=run_environment_statements,
             audit_only=audit_only,
+            restatements=remove_intervals,
         )
 
         return CompletionStatus.FAILURE if errors else CompletionStatus.SUCCESS

sqlmesh/core/snapshot/evaluator.py

@@ -140,6 +140,7 @@ class SnapshotEvaluator:
         snapshots: t.Dict[str, Snapshot],
         deployability_index: t.Optional[DeployabilityIndex] = None,
         batch_index: int = 0,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> t.Optional[str]:
         """Renders the snapshot's model, executes it and stores the result in the snapshot's physical table.
@@ -165,6 +166,7 @@ class SnapshotEvaluator:
             snapshots,
             deployability_index=deployability_index,
             batch_index=batch_index,
+            is_restatement=is_restatement,
             **kwargs,
         )
         if result is None or isinstance(result, str):
@@ -622,6 +624,7 @@ class SnapshotEvaluator:
         limit: t.Optional[int] = None,
         deployability_index: t.Optional[DeployabilityIndex] = None,
         batch_index: int = 0,
+        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> DF | str | None:
         """Renders the snapshot's model and executes it. The return value depends on whether the limit was specified.
@@ -694,6 +697,7 @@ class SnapshotEvaluator:
                     end=end,
                     execution_time=execution_time,
                     physical_properties=rendered_physical_properties,
+                    is_restatement=is_restatement,
                 )
             else:
                 logger.info(
@@ -715,6 +719,7 @@ class SnapshotEvaluator:
                     end=end,
                     execution_time=execution_time,
                     physical_properties=rendered_physical_properties,
+                    is_restatement=is_restatement,
                 )
 
         with (
@@ -1833,6 +1838,7 @@ class SCDType2Strategy(MaterializableStrategy):
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
                 start=kwargs["start"],
+                is_restatement=kwargs.get("is_restatement", False),
             )
         elif isinstance(model.kind, SCDType2ByColumnKind):
             self.adapter.scd_type_2_by_column(
@@ -1851,6 +1857,7 @@ class SCDType2Strategy(MaterializableStrategy):
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
                 start=kwargs["start"],
+                is_restatement=kwargs.get("is_restatement", False),
             )
         else:
             raise SQLMeshError(

{sqlmesh-0.201.1.dev1.dist-info → sqlmesh-0.201.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.201.1.dev1
+Version: 0.201.1.dev3
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.201.1.dev1.dist-info → sqlmesh-0.201.1.dev3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=wWb0WF0HqqOTVtFmZQ_YD72bQI0-lNwObrfk_adzW1w,528
+sqlmesh/_version.py,sha256=1tTr8NASXh9g0FtenQtx6UvZoEwK-N9sry6pOINtvwU,528
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -24,7 +24,7 @@ sqlmesh/core/node.py,sha256=3ChBz1PIMJhp-jzed_GLBrtmEF2v_BVDEM5WNe0ssPk,14658
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
 sqlmesh/core/renderer.py,sha256=SVE1THHK1UVdPPHjmqNbBVIjmEYcK8tz5pWh1ss8xGs,26684
-sqlmesh/core/scheduler.py,sha256=I30K5NyeM5HuZxyZ62Vgs1VoMWPNA0gev0nveWugmLM,40544
+sqlmesh/core/scheduler.py,sha256=CeI-15GrraLUI5YO8fQUACj7WG7dY6otE4TaJOp2_IU,41041
 sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
 sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
 sqlmesh/core/selector.py,sha256=CBuCbaFzidinaoS66o4MY9F-1C3k9YddNMyUv-5Z2i8,12815
@@ -59,9 +59,9 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=_63pYCThfC39UtPVHY3N0H6qJ1YBT32OqwwR3swolGM,1037
 sqlmesh/core/engine_adapter/athena.py,sha256=AS1w_l_VrFZsc2CIicElg1dtS9y6q5hHHmTg4ECH-xs,26453
-sqlmesh/core/engine_adapter/base.py,sha256=vCnqWyZCUsLVZArCdyEQywvAliwFCDDbGMig7EyOaTU,104361
+sqlmesh/core/engine_adapter/base.py,sha256=QUHO4z-5IibBD1pKPfYr04yTroDZ4katsV-GHUmFC-Y,104590
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
-sqlmesh/core/engine_adapter/bigquery.py,sha256=qJE8vHLKKAeWMvltmYaUvWS95jM_UEZ6UzYLlr58sGo,50424
+sqlmesh/core/engine_adapter/bigquery.py,sha256=hkEbCiWYy6qS9VHwOZ67yn1t50apWOoV8mn2tetjIVE,50809
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=LoPdm_s_s-ZxS1XJPA3Et0iOHeJ1LgqTAFbwBdiqE7A,34912
 sqlmesh/core/engine_adapter/databricks.py,sha256=8ZExPIeT3dDh9MTJW8udvmh9LiY-f9HkRAN1Q5Qebhs,14541
 sqlmesh/core/engine_adapter/duckdb.py,sha256=TlSFSWNzUfjGdhuMyCi1CLwhBaEKyNKtasub9OsYwrc,5199
@@ -74,7 +74,7 @@ sqlmesh/core/engine_adapter/risingwave.py,sha256=GY0u085TWerl8kTRNJpEutdD5FBsoPi
 sqlmesh/core/engine_adapter/shared.py,sha256=zL2W2t9af9mo80aSO0fz5TqV6mY8_kI5VYLswbKdZyk,13369
 sqlmesh/core/engine_adapter/snowflake.py,sha256=IEEzepxNpfqBIMe4ziz2k8fjOcJPeQPZyM3XljymAj0,28874
 sqlmesh/core/engine_adapter/spark.py,sha256=sYkVOPS0X43UPbI0z7Hh_EpKlI36qkOcr4NNFOTQlLs,21045
-sqlmesh/core/engine_adapter/trino.py,sha256=Ik5yOW_gYahVq6ksZPzE2Um-5J4A2AJv-QZiZYUPeP8,15842
+sqlmesh/core/engine_adapter/trino.py,sha256=Tws3GxqnC6BmXFohqru1TDxsgPH-21qV6Oky7FaaLpE,15908
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=pXDxl8TxTGNuVc1B5ywYmaH7dDjsoMRvCtLCK_co-ek,5567
 sqlmesh/core/linter/helpers.py,sha256=ygumMRAShxHITK_5vI64tHJoEa2mAdKen5hlMxaXKoQ,5022
@@ -96,14 +96,14 @@ sqlmesh/core/model/seed.py,sha256=a0M-1zY1gOkN5ph2GQyataEdBSCtq50YjeFk2LyvInI,50
 sqlmesh/core/plan/__init__.py,sha256=NKSvM7ZBVjw9ho3J65M1wFvG3KURB8PJ0FHHLmtSF44,443
 sqlmesh/core/plan/builder.py,sha256=TQVSGiGbQi9tro0tBlcL2f5BEvyOYNiLuB3yz8PRUzA,41859
 sqlmesh/core/plan/definition.py,sha256=yrX_uZMXi29ltkuowWhNUOi1kd0Uq0TQoM574EI6JHo,14019
-sqlmesh/core/plan/evaluator.py,sha256=OS6ayFwb_o1CBccVfiWPse8ecpI4nCcV8Ubr9yEp7yE,21527
+sqlmesh/core/plan/evaluator.py,sha256=KOBZD9-hn1vVpFR9WGSIu2gt-C66R6--bwA6GxgNick,21821
 sqlmesh/core/plan/explainer.py,sha256=aBkvtM9eog_2NIglazSDdG_i9TTCdFSVOkCw5N8O-1k,12028
 sqlmesh/core/plan/stages.py,sha256=JN7rwo6ibY5Ctqof3Us7Eu2x85n5GXH0rCA47BBvP1o,22228
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
 sqlmesh/core/snapshot/definition.py,sha256=DwYTNcDfPQUQ1osF_63SdicSj7oqVjtmPPEzqwsxDLU,85748
-sqlmesh/core/snapshot/evaluator.py,sha256=CPjpzUmFwjM9wTECe6KJB5FaY2C7LwxvFT9fXSorfTs,91489
+sqlmesh/core/snapshot/evaluator.py,sha256=9l73YNKTEu9N91JTSxnxLqQo-lAaX6ihPxVNi4wUfFQ,91846
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=AdqWmbhnvRFqVcFK6L01uCvpBVAYGuMrm4P0D3u_NaU,18334
 sqlmesh/core/state_sync/cache.py,sha256=0AsYdQyBqbLdJvkBQ-hDAipsvnn_jSps-nya-Q4XeL0,5337
@@ -284,7 +284,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.201.1.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.201.1.dev3.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -402,8 +402,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.201.1.dev1.dist-info/METADATA,sha256=vWHfmSD4VYjZggUvLfXm1xg9zr9EXfb1XMaC_N9Jf5Q,26678
-sqlmesh-0.201.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.201.1.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.201.1.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.201.1.dev1.dist-info/RECORD,,
+sqlmesh-0.201.1.dev3.dist-info/METADATA,sha256=XGPAejnoSVf067feuPqNfZeRZvbU29oMrREeTSD7QVo,26678
+sqlmesh-0.201.1.dev3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.201.1.dev3.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.201.1.dev3.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.201.1.dev3.dist-info/RECORD,,