sqlmesh 0.196.5.dev15__py3-none-any.whl → 0.196.5.dev17__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.196.5.dev15'
-__version_tuple__ = version_tuple = (0, 196, 5, 'dev15')
+__version__ = version = '0.196.5.dev17'
+__version_tuple__ = version_tuple = (0, 196, 5, 'dev17')

sqlmesh/core/engine_adapter/base.py

@@ -1514,6 +1514,7 @@ class EngineAdapter:
         unique_key: t.Sequence[exp.Expression],
         valid_from_col: exp.Column,
         valid_to_col: exp.Column,
+        start: TimeLike,
         execution_time: t.Union[TimeLike, exp.Column],
         invalidate_hard_deletes: bool = True,
         updated_at_col: t.Optional[exp.Column] = None,
@@ -1708,8 +1709,14 @@ class EngineAdapter:
         existing_rows_query = exp.select(*table_columns, exp.true().as_("_exists")).from_(
             target_table
         )
+
+        cleanup_ts = None
         if truncate:
             existing_rows_query = existing_rows_query.limit(0)
+        else:
+            # If truncate is false it is not the first insert
+            # Determine the cleanup timestamp for restatement or a regular incremental run
+            cleanup_ts = to_time_column(start, time_data_type, self.dialect, nullable=True)
 
         with source_queries[0] as source_query:
             prefixed_columns_to_types = []
@@ -1747,12 +1754,41 @@ class EngineAdapter:
                 # Historical Records that Do Not Change
                 .with_(
                     "static",
-                    existing_rows_query.where(valid_to_col.is_(exp.Null()).not_()),
+                    existing_rows_query.where(valid_to_col.is_(exp.Null()).not_())
+                    if truncate
+                    else existing_rows_query.where(
+                        exp.and_(
+                            valid_to_col.is_(exp.Null().not_()),
+                            valid_to_col < cleanup_ts,
+                        ),
+                    ),
                 )
                 # Latest Records that can be updated
                 .with_(
                     "latest",
-                    existing_rows_query.where(valid_to_col.is_(exp.Null())),
+                    existing_rows_query.where(valid_to_col.is_(exp.Null()))
+                    if truncate
+                    else exp.select(
+                        *(
+                            to_time_column(
+                                exp.null(), time_data_type, self.dialect, nullable=True
+                            ).as_(col)
+                            if col == valid_to_col.name
+                            else exp.column(col)
+                            for col in columns_to_types
+                        ),
+                        exp.true().as_("_exists"),
+                    )
+                    .from_(target_table)
+                    .where(
+                        exp.and_(
+                            valid_from_col <= cleanup_ts,
+                            exp.or_(
+                                valid_to_col.is_(exp.null()),
+                                valid_to_col >= cleanup_ts,
+                            ),
+                        )
+                    ),
                 )
                 # Deleted records which can be used to determine `valid_from` for undeleted source records
                 .with_(

sqlmesh/core/engine_adapter/trino.py

@@ -256,6 +256,7 @@ class TrinoEngineAdapter(
         unique_key: t.Sequence[exp.Expression],
         valid_from_col: exp.Column,
         valid_to_col: exp.Column,
+        start: TimeLike,
         execution_time: t.Union[TimeLike, exp.Column],
         invalidate_hard_deletes: bool = True,
         updated_at_col: t.Optional[exp.Column] = None,
@@ -277,6 +278,7 @@ class TrinoEngineAdapter(
             unique_key,
             valid_from_col,
             valid_to_col,
+            start,
             execution_time,
             invalidate_hard_deletes,
             updated_at_col,

sqlmesh/core/model/kind.py

@@ -140,7 +140,6 @@ class ModelKindMixin:
             self.is_incremental_unmanaged
             or self.is_incremental_by_unique_key
             or self.is_incremental_by_partition
-            or self.is_scd_type_2
             or self.is_managed
             or self.is_full
             or self.is_view

sqlmesh/core/plan/builder.py

@@ -405,7 +405,7 @@ class PlanBuilder:
                 elif (not self._is_dev or not snapshot.is_paused) and snapshot.disable_restatement:
                     self._console.log_warning(
                         f"Cannot restate model '{snapshot.name}'. "
-                        "Restatement is disabled for this model to prevent possible data loss."
+                        "Restatement is disabled for this model to prevent possible data loss. "
                         "If you want to restate this model, change the model's `disable_restatement` setting to `false`."
                     )
                     continue

sqlmesh/core/snapshot/definition.py

@@ -797,6 +797,21 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
 
             removal_interval = expanded_removal_interval
 
+        # SCD Type 2 validation that end date is the latest interval if it was provided
+        if not is_preview and self.is_scd_type_2 and self.intervals:
+            requested_start, requested_end = removal_interval
+            latest_end = self.intervals[-1][1]
+            if requested_end < latest_end:
+                from sqlmesh.core.console import get_console
+
+                get_console().log_warning(
+                    f"SCD Type 2 model '{self.model.name}' does not support end date in restatements.\n"
+                    f"Requested end date [{to_ts(requested_end)}] is less than the latest interval end date.\n"
+                    f"The requested end date will be ignored. Using the latest interval end instead: [{to_ts(latest_end)}]"
+                )
+
+                removal_interval = self.inclusive_exclusive(requested_start, latest_end, strict)
+
         return removal_interval
 
     @property

sqlmesh/core/snapshot/evaluator.py

@@ -1788,6 +1788,7 @@ class SCDType2Strategy(MaterializableStrategy):
                 table_description=model.description,
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
+                start=kwargs["start"],
             )
         elif isinstance(model.kind, SCDType2ByColumnKind):
             self.adapter.scd_type_2_by_column(
@@ -1805,6 +1806,7 @@ class SCDType2Strategy(MaterializableStrategy):
                 table_description=model.description,
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
+                start=kwargs["start"],
             )
         else:
             raise SQLMeshError(

{sqlmesh-0.196.5.dev15.dist-info → sqlmesh-0.196.5.dev17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.196.5.dev15
+Version: 0.196.5.dev17
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -305,6 +305,7 @@ Provides-Extra: dbt
 Requires-Dist: dbt-core<2; extra == "dbt"
 Provides-Extra: dlt
 Requires-Dist: dlt; extra == "dlt"
+Provides-Extra: duckdb
 Provides-Extra: gcppostgres
 Requires-Dist: cloud-sql-python-connector[pg8000]>=1.8.0; extra == "gcppostgres"
 Provides-Extra: github
@@ -312,6 +313,7 @@ Requires-Dist: PyGithub~=2.5.0; extra == "github"
 Provides-Extra: llm
 Requires-Dist: langchain; extra == "llm"
 Requires-Dist: openai; extra == "llm"
+Provides-Extra: motherduck
 Provides-Extra: mssql
 Requires-Dist: pymssql; extra == "mssql"
 Provides-Extra: mssql-odbc

{sqlmesh-0.196.5.dev15.dist-info → sqlmesh-0.196.5.dev17.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=8DyeJ8ypHhs8Pr086v9kGJIGX44ijXJHjIPXL-J7j2I,530
+sqlmesh/_version.py,sha256=rLbV4hLoU9ecx9UrZoa9wKIl0Owes29-dAHNbw7wyx8,530
 sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,7 +59,7 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=_63pYCThfC39UtPVHY3N0H6qJ1YBT32OqwwR3swolGM,1037
 sqlmesh/core/engine_adapter/athena.py,sha256=WBVAB_YzBJXdiqq51LTd2jmYoMtGrLmtsA8IrIFVnQ0,26199
-sqlmesh/core/engine_adapter/base.py,sha256=R90ZhxpVb58Vl-zVse9fkNVbm3eD3lqi1PhW_WBOtAQ,102263
+sqlmesh/core/engine_adapter/base.py,sha256=9dMfOQtgjTi551xM9Lxm5FK4MUoopvzwwgHjjDFnpzw,103788
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
 sqlmesh/core/engine_adapter/bigquery.py,sha256=qJE8vHLKKAeWMvltmYaUvWS95jM_UEZ6UzYLlr58sGo,50424
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=LoPdm_s_s-ZxS1XJPA3Et0iOHeJ1LgqTAFbwBdiqE7A,34912
@@ -74,7 +74,7 @@ sqlmesh/core/engine_adapter/risingwave.py,sha256=YYCQ0EJaSrR9_P1_2_LmElQ6zSxqkZz
 sqlmesh/core/engine_adapter/shared.py,sha256=zL2W2t9af9mo80aSO0fz5TqV6mY8_kI5VYLswbKdZyk,13369
 sqlmesh/core/engine_adapter/snowflake.py,sha256=IEEzepxNpfqBIMe4ziz2k8fjOcJPeQPZyM3XljymAj0,28874
 sqlmesh/core/engine_adapter/spark.py,sha256=sYkVOPS0X43UPbI0z7Hh_EpKlI36qkOcr4NNFOTQlLs,21045
-sqlmesh/core/engine_adapter/trino.py,sha256=7WdLFlT0AyD5NF0EDP7GqX0SHF8sTm7cafBZxk84cmY,15798
+sqlmesh/core/engine_adapter/trino.py,sha256=Ik5yOW_gYahVq6ksZPzE2Um-5J4A2AJv-QZiZYUPeP8,15842
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=pXDxl8TxTGNuVc1B5ywYmaH7dDjsoMRvCtLCK_co-ek,5567
 sqlmesh/core/linter/helpers.py,sha256=VF4hsdzGZ8srpbtCjdRN_Hd8S_er5KTAaLXLde27Yxo,4032
@@ -89,12 +89,12 @@ sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7
 sqlmesh/core/model/common.py,sha256=PcQS3rHFNIXV_Z1dGwLQ7gLPbYVlrbWGGKAyc58-9Xk,17379
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
 sqlmesh/core/model/definition.py,sha256=QUxpnWAmUDKw8GfdISPgL_W4cYII8LrwOj4-49e8XgE,111143
-sqlmesh/core/model/kind.py,sha256=iTfkZIeZ5GfJvW8MpjSOAIXJo-RtS-6JhGmgGSt_Q8s,36143
+sqlmesh/core/model/kind.py,sha256=_SBjhrTUrCGkmcsKmabbmBIe1s_6aMIEPEaQh0cDIok,36109
 sqlmesh/core/model/meta.py,sha256=D9InI9M2-IXeb34vkXNQSLALL8xVXjZ_LXwI-Tqxm7I,21197
 sqlmesh/core/model/schema.py,sha256=OClbAeA8LpW-IiSiLT_TlDWkEqmIIvSk0GXq8SWI-GQ,3443
 sqlmesh/core/model/seed.py,sha256=a0M-1zY1gOkN5ph2GQyataEdBSCtq50YjeFk2LyvInI,5058
 sqlmesh/core/plan/__init__.py,sha256=NKSvM7ZBVjw9ho3J65M1wFvG3KURB8PJ0FHHLmtSF44,443
-sqlmesh/core/plan/builder.py,sha256=9M8Bzf6SZ3nnawowvu0jHSVCCVblz3srevJoOS7ppkM,41415
+sqlmesh/core/plan/builder.py,sha256=g07h-NOsOr_4spnOrlQ4KrFjh3RmbvHSydEuSOsShNc,41416
 sqlmesh/core/plan/definition.py,sha256=OxgbvkMGNdlvYF2BjFJt_lINsBw38Fq1IX25P8_Z_Z0,13734
 sqlmesh/core/plan/evaluator.py,sha256=DkpJ78R33_RBx5MVY13CcGU0x2-EcKWkcrETsjuyn10,20484
 sqlmesh/core/plan/explainer.py,sha256=BEtww_UVa1gMkzL6-y2yrcy1EMQ8MbkElvO6L6s_TYU,12011
@@ -102,8 +102,8 @@ sqlmesh/core/plan/stages.py,sha256=5cvEbrQsw_cx1gayxwLRwkIehQPixrbUT9B4BbEOj3A,2
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
-sqlmesh/core/snapshot/definition.py,sha256=a0pGUPHqlK_vBAaR0RZYloSiwkned07vT_KbxDs1ak8,84575
-sqlmesh/core/snapshot/evaluator.py,sha256=RA-ZFwG_BtkXSmls41gHRaCqa-h-evxXsDLE5jtgxvU,90173
+sqlmesh/core/snapshot/definition.py,sha256=zbNqxmIi136n4Ee-25vvEVK0pICQM2KVsgm2qGR12l0,85447
+sqlmesh/core/snapshot/evaluator.py,sha256=i5T5EZ4lEw7X7ZGapIhau6IxJ52nJcdvSixBu_utrhE,90251
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=Drq4MDWWxO_J-sJIxsMG1-YSgj2L42n5WM2vh1sr6bc,18334
 sqlmesh/core/state_sync/cache.py,sha256=0AsYdQyBqbLdJvkBQ-hDAipsvnn_jSps-nya-Q4XeL0,5337
@@ -283,7 +283,7 @@ sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,116
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.196.5.dev15.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.196.5.dev17.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=kNpFjWxt34Pf-P-cROtPWi3GNSVk6zqA3eoN9tlZXI0,5472
 web/client/dist/assets/Audits-DKgXravk.js,sha256=SaVQhNlislD_yQc_KivO7SPi1JH0ts1GnDPD5Zssutw,1041
@@ -401,8 +401,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.196.5.dev15.dist-info/METADATA,sha256=Q4o9-GOU6aYoI4s79m1chXldPBKyMML5SuYcRIE6xqs,26571
-sqlmesh-0.196.5.dev15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.196.5.dev15.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.196.5.dev15.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.196.5.dev15.dist-info/RECORD,,
+sqlmesh-0.196.5.dev17.dist-info/METADATA,sha256=tafFP47LimiJmXpFam5y9UY22QMDLypbXp1HuJv1JeI,26621
+sqlmesh-0.196.5.dev17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.196.5.dev17.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.196.5.dev17.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.196.5.dev17.dist-info/RECORD,,