sqlmesh 0.209.1.dev19__tar.gz → 0.209.1.dev21__tar.gz

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.209.1.dev19
+Version: 0.209.1.dev21
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.209.1.dev19'
-__version_tuple__ = version_tuple = (0, 209, 1, 'dev19')
+__version__ = version = '0.209.1.dev21'
+__version_tuple__ = version_tuple = (0, 209, 1, 'dev21')

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/engine_adapter/base.py

@@ -1497,7 +1497,6 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         self._scd_type_2(
@@ -1514,7 +1513,6 @@ class EngineAdapter:
             table_description=table_description,
             column_descriptions=column_descriptions,
             truncate=truncate,
-            is_restatement=is_restatement,
             **kwargs,
         )
 
@@ -1533,7 +1531,6 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         self._scd_type_2(
@@ -1550,7 +1547,6 @@ class EngineAdapter:
             table_description=table_description,
             column_descriptions=column_descriptions,
             truncate=truncate,
-            is_restatement=is_restatement,
             **kwargs,
         )
 
@@ -1561,7 +1557,6 @@ class EngineAdapter:
         unique_key: t.Sequence[exp.Expression],
         valid_from_col: exp.Column,
         valid_to_col: exp.Column,
-        start: TimeLike,
         execution_time: t.Union[TimeLike, exp.Column],
         invalidate_hard_deletes: bool = True,
         updated_at_col: t.Optional[exp.Column] = None,
@@ -1572,7 +1567,6 @@ class EngineAdapter:
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         def remove_managed_columns(
@@ -1757,17 +1751,9 @@ class EngineAdapter:
         existing_rows_query = exp.select(*table_columns, exp.true().as_("_exists")).from_(
             target_table
         )
-
         if truncate:
             existing_rows_query = existing_rows_query.limit(0)
 
-        # Only set cleanup_ts if is_restatement is True and truncate is False (this to enable full restatement)
-        cleanup_ts = (
-            to_time_column(start, time_data_type, self.dialect, nullable=True)
-            if is_restatement and not truncate
-            else None
-        )
-
         with source_queries[0] as source_query:
             prefixed_columns_to_types = []
             for column in columns_to_types:
@@ -1804,41 +1790,12 @@ class EngineAdapter:
                 # Historical Records that Do Not Change
                 .with_(
                     "static",
-                    existing_rows_query.where(valid_to_col.is_(exp.Null()).not_())
-                    if cleanup_ts is None
-                    else existing_rows_query.where(
-                        exp.and_(
-                            valid_to_col.is_(exp.Null().not_()),
-                            valid_to_col < cleanup_ts,
-                        ),
-                    ),
+                    existing_rows_query.where(valid_to_col.is_(exp.Null()).not_()),
                 )
                 # Latest Records that can be updated
                 .with_(
                     "latest",
-                    existing_rows_query.where(valid_to_col.is_(exp.Null()))
-                    if cleanup_ts is None
-                    else exp.select(
-                        *(
-                            to_time_column(
-                                exp.null(), time_data_type, self.dialect, nullable=True
-                            ).as_(col)
-                            if col == valid_to_col.name
-                            else exp.column(col)
-                            for col in columns_to_types
-                        ),
-                        exp.true().as_("_exists"),
-                    )
-                    .from_(target_table)
-                    .where(
-                        exp.and_(
-                            valid_from_col <= cleanup_ts,
-                            exp.or_(
-                                valid_to_col.is_(exp.null()),
-                                valid_to_col >= cleanup_ts,
-                            ),
-                        )
-                    ),
+                    existing_rows_query.where(valid_to_col.is_(exp.Null())),
                 )
                 # Deleted records which can be used to determine `valid_from` for undeleted source records
                 .with_(

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/engine_adapter/trino.py

@@ -256,7 +256,6 @@ class TrinoEngineAdapter(
         unique_key: t.Sequence[exp.Expression],
         valid_from_col: exp.Column,
         valid_to_col: exp.Column,
-        start: TimeLike,
         execution_time: t.Union[TimeLike, exp.Column],
         invalidate_hard_deletes: bool = True,
         updated_at_col: t.Optional[exp.Column] = None,
@@ -267,7 +266,6 @@ class TrinoEngineAdapter(
         table_description: t.Optional[str] = None,
         column_descriptions: t.Optional[t.Dict[str, str]] = None,
         truncate: bool = False,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> None:
         if columns_to_types and self.current_catalog_type == "delta_lake":
@@ -279,7 +277,6 @@ class TrinoEngineAdapter(
             unique_key,
             valid_from_col,
             valid_to_col,
-            start,
             execution_time,
             invalidate_hard_deletes,
             updated_at_col,
@@ -290,7 +287,6 @@ class TrinoEngineAdapter(
             table_description,
             column_descriptions,
             truncate,
-            is_restatement,
             **kwargs,
         )
 

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/model/kind.py

@@ -141,6 +141,7 @@ class ModelKindMixin:
             self.is_incremental_unmanaged
             or self.is_incremental_by_unique_key
             or self.is_incremental_by_partition
+            or self.is_scd_type_2
             or self.is_managed
             or self.is_full
             or self.is_view

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/plan/evaluator.py

@@ -235,11 +235,6 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             return
 
         scheduler = self.create_scheduler(stage.all_snapshots.values(), self.snapshot_evaluator)
-        # Convert model name restatements to snapshot ID restatements
-        restatements_by_snapshot_id = {
-            stage.all_snapshots[name].snapshot_id: interval
-            for name, interval in plan.restatements.items()
-        }
         errors, _ = scheduler.run_merged_intervals(
             merged_intervals=stage.snapshot_to_intervals,
             deployability_index=stage.deployability_index,
@@ -248,7 +243,6 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             circuit_breaker=self._circuit_breaker,
             start=plan.start,
             end=plan.end,
-            restatements=restatements_by_snapshot_id,
         )
         if errors:
             raise PlanError("Plan application failed.")

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/scheduler.py

@@ -161,7 +161,6 @@ class Scheduler:
         deployability_index: DeployabilityIndex,
         batch_index: int,
         environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> t.List[AuditResult]:
         """Evaluate a snapshot and add the processed interval to the state sync.
@@ -193,7 +192,6 @@ class Scheduler:
             snapshots=snapshots,
             deployability_index=deployability_index,
             batch_index=batch_index,
-            is_restatement=is_restatement,
             **kwargs,
         )
         audit_results = self._audit_snapshot(
@@ -373,7 +371,6 @@ class Scheduler:
         end: t.Optional[TimeLike] = None,
         run_environment_statements: bool = False,
         audit_only: bool = False,
-        restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
     ) -> t.Tuple[t.List[NodeExecutionFailedError[SchedulingUnit]], t.List[SchedulingUnit]]:
         """Runs precomputed batches of missing intervals.
 
@@ -450,10 +447,6 @@ class Scheduler:
                         execution_time=execution_time,
                     )
                 else:
-                    # Determine if this snapshot and interval is a restatement (for SCD type 2)
-                    is_restatement = (
-                        restatements is not None and snapshot.snapshot_id in restatements
-                    )
                     audit_results = self.evaluate(
                         snapshot=snapshot,
                         environment_naming_info=environment_naming_info,
@@ -462,7 +455,6 @@ class Scheduler:
                         execution_time=execution_time,
                         deployability_index=deployability_index,
                         batch_index=batch_idx,
-                        is_restatement=is_restatement,
                     )
 
                 evaluation_duration_ms = now_timestamp() - execution_start_ts
@@ -671,7 +663,6 @@ class Scheduler:
             end=end,
             run_environment_statements=run_environment_statements,
             audit_only=audit_only,
-            restatements=remove_intervals,
         )
 
         return CompletionStatus.FAILURE if errors else CompletionStatus.SUCCESS

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/snapshot/definition.py

@@ -837,21 +837,6 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
 
             removal_interval = expanded_removal_interval
 
-        # SCD Type 2 validation that end date is the latest interval if it was provided
-        if not is_preview and self.is_scd_type_2 and self.intervals:
-            requested_start, requested_end = removal_interval
-            latest_end = self.intervals[-1][1]
-            if requested_end < latest_end:
-                from sqlmesh.core.console import get_console
-
-                get_console().log_warning(
-                    f"SCD Type 2 model '{self.model.name}' does not support end date in restatements.\n"
-                    f"Requested end date [{to_ts(requested_end)}] is less than the latest interval end date.\n"
-                    f"The requested end date will be ignored. Using the latest interval end instead: [{to_ts(latest_end)}]"
-                )
-
-                removal_interval = self.inclusive_exclusive(requested_start, latest_end, strict)
-
         return removal_interval
 
     @property

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh/core/snapshot/evaluator.py

@@ -140,7 +140,6 @@ class SnapshotEvaluator:
         snapshots: t.Dict[str, Snapshot],
         deployability_index: t.Optional[DeployabilityIndex] = None,
         batch_index: int = 0,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> t.Optional[str]:
         """Renders the snapshot's model, executes it and stores the result in the snapshot's physical table.
@@ -166,7 +165,6 @@ class SnapshotEvaluator:
             snapshots,
             deployability_index=deployability_index,
             batch_index=batch_index,
-            is_restatement=is_restatement,
             **kwargs,
         )
         if result is None or isinstance(result, str):
@@ -624,7 +622,6 @@ class SnapshotEvaluator:
         limit: t.Optional[int] = None,
         deployability_index: t.Optional[DeployabilityIndex] = None,
         batch_index: int = 0,
-        is_restatement: bool = False,
         **kwargs: t.Any,
     ) -> DF | str | None:
         """Renders the snapshot's model and executes it. The return value depends on whether the limit was specified.
@@ -697,7 +694,6 @@ class SnapshotEvaluator:
                     end=end,
                     execution_time=execution_time,
                     physical_properties=rendered_physical_properties,
-                    is_restatement=is_restatement,
                 )
             else:
                 logger.info(
@@ -719,7 +715,6 @@ class SnapshotEvaluator:
                     end=end,
                     execution_time=execution_time,
                     physical_properties=rendered_physical_properties,
-                    is_restatement=is_restatement,
                 )
 
         with (
@@ -1840,8 +1835,6 @@ class SCDType2Strategy(MaterializableStrategy):
                 table_description=model.description,
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
-                start=kwargs["start"],
-                is_restatement=kwargs.get("is_restatement", False),
             )
         elif isinstance(model.kind, SCDType2ByColumnKind):
             self.adapter.scd_type_2_by_column(
@@ -1859,8 +1852,6 @@ class SCDType2Strategy(MaterializableStrategy):
                 table_description=model.description,
                 column_descriptions=model.column_descriptions,
                 truncate=is_first_insert,
-                start=kwargs["start"],
-                is_restatement=kwargs.get("is_restatement", False),
             )
         else:
             raise SQLMeshError(

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.209.1.dev19
+Version: 0.209.1.dev21
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.209.1.dev19 → sqlmesh-0.209.1.dev21}/tests/core/engine_adapter/integration/test_integration.py

@@ -744,7 +744,6 @@ def test_scd_type_2_by_time(ctx_query_and_df: TestContext):
         columns_to_types=input_schema,
         table_format=ctx.default_table_format,
         truncate=True,
-        start="2022-01-01 00:00:00",
     )
     results = ctx.get_metadata_results()
     assert len(results.views) == 0
@@ -808,7 +807,6 @@ def test_scd_type_2_by_time(ctx_query_and_df: TestContext):
         columns_to_types=input_schema,
         table_format=ctx.default_table_format,
         truncate=False,
-        start="2022-01-01 00:00:00",
     )
     results = ctx.get_metadata_results()
     assert len(results.views) == 0
@@ -901,7 +899,6 @@ def test_scd_type_2_by_column(ctx_query_and_df: TestContext):
         execution_time_as_valid_from=False,
         columns_to_types=ctx.columns_to_types,
         truncate=True,
-        start="2023-01-01",
     )
     results = ctx.get_metadata_results()
     assert len(results.views) == 0
@@ -973,7 +970,6 @@ def test_scd_type_2_by_column(ctx_query_and_df: TestContext):
         execution_time_as_valid_from=False,
         columns_to_types=ctx.columns_to_types,
         truncate=False,
-        start="2023-01-01",
     )
     results = ctx.get_metadata_results()
     assert len(results.views) == 0