sqlmesh 0.199.1.dev17__py3-none-any.whl → 0.199.1.dev19__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.199.1.dev17'
-__version_tuple__ = version_tuple = (0, 199, 1, 'dev17')
+__version__ = version = '0.199.1.dev19'
+__version_tuple__ = version_tuple = (0, 199, 1, 'dev19')

sqlmesh/cli/main.py

@@ -520,6 +520,11 @@ def diff(ctx: click.Context, environment: t.Optional[str] = None) -> None:
     help="Explain the plan instead of applying it.",
     default=None,
 )
+@click.option(
+    "--min-intervals",
+    default=0,
+    help="For every model, ensure at least this many intervals are covered by a missing intervals check regardless of the plan start date",
+)
 @opt.verbose
 @click.pass_context
 @error_handler

sqlmesh/core/console.py

@@ -2073,8 +2073,8 @@ class TerminalConsole(Console):
             if not plan_builder.override_end:
                 if plan.provided_end:
                     blank_meaning = f"'{time_like_to_str(plan.provided_end)}'"
-                elif plan.interval_end_per_model:
-                    max_end = max(plan.interval_end_per_model.values())
+                elif plan.end_override_per_model:
+                    max_end = max(plan.end_override_per_model.values())
                     blank_meaning = f"'{time_like_to_str(max_end)}'"
                 else:
                     blank_meaning = "now"

sqlmesh/core/context.py

@@ -46,6 +46,7 @@ from itertools import chain
 from pathlib import Path
 from shutil import rmtree
 from types import MappingProxyType
+from datetime import datetime
 
 from sqlglot import Dialect, exp
 from sqlglot.helper import first
@@ -125,6 +126,8 @@ from sqlmesh.utils.date import (
     format_tz_datetime,
     now_timestamp,
     now,
+    to_datetime,
+    make_exclusive,
 )
 from sqlmesh.utils.errors import (
     CircuitBreakerError,
@@ -1222,6 +1225,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         diff_rendered: t.Optional[bool] = None,
         skip_linter: t.Optional[bool] = None,
         explain: t.Optional[bool] = None,
+        min_intervals: t.Optional[int] = None,
     ) -> Plan:
         """Interactively creates a plan.
 
@@ -1268,6 +1272,8 @@ class GenericContext(BaseContext, t.Generic[C]):
             diff_rendered: Whether the diff should compare raw vs rendered models
             skip_linter: Linter runs by default so this will skip it if enabled
             explain: Whether to explain the plan instead of applying it.
+            min_intervals: Adjust the plan start date on a per-model basis in order to ensure at least this many intervals are covered
+                on every model when checking for missing intervals
 
         Returns:
             The populated Plan object.
@@ -1296,6 +1302,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             diff_rendered=diff_rendered,
             skip_linter=skip_linter,
             explain=explain,
+            min_intervals=min_intervals,
         )
 
         plan = plan_builder.build()
@@ -1345,6 +1352,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         diff_rendered: t.Optional[bool] = None,
         skip_linter: t.Optional[bool] = None,
         explain: t.Optional[bool] = None,
+        min_intervals: t.Optional[int] = None,
     ) -> PlanBuilder:
         """Creates a plan builder.
 
@@ -1381,6 +1389,8 @@ class GenericContext(BaseContext, t.Generic[C]):
             enable_preview: Indicates whether to enable preview for forward-only models in development environments.
             run: Whether to run latest intervals as part of the plan application.
             diff_rendered: Whether the diff should compare raw vs rendered models
+            min_intervals: Adjust the plan start date on a per-model basis in order to ensure at least this many intervals are covered
+                on every model when checking for missing intervals
 
         Returns:
             The plan builder.
@@ -1408,6 +1418,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             "run": run,
             "diff_rendered": diff_rendered,
             "skip_linter": skip_linter,
+            "min_intervals": min_intervals,
         }
         user_provided_flags: t.Dict[str, UserProvidedFlags] = {
             k: v for k, v in kwargs.items() if v is not None
@@ -1530,6 +1541,16 @@ class GenericContext(BaseContext, t.Generic[C]):
             # Refresh snapshot intervals to ensure that they are up to date with values reflected in the max_interval_end_per_model.
             self.state_sync.refresh_snapshot_intervals(context_diff.snapshots.values())
 
+        start_override_per_model = self._calculate_start_override_per_model(
+            min_intervals,
+            start or default_start,
+            end or default_end,
+            execution_time or now(),
+            backfill_models,
+            snapshots,
+            max_interval_end_per_model,
+        )
+
         return self.PLAN_BUILDER_TYPE(
             context_diff=context_diff,
             start=start,
@@ -1560,7 +1581,8 @@ class GenericContext(BaseContext, t.Generic[C]):
             ),
             end_bounded=not run,
             ensure_finalized_snapshots=self.config.plan.use_finalized_state,
-            interval_end_per_model=max_interval_end_per_model,
+            start_override_per_model=start_override_per_model,
+            end_override_per_model=max_interval_end_per_model,
             console=self.console,
             user_provided_flags=user_provided_flags,
             explain=explain or False,
@@ -2850,7 +2872,7 @@ class GenericContext(BaseContext, t.Generic[C]):
     def _get_plan_default_start_end(
         self,
         snapshots: t.Dict[str, Snapshot],
-        max_interval_end_per_model: t.Dict[str, int],
+        max_interval_end_per_model: t.Dict[str, datetime],
         backfill_models: t.Optional[t.Set[str]],
         modified_model_names: t.Set[str],
         execution_time: t.Optional[TimeLike] = None,
@@ -2858,7 +2880,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         if not max_interval_end_per_model:
             return None, None
 
-        default_end = max(max_interval_end_per_model.values())
+        default_end = to_timestamp(max(max_interval_end_per_model.values()))
         default_start: t.Optional[int] = None
         # Infer the default start by finding the smallest interval start that corresponds to the default end.
         for model_name in backfill_models or modified_model_names or max_interval_end_per_model:
@@ -2887,19 +2909,101 @@ class GenericContext(BaseContext, t.Generic[C]):
 
         return default_start, default_end
 
+    def _calculate_start_override_per_model(
+        self,
+        min_intervals: t.Optional[int],
+        plan_start: t.Optional[TimeLike],
+        plan_end: t.Optional[TimeLike],
+        plan_execution_time: TimeLike,
+        backfill_model_fqns: t.Optional[t.Set[str]],
+        snapshots_by_model_fqn: t.Dict[str, Snapshot],
+        end_override_per_model: t.Optional[t.Dict[str, datetime]],
+    ) -> t.Dict[str, datetime]:
+        if not min_intervals or not backfill_model_fqns or not plan_start:
+            # If there are no models to backfill, there are no intervals to consider for backfill, so we dont need to consider a minimum number
+            # If the plan doesnt have a start date, all intervals are considered already so we dont need to consider a minimum number
+            # If we dont have a minimum number of intervals to consider, then we dont need to adjust the start date on a per-model basis
+            return {}
+
+        start_overrides: t.Dict[str, datetime] = {}
+        end_override_per_model = end_override_per_model or {}
+
+        plan_execution_time_dt = to_datetime(plan_execution_time)
+        plan_start_dt = to_datetime(plan_start, relative_base=plan_execution_time_dt)
+        plan_end_dt = to_datetime(
+            plan_end or plan_execution_time_dt, relative_base=plan_execution_time_dt
+        )
+
+        # we need to take the DAG into account so that parent models can be expanded to cover at least as much as their children
+        # for example, A(hourly) <- B(daily)
+        # if min_intervals=1, A would have 1 hour and B would have 1 day
+        # but B depends on A so in order for B to have 1 valid day, A needs to be expanded to 24 hours
+        backfill_dag: DAG[str] = DAG()
+        for fqn in backfill_model_fqns:
+            backfill_dag.add(
+                fqn,
+                [
+                    p.name
+                    for p in snapshots_by_model_fqn[fqn].parents
+                    if p.name in backfill_model_fqns
+                ],
+            )
+
+        # start from the leaf nodes and work back towards the root because the min_start at the root node is determined by the calculated starts in the leaf nodes
+        reversed_dag = backfill_dag.reversed
+        graph = reversed_dag.graph
+
+        for model_fqn in reversed_dag:
+            # Get the earliest start from all immediate children of this snapshot
+            # this works because topological ordering guarantees that they've already been visited
+            # and we always set a start override
+            min_child_start = min(
+                [start_overrides[immediate_child_fqn] for immediate_child_fqn in graph[model_fqn]],
+                default=plan_start_dt,
+            )
+
+            snapshot = snapshots_by_model_fqn.get(model_fqn)
+
+            if not snapshot:
+                continue
+
+            starting_point = end_override_per_model.get(model_fqn, plan_end_dt)
+            if node_end := snapshot.node.end:
+                # if we dont do this, if the node end is a *date* (as opposed to a timestamp)
+                # we end up incorrectly winding back an extra day
+                node_end_dt = make_exclusive(node_end)
+
+                if node_end_dt < plan_end_dt:
+                    # if the model has an end date that has already elapsed, use that as a starting point for calculating min_intervals
+                    # instead of the plan end. If we use the plan end, we will return intervals in the future which are invalid
+                    starting_point = node_end_dt
+
+            snapshot_start = snapshot.node.cron_floor(starting_point)
+
+            for _ in range(min_intervals):
+                # wind back the starting point by :min_intervals intervals to arrive at the minimum snapshot start date
+                snapshot_start = snapshot.node.cron_prev(snapshot_start)
+
+            start_overrides[model_fqn] = min(min_child_start, snapshot_start)
+
+        return start_overrides
+
     def _get_max_interval_end_per_model(
         self, snapshots: t.Dict[str, Snapshot], backfill_models: t.Optional[t.Set[str]]
-    ) -> t.Dict[str, int]:
+    ) -> t.Dict[str, datetime]:
         models_for_interval_end = (
             self._get_models_for_interval_end(snapshots, backfill_models)
             if backfill_models is not None
             else None
         )
-        return self.state_sync.max_interval_end_per_model(
-            c.PROD,
-            models=models_for_interval_end,
-            ensure_finalized_snapshots=self.config.plan.use_finalized_state,
-        )
+        return {
+            model_fqn: to_datetime(ts)
+            for model_fqn, ts in self.state_sync.max_interval_end_per_model(
+                c.PROD,
+                models=models_for_interval_end,
+                ensure_finalized_snapshots=self.config.plan.use_finalized_state,
+            ).items()
+        }
 
     @staticmethod
     def _get_models_for_interval_end(

sqlmesh/core/node.py

@@ -31,6 +31,9 @@ class IntervalUnit(str, Enum):
     IntervalUnit can be one of 5 types, YEAR, MONTH, DAY, HOUR, MINUTE. The unit is inferred
     based on the cron schedule of a node. The minimum time delta between a sample set of dates
     is used to determine which unit a node's schedule is.
+
+    It's designed to align with common partitioning schemes, hence why there is no WEEK unit
+    because generally tables are not partitioned by week
     """
 
     YEAR = "year"

sqlmesh/core/plan/builder.py

@@ -5,6 +5,7 @@ import re
 import typing as t
 from collections import defaultdict
 from functools import cached_property
+from datetime import datetime
 
 
 from sqlmesh.core.console import PlanBuilderConsole, get_console
@@ -85,7 +86,8 @@ class PlanBuilder:
         ensure_finalized_snapshots: Whether to compare against snapshots from the latest finalized
             environment state, or to use whatever snapshots are in the current environment state even if
             the environment is not finalized.
-        interval_end_per_model: The mapping from model FQNs to target end dates.
+        start_override_per_model: A mapping of model FQNs to target start dates.
+        end_override_per_model: A mapping of model FQNs to target end dates.
         explain: Whether to explain the plan instead of applying it.
     """
 
@@ -117,7 +119,8 @@ class PlanBuilder:
         end_bounded: bool = False,
         ensure_finalized_snapshots: bool = False,
         explain: bool = False,
-        interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+        end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
         console: t.Optional[PlanBuilderConsole] = None,
         user_provided_flags: t.Optional[t.Dict[str, UserProvidedFlags]] = None,
     ):
@@ -133,7 +136,8 @@ class PlanBuilder:
         self._enable_preview = enable_preview
         self._end_bounded = end_bounded
         self._ensure_finalized_snapshots = ensure_finalized_snapshots
-        self._interval_end_per_model = interval_end_per_model
+        self._start_override_per_model = start_override_per_model
+        self._end_override_per_model = end_override_per_model
         self._environment_ttl = environment_ttl
         self._categorizer_config = categorizer_config or CategorizerConfig()
         self._auto_categorization_enabled = auto_categorization_enabled
@@ -280,7 +284,11 @@ class PlanBuilder:
         self._adjust_new_snapshot_intervals()
 
         deployability_index = (
-            DeployabilityIndex.create(self._context_diff.snapshots.values(), start=self._start)
+            DeployabilityIndex.create(
+                self._context_diff.snapshots.values(),
+                start=self._start,
+                start_override_per_model=self._start_override_per_model,
+            )
             if self._is_dev
             else DeployabilityIndex.all_deployable()
         )
@@ -291,11 +299,11 @@ class PlanBuilder:
         )
         models_to_backfill = self._build_models_to_backfill(dag, restatements)
 
-        interval_end_per_model = self._interval_end_per_model
-        if interval_end_per_model and self.override_end:
+        end_override_per_model = self._end_override_per_model
+        if end_override_per_model and self.override_end:
             # If the end date was provided explicitly by a user, then interval end for each individual
             # model should be ignored.
-            interval_end_per_model = None
+            end_override_per_model = None
 
         # this deliberately uses the passed in self._execution_time and not self.execution_time cached property
         # the reason is because that there can be a delay between the Plan being built and the Plan being actually run,
@@ -322,7 +330,8 @@ class PlanBuilder:
             indirectly_modified=indirectly_modified,
             deployability_index=deployability_index,
             restatements=restatements,
-            interval_end_per_model=interval_end_per_model,
+            start_override_per_model=self._start_override_per_model,
+            end_override_per_model=end_override_per_model,
             selected_models_to_backfill=self._backfill_models,
             models_to_backfill=models_to_backfill,
             effective_from=self._effective_from,

sqlmesh/core/plan/definition.py

@@ -57,7 +57,8 @@ class Plan(PydanticModel, frozen=True):
 
     deployability_index: DeployabilityIndex
     restatements: t.Dict[SnapshotId, Interval]
-    interval_end_per_model: t.Optional[t.Dict[str, int]]
+    start_override_per_model: t.Optional[t.Dict[str, datetime]]
+    end_override_per_model: t.Optional[t.Dict[str, datetime]]
 
     selected_models_to_backfill: t.Optional[t.Set[str]] = None
     """Models that have been explicitly selected for backfill by a user."""
@@ -177,7 +178,8 @@ class Plan(PydanticModel, frozen=True):
                 execution_time=self.execution_time,
                 restatements=self.restatements,
                 deployability_index=self.deployability_index,
-                interval_end_per_model=self.interval_end_per_model,
+                start_override_per_model=self.start_override_per_model,
+                end_override_per_model=self.end_override_per_model,
                 end_bounded=self.end_bounded,
             ).items()
             if snapshot.is_model and missing
@@ -265,7 +267,8 @@ class Plan(PydanticModel, frozen=True):
             removed_snapshots=sorted(self.context_diff.removed_snapshots),
             requires_backfill=self.requires_backfill,
             models_to_backfill=self.models_to_backfill,
-            interval_end_per_model=self.interval_end_per_model,
+            start_override_per_model=self.start_override_per_model,
+            end_override_per_model=self.end_override_per_model,
             execution_time=self.execution_time,
             disabled_restatement_models={
                 s.name
@@ -303,7 +306,8 @@ class EvaluatablePlan(PydanticModel):
     removed_snapshots: t.List[SnapshotId]
     requires_backfill: bool
     models_to_backfill: t.Optional[t.Set[str]] = None
-    interval_end_per_model: t.Optional[t.Dict[str, int]] = None
+    start_override_per_model: t.Optional[t.Dict[str, datetime]] = None
+    end_override_per_model: t.Optional[t.Dict[str, datetime]] = None
     execution_time: t.Optional[TimeLike] = None
     disabled_restatement_models: t.Set[str]
     environment_statements: t.Optional[t.List[EnvironmentStatements]] = None

sqlmesh/core/plan/evaluator.py

@@ -256,7 +256,8 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             plan.end,
             execution_time=plan.execution_time,
             end_bounded=plan.end_bounded,
-            interval_end_per_model=plan.interval_end_per_model,
+            start_override_per_model=plan.start_override_per_model,
+            end_override_per_model=plan.end_override_per_model,
         )
 
         if completion_status.is_failure:

sqlmesh/core/plan/stages.py

@@ -524,7 +524,8 @@ class PlanStagesBuilder:
             },
             deployability_index=deployability_index,
             end_bounded=plan.end_bounded,
-            interval_end_per_model=plan.interval_end_per_model,
+            start_override_per_model=plan.start_override_per_model,
+            end_override_per_model=plan.end_override_per_model,
         )
 
     def _get_audit_only_snapshots(

sqlmesh/core/scheduler.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import logging
 import typing as t
 import time
+from datetime import datetime
 from sqlglot import exp
 from sqlmesh.core import constants as c
 from sqlmesh.core.console import Console, get_console
@@ -105,7 +106,8 @@ class Scheduler:
         execution_time: t.Optional[TimeLike] = None,
         deployability_index: t.Optional[DeployabilityIndex] = None,
         restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
-        interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+        end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
         ignore_cron: bool = False,
         end_bounded: bool = False,
         selected_snapshots: t.Optional[t.Set[str]] = None,
@@ -123,7 +125,8 @@ class Scheduler:
             execution_time: The date/time reference to use for execution time. Defaults to now.
             deployability_index: Determines snapshots that are deployable in the context of this evaluation.
             restatements: A set of snapshot names being restated.
-            interval_end_per_model: The mapping from model FQNs to target end dates.
+            start_override_per_model: A mapping of model FQNs to target start dates.
+            end_override_per_model: A mapping of model FQNs to target end dates.
             ignore_cron: Whether to ignore the node's cron schedule.
             end_bounded: If set to true, the returned intervals will be bounded by the target end date, disregarding lookback,
                 allow_partials, and other attributes that could cause the intervals to exceed the target end date.
@@ -136,7 +139,8 @@ class Scheduler:
             execution_time=execution_time,
             deployability_index=deployability_index,
             restatements=restatements,
-            interval_end_per_model=interval_end_per_model,
+            start_override_per_model=start_override_per_model,
+            end_override_per_model=end_override_per_model,
             ignore_cron=ignore_cron,
             end_bounded=end_bounded,
         )
@@ -212,7 +216,8 @@ class Scheduler:
         end: t.Optional[TimeLike] = None,
         execution_time: t.Optional[TimeLike] = None,
         restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
-        interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+        end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
         ignore_cron: bool = False,
         end_bounded: bool = False,
         selected_snapshots: t.Optional[t.Set[str]] = None,
@@ -227,7 +232,8 @@ class Scheduler:
             end=end,
             execution_time=execution_time,
             remove_intervals=restatements,
-            interval_end_per_model=interval_end_per_model,
+            start_override_per_model=start_override_per_model,
+            end_override_per_model=end_override_per_model,
             ignore_cron=ignore_cron,
             end_bounded=end_bounded,
             selected_snapshots=selected_snapshots,
@@ -243,7 +249,8 @@ class Scheduler:
         start: TimeLike,
         end: TimeLike,
         execution_time: t.Optional[TimeLike] = None,
-        interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+        end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
         ignore_cron: bool = False,
         end_bounded: bool = False,
         selected_snapshots: t.Optional[t.Set[str]] = None,
@@ -266,7 +273,8 @@ class Scheduler:
             end=end,
             execution_time=execution_time,
             remove_intervals=remove_intervals,
-            interval_end_per_model=interval_end_per_model,
+            start_override_per_model=start_override_per_model,
+            end_override_per_model=end_override_per_model,
             ignore_cron=ignore_cron,
             end_bounded=end_bounded,
             selected_snapshots=selected_snapshots,
@@ -565,7 +573,8 @@ class Scheduler:
         end: t.Optional[TimeLike] = None,
         execution_time: t.Optional[TimeLike] = None,
         remove_intervals: t.Optional[t.Dict[SnapshotId, Interval]] = None,
-        interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+        end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
         ignore_cron: bool = False,
         end_bounded: bool = False,
         selected_snapshots: t.Optional[t.Set[str]] = None,
@@ -586,7 +595,8 @@ class Scheduler:
             execution_time: The date/time time reference to use for execution time. Defaults to now.
             remove_intervals: A dict of snapshots to their intervals. For evaluation, these are the intervals that will be restated. For audits,
                               these are the intervals that will be reaudited
-            interval_end_per_model: The mapping from model FQNs to target end dates.
+            start_override_per_model: A mapping of model FQNs to target start dates.
+            end_override_per_model: A mapping of model FQNs to target end dates.
             ignore_cron: Whether to ignore the node's cron schedule.
             end_bounded: If set to true, the evaluated intervals will be bounded by the target end date, disregarding lookback,
                 allow_partials, and other attributes that could cause the intervals to exceed the target end date.
@@ -634,7 +644,8 @@ class Scheduler:
             execution_time,
             deployability_index=deployability_index,
             restatements=remove_intervals,
-            interval_end_per_model=interval_end_per_model,
+            start_override_per_model=start_override_per_model,
+            end_override_per_model=end_override_per_model,
             ignore_cron=ignore_cron,
             end_bounded=end_bounded,
             selected_snapshots=selected_snapshots,
@@ -797,7 +808,8 @@ def merged_missing_intervals(
     execution_time: t.Optional[TimeLike] = None,
     deployability_index: t.Optional[DeployabilityIndex] = None,
     restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
-    interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+    start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+    end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
     ignore_cron: bool = False,
     end_bounded: bool = False,
 ) -> SnapshotToIntervals:
@@ -815,7 +827,8 @@ def merged_missing_intervals(
         execution_time: The date/time reference to use for execution time. Defaults to now.
         deployability_index: Determines snapshots that are deployable in the context of this evaluation.
         restatements: A set of snapshot names being restated.
-        interval_end_per_model: The mapping from model FQNs to target end dates.
+        start_override_per_model: A mapping of model FQNs to target start dates.
+        end_override_per_model: A mapping of model FQNs to target end dates.
         ignore_cron: Whether to ignore the node's cron schedule.
         end_bounded: If set to true, the returned intervals will be bounded by the target end date, disregarding lookback,
             allow_partials, and other attributes that could cause the intervals to exceed the target end date.
@@ -830,7 +843,8 @@ def merged_missing_intervals(
         deployability_index=deployability_index,
         execution_time=execution_time or now_timestamp(),
         restatements=restatements,
-        interval_end_per_model=interval_end_per_model,
+        start_override_per_model=start_override_per_model,
+        end_override_per_model=end_override_per_model,
         ignore_cron=ignore_cron,
         end_bounded=end_bounded,
     )
@@ -844,7 +858,8 @@ def compute_interval_params(
     deployability_index: t.Optional[DeployabilityIndex] = None,
     execution_time: t.Optional[TimeLike] = None,
     restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
-    interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+    start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+    end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
     ignore_cron: bool = False,
     end_bounded: bool = False,
 ) -> SnapshotToIntervals:
@@ -862,7 +877,8 @@ def compute_interval_params(
         deployability_index: Determines snapshots that are deployable in the context of this evaluation.
         execution_time: The date/time reference to use for execution time.
         restatements: A dict of snapshot names being restated and their intervals.
-        interval_end_per_model: The mapping from model FQNs to target end dates.
+        start_override_per_model: A mapping of model FQNs to target start dates.
+        end_override_per_model: A mapping of model FQNs to target end dates.
         ignore_cron: Whether to ignore the node's cron schedule.
         end_bounded: If set to true, the returned intervals will be bounded by the target end date, disregarding lookback,
             allow_partials, and other attributes that could cause the intervals to exceed the target end date.
@@ -879,7 +895,8 @@ def compute_interval_params(
         execution_time=execution_time,
         restatements=restatements,
         deployability_index=deployability_index,
-        interval_end_per_model=interval_end_per_model,
+        start_override_per_model=start_override_per_model,
+        end_override_per_model=end_override_per_model,
         ignore_cron=ignore_cron,
         end_bounded=end_bounded,
     ).items():

sqlmesh/core/snapshot/definition.py

@@ -1469,7 +1469,8 @@ class DeployabilityIndex(PydanticModel, frozen=True):
     def create(
         cls,
         snapshots: t.Dict[SnapshotId, Snapshot] | t.Collection[Snapshot],
-        start: t.Optional[TimeLike] = None,
+        start: t.Optional[TimeLike] = None,  # plan start
+        start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
     ) -> DeployabilityIndex:
         if not isinstance(snapshots, dict):
             snapshots = {s.snapshot_id: s for s in snapshots}
@@ -1477,6 +1478,7 @@ class DeployabilityIndex(PydanticModel, frozen=True):
         deployability_mapping: t.Dict[SnapshotId, bool] = {}
         children_deployability_mapping: t.Dict[SnapshotId, bool] = {}
         representative_shared_version_ids: t.Set[SnapshotId] = set()
+        start_override_per_model = start_override_per_model or {}
 
         start_date_cache: t.Optional[t.Dict[str, datetime]] = {}
 
@@ -1499,12 +1501,12 @@ class DeployabilityIndex(PydanticModel, frozen=True):
                     snapshot.is_model and snapshot.model.auto_restatement_cron is not None
                 )
 
+                snapshot_start = start_override_per_model.get(
+                    node.name, start_date(snapshot, snapshots.values(), cache=start_date_cache)
+                )
+
                 is_valid_start = (
-                    snapshot.is_valid_start(
-                        start, start_date(snapshot, snapshots.values(), start_date_cache)
-                    )
-                    if start is not None
-                    else True
+                    snapshot.is_valid_start(start, snapshot_start) if start is not None else True
                 )
 
                 if (
@@ -1800,7 +1802,8 @@ def missing_intervals(
     execution_time: t.Optional[TimeLike] = None,
     restatements: t.Optional[t.Dict[SnapshotId, Interval]] = None,
     deployability_index: t.Optional[DeployabilityIndex] = None,
-    interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
+    start_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
+    end_override_per_model: t.Optional[t.Dict[str, datetime]] = None,
     ignore_cron: bool = False,
     end_bounded: bool = False,
 ) -> t.Dict[Snapshot, Intervals]:
@@ -1817,13 +1820,15 @@ def missing_intervals(
         else earliest_start_date(snapshots, cache=cache, relative_to=end_date)
     )
     restatements = restatements or {}
-    interval_end_per_model = interval_end_per_model or {}
+    start_override_per_model = start_override_per_model or {}
+    end_override_per_model = end_override_per_model or {}
     deployability_index = deployability_index or DeployabilityIndex.all_deployable()
 
     for snapshot in snapshots.values():
         if not snapshot.evaluatable:
             continue
-        snapshot_start_date = start_dt
+
+        snapshot_start_date = start_override_per_model.get(snapshot.name, start_dt)
         snapshot_end_date: TimeLike = end_date
 
         restated_interval = restatements.get(snapshot.snapshot_id)
@@ -1833,9 +1838,9 @@ def missing_intervals(
             snapshot.intervals = snapshot.intervals.copy()
             snapshot.remove_interval(restated_interval)
 
-        existing_interval_end = interval_end_per_model.get(snapshot.name)
+        existing_interval_end = end_override_per_model.get(snapshot.name)
         if existing_interval_end:
-            if to_timestamp(snapshot_start_date) >= existing_interval_end:
+            if snapshot_start_date >= existing_interval_end:
                 # The start exceeds the provided interval end, so we can skip this snapshot
                 # since it doesn't have missing intervals by definition
                 continue

{sqlmesh-0.199.1.dev17.dist-info → sqlmesh-0.199.1.dev19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.199.1.dev17
+Version: 0.199.1.dev19
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.199.1.dev17.dist-info → sqlmesh-0.199.1.dev19.dist-info}/RECORD

@@ -1,30 +1,30 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=r6O2EtRY7oIsLR32rAgzPfCqazoeUCzIn1NNqWv7_vQ,530
+sqlmesh/_version.py,sha256=8TEWJVO9dASKRsPIXBTYMc8BKwc4f8TClP6jWf6SAkE,530
 sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/bot.py,sha256=2zlbn-DXkqQzr3lA0__IGU4XaIfXBXBKLWXNI2DRJX8,759
 sqlmesh/cicd/config.py,sha256=mpgkisSBBovlNrNZir2unT8Wes6b38RW2UsU_o4e8vY,109
 sqlmesh/cli/__init__.py,sha256=MAXAyO7isbyM-PaSdFjkJNcjn32WjOl0XzxT5gvp_vo,1690
-sqlmesh/cli/main.py,sha256=XXWvOcqxb-NYkETEQ7vKRMO1mdiW_xwuFZHW60yTQWg,35858
+sqlmesh/cli/main.py,sha256=vzQmFtz80_nVPJKYzFVw9ea5f-fM41Ux4-qI4GeI0_w,36053
 sqlmesh/cli/options.py,sha256=-4ZclMIeIg8tghLowtmQNmpEFlm0QtOVEh40NqRdFHA,2499
 sqlmesh/cli/project_init.py,sha256=5JmDmUK0WJs39hM-p76uzHYdo1Hnx6fKuJg_pR3Hn5A,16419
 sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
-sqlmesh/core/console.py,sha256=ieSXS0G--HpVgziwAqgcJvE4zUn60xRswljpc1GO6pU,160753
+sqlmesh/core/console.py,sha256=ObpdArJOXDOihWPXsWIUuwiQ64cMlroNWJHqQPkVqC0,160753
 sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
-sqlmesh/core/context.py,sha256=K9bQqk5ATZ0cl7v7qaiOQq-b7hYfPIgdo9G9v51Y6y8,119111
+sqlmesh/core/context.py,sha256=VO2Sp0u3CRmUoQI0z2Qwfzp4bhqKdtxlG0az1tND2_w,124223
 sqlmesh/core/context_diff.py,sha256=CeUmMBJzN__98YohrtAiIYPIZI9mrGBk6bVADWBLUw8,21692
 sqlmesh/core/dialect.py,sha256=1ykYwb9WZn_nCaX08PcQ0BJh2Pihv9SPvoMNjpMeCfs,50158
 sqlmesh/core/environment.py,sha256=7DbJX0r5nZRU3LhcgIxbSV-JosQYDjAt3V5Wc2Fpns8,13005
 sqlmesh/core/lineage.py,sha256=zjB0Zfamo2Fja2r5SSZPMbrEKTXr1WjozZPVqvxdybI,3143
 sqlmesh/core/loader.py,sha256=Imb65SX5TtvQrtlilin2hkYg12gm9e9c30I8o95nkVE,40534
 sqlmesh/core/macros.py,sha256=Me4QjgW_K4DzQVi8I3G0XwoC3cXSFFWt3sJyynZ2GOM,60585
-sqlmesh/core/node.py,sha256=r4_1MRTPcU0MAlwZ5KzrykLT58B7dkKvHzPCsJmEKD8,14497
+sqlmesh/core/node.py,sha256=9V2zLRRODgondxQdB6jA5wVH0KCYon70gre0LFBgrM8,14648
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
 sqlmesh/core/renderer.py,sha256=ksdD2x2Q5Ns5fOLIGfWIznx_T3fXbP3CfwUhD5kVSOQ,26674
-sqlmesh/core/scheduler.py,sha256=lA3zhpz52HV-QBGoPirtqd2KakivnTuqiqZp09pP8js,39318
+sqlmesh/core/scheduler.py,sha256=5qSC-EHKotWpwFFvnwDHw3feYfz9c8_9gBGRfdeu8P4,40512
 sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
 sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
 sqlmesh/core/selector.py,sha256=CBuCbaFzidinaoS66o4MY9F-1C3k9YddNMyUv-5Z2i8,12815
@@ -94,15 +94,15 @@ sqlmesh/core/model/meta.py,sha256=D9InI9M2-IXeb34vkXNQSLALL8xVXjZ_LXwI-Tqxm7I,21
 sqlmesh/core/model/schema.py,sha256=_HMYfzK9wWXh7_CQDIIGnuQUD4aiX3o5D2cRp2sERzc,3387
 sqlmesh/core/model/seed.py,sha256=a0M-1zY1gOkN5ph2GQyataEdBSCtq50YjeFk2LyvInI,5058
 sqlmesh/core/plan/__init__.py,sha256=NKSvM7ZBVjw9ho3J65M1wFvG3KURB8PJ0FHHLmtSF44,443
-sqlmesh/core/plan/builder.py,sha256=g07h-NOsOr_4spnOrlQ4KrFjh3RmbvHSydEuSOsShNc,41416
-sqlmesh/core/plan/definition.py,sha256=OxgbvkMGNdlvYF2BjFJt_lINsBw38Fq1IX25P8_Z_Z0,13734
-sqlmesh/core/plan/evaluator.py,sha256=4Gt9svTUe-0YSDawlSbJNed5fkZASbTPsYyytQFXy14,20559
+sqlmesh/core/plan/builder.py,sha256=TQVSGiGbQi9tro0tBlcL2f5BEvyOYNiLuB3yz8PRUzA,41859
+sqlmesh/core/plan/definition.py,sha256=yrX_uZMXi29ltkuowWhNUOi1kd0Uq0TQoM574EI6JHo,14019
+sqlmesh/core/plan/evaluator.py,sha256=FNA4oQ29lZsabKZU36T2wu7QQymwAfCu6zhirdPKuCg,20627
 sqlmesh/core/plan/explainer.py,sha256=aBkvtM9eog_2NIglazSDdG_i9TTCdFSVOkCw5N8O-1k,12028
-sqlmesh/core/plan/stages.py,sha256=5cvEbrQsw_cx1gayxwLRwkIehQPixrbUT9B4BbEOj3A,21972
+sqlmesh/core/plan/stages.py,sha256=7iiVZLSRoI1kSEMyOuuW6gvUA-eyALUfdW5vDh_N4jU,22040
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
-sqlmesh/core/snapshot/definition.py,sha256=L7aNCiUUmsEdkvQOfroB9aZDW6ZitHRph1eYYq1rTLE,85363
+sqlmesh/core/snapshot/definition.py,sha256=JBrvHx9ZPVWxY649IOXOYs4RKPzgLYvfdxXsdCewad0,85738
 sqlmesh/core/snapshot/evaluator.py,sha256=3mjrSp9ggd9uKhuZURrY0HtuCc_WAPsH4PYvQlqmn9o,90443
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=AdqWmbhnvRFqVcFK6L01uCvpBVAYGuMrm4P0D3u_NaU,18334
@@ -284,7 +284,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.199.1.dev17.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.199.1.dev19.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -402,8 +402,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.199.1.dev17.dist-info/METADATA,sha256=pBNtjMaCE4wbfEABFb-Bp0vgqrPvWOk52rBKj95Zvw4,26659
-sqlmesh-0.199.1.dev17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.199.1.dev17.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.199.1.dev17.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.199.1.dev17.dist-info/RECORD,,
+sqlmesh-0.199.1.dev19.dist-info/METADATA,sha256=rBj57KhTPJ8EL5R4NF3jtAeY73CnajeEUNzzQc0viiQ,26659
+sqlmesh-0.199.1.dev19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.199.1.dev19.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.199.1.dev19.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.199.1.dev19.dist-info/RECORD,,