sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev20__py3-none-any.whl

sqlmesh/core/plan/common.py

@@ -1,19 +1,40 @@
 from __future__ import annotations
+import typing as t
+import logging
+from dataclasses import dataclass, field
 
-from sqlmesh.core.snapshot import Snapshot
+from sqlmesh.core.state_sync import StateReader
+from sqlmesh.core.snapshot import Snapshot, SnapshotId, SnapshotIdAndVersion, SnapshotNameVersion
+from sqlmesh.core.snapshot.definition import Interval
+from sqlmesh.utils.dag import DAG
+from sqlmesh.utils.date import now_timestamp
+
+logger = logging.getLogger(__name__)
 
 
 def should_force_rebuild(old: Snapshot, new: Snapshot) -> bool:
     if new.is_view and new.is_indirect_non_breaking and not new.is_forward_only:
         # View models always need to be rebuilt to reflect updated upstream dependencies
         return True
-    if new.is_seed:
+    if new.is_seed and not (
+        new.is_metadata
+        and new.previous_version
+        and new.previous_version.snapshot_id(new.name) == old.snapshot_id
+    ):
         # Seed models always need to be rebuilt to reflect changes in the seed file
+        # Unless only their metadata has been updated (eg description added) and the seed file has not been touched
         return True
     return is_breaking_kind_change(old, new)
 
 
 def is_breaking_kind_change(old: Snapshot, new: Snapshot) -> bool:
+    if new.is_model != old.is_model:
+        # If one is a model and the other isn't, then we need to rebuild
+        return True
+    if not new.is_model or not old.is_model:
+        # If neither are models, then we don't need to rebuild
+        # Note that the remaining checks only apply to model snapshots
+        return False
     if old.virtual_environment_mode != new.virtual_environment_mode:
         # If the virtual environment mode has changed, then we need to rebuild
         return True
@@ -27,3 +48,176 @@ def is_breaking_kind_change(old: Snapshot, new: Snapshot) -> bool:
         # If the partitioning hasn't changed, then we don't need to rebuild
         return False
     return True
+
+
+@dataclass
+class SnapshotIntervalClearRequest:
+    # affected snapshot
+    snapshot: SnapshotIdAndVersion
+
+    # which interval to clear
+    interval: Interval
+
+    # which environments this snapshot is currently promoted
+    # note that this can be empty if the snapshot exists because its ttl has not expired
+    # but it is not part of any particular environment
+    environment_names: t.Set[str] = field(default_factory=set)
+
+    @property
+    def snapshot_id(self) -> SnapshotId:
+        return self.snapshot.snapshot_id
+
+    @property
+    def sorted_environment_names(self) -> t.List[str]:
+        return list(sorted(self.environment_names))
+
+
+def identify_restatement_intervals_across_snapshot_versions(
+    state_reader: StateReader,
+    prod_restatements: t.Dict[str, Interval],
+    disable_restatement_models: t.Set[str],
+    loaded_snapshots: t.Dict[SnapshotId, Snapshot],
+    current_ts: t.Optional[int] = None,
+) -> t.Dict[SnapshotId, SnapshotIntervalClearRequest]:
+    """
+    Given a map of snapshot names + intervals to restate in prod:
+        - Look up matching snapshots (match based on name - regardless of version, to get all versions)
+        - For each match, also match downstream snapshots in each dev environment while filtering out models that have restatement disabled
+        - Return a list of all snapshots that are affected + the interval that needs to be cleared for each
+
+    The goal here is to produce a list of intervals to invalidate across all dev snapshots so that a subsequent plan or
+    cadence run in those environments causes the intervals to be repopulated.
+    """
+    if not prod_restatements:
+        return {}
+
+    # Although :loaded_snapshots is sourced from RestatementStage.all_snapshots, since the only time we ever need
+    # to clear intervals across all environments is for prod, the :loaded_snapshots here are always from prod
+    prod_name_versions: t.Set[SnapshotNameVersion] = {
+        s.name_version for s in loaded_snapshots.values()
+    }
+
+    snapshot_intervals_to_clear: t.Dict[SnapshotId, SnapshotIntervalClearRequest] = {}
+
+    for env_summary in state_reader.get_environments_summary():
+        # Fetch the full environment object one at a time to avoid loading all environments into memory at once
+        env = state_reader.get_environment(env_summary.name)
+        if not env:
+            logger.warning("Environment %s not found", env_summary.name)
+            continue
+
+        snapshots_by_name = {s.name: s.table_info for s in env.snapshots}
+
+        # We dont just restate matching snapshots, we also have to restate anything downstream of them
+        # so that if A gets restated in prod and dev has A <- B <- C, B and C get restated in dev
+        env_dag = DAG({s.name: {p.name for p in s.parents} for s in env.snapshots})
+
+        for restate_snapshot_name, interval in prod_restatements.items():
+            if restate_snapshot_name not in snapshots_by_name:
+                # snapshot is not promoted in this environment
+                continue
+
+            affected_snapshot_names = [
+                x
+                for x in ([restate_snapshot_name] + env_dag.downstream(restate_snapshot_name))
+                if x not in disable_restatement_models
+            ]
+
+            for affected_snapshot_name in affected_snapshot_names:
+                affected_snapshot = snapshots_by_name[affected_snapshot_name]
+
+                # Don't clear intervals for a dev snapshot if it shares the same physical version with prod.
+                # Otherwise, prod will be affected by what should be a dev operation
+                if affected_snapshot.name_version in prod_name_versions:
+                    continue
+
+                clear_request = snapshot_intervals_to_clear.get(affected_snapshot.snapshot_id)
+                if not clear_request:
+                    clear_request = SnapshotIntervalClearRequest(
+                        snapshot=affected_snapshot.id_and_version, interval=interval
+                    )
+                    snapshot_intervals_to_clear[affected_snapshot.snapshot_id] = clear_request
+
+                clear_request.environment_names |= set([env.name])
+
+    # snapshot_intervals_to_clear now contains the entire hierarchy of affected snapshots based
+    # on building the DAG for each environment and including downstream snapshots
+    # but, what if there are affected snapshots that arent part of any environment?
+    unique_snapshot_names = set(snapshot_id.name for snapshot_id in snapshot_intervals_to_clear)
+
+    current_ts = current_ts or now_timestamp()
+    all_matching_non_prod_snapshots = {
+        s.snapshot_id: s
+        for s in state_reader.get_snapshots_by_names(
+            snapshot_names=unique_snapshot_names, current_ts=current_ts, exclude_expired=True
+        )
+        # Don't clear intervals for a snapshot if it shares the same physical version with prod.
+        # Otherwise, prod will be affected by what should be a dev operation
+        if s.name_version not in prod_name_versions
+    }
+
+    # identify the ones that we havent picked up yet, which are the ones that dont exist in any environment
+    if remaining_snapshot_ids := set(all_matching_non_prod_snapshots).difference(
+        snapshot_intervals_to_clear
+    ):
+        # these snapshot id's exist in isolation and may be related to a downstream dependency of the :prod_restatements,
+        # rather than directly related, so we can't simply look up the interval to clear based on :prod_restatements.
+        # To figure out the interval that should be cleared, we can match to the existing list based on name
+        # and conservatively take the widest interval that shows up
+        snapshot_name_to_widest_interval: t.Dict[str, Interval] = {}
+        for s_id, clear_request in snapshot_intervals_to_clear.items():
+            current_start, current_end = snapshot_name_to_widest_interval.get(
+                s_id.name, clear_request.interval
+            )
+            next_start, next_end = clear_request.interval
+
+            next_start = min(current_start, next_start)
+            next_end = max(current_end, next_end)
+
+            snapshot_name_to_widest_interval[s_id.name] = (next_start, next_end)
+
+        for remaining_snapshot_id in remaining_snapshot_ids:
+            remaining_snapshot = all_matching_non_prod_snapshots[remaining_snapshot_id]
+            snapshot_intervals_to_clear[remaining_snapshot_id] = SnapshotIntervalClearRequest(
+                snapshot=remaining_snapshot,
+                interval=snapshot_name_to_widest_interval[remaining_snapshot_id.name],
+            )
+
+    # for any affected full_history_restatement_only snapshots, we need to widen the intervals being restated to
+    # include the whole time range for that snapshot. This requires a call to state to load the full snapshot record,
+    # so we only do it if necessary
+    full_history_restatement_snapshot_ids = [
+        # FIXME: full_history_restatement_only is just one indicator that the snapshot can only be fully refreshed, the other one is Model.depends_on_self
+        # however, to figure out depends_on_self, we have to render all the model queries which, alongside having to fetch full snapshots from state,
+        # is problematic in secure environments that are deliberately isolated from arbitrary user code (since rendering a query may require user macros to be present)
+        # So for now, these are not considered
+        s_id
+        for s_id, s in snapshot_intervals_to_clear.items()
+        if s.snapshot.full_history_restatement_only
+    ]
+    if full_history_restatement_snapshot_ids:
+        # only load full snapshot records that we havent already loaded
+        additional_snapshots = state_reader.get_snapshots(
+            [
+                s.snapshot_id
+                for s in full_history_restatement_snapshot_ids
+                if s.snapshot_id not in loaded_snapshots
+            ]
+        )
+
+        all_snapshots = loaded_snapshots | additional_snapshots
+
+        for full_snapshot_id in full_history_restatement_snapshot_ids:
+            full_snapshot = all_snapshots[full_snapshot_id]
+            intervals_to_clear = snapshot_intervals_to_clear[full_snapshot_id]
+
+            original_start, original_end = intervals_to_clear.interval
+
+            # get_removal_interval() widens intervals if necessary
+            new_interval = full_snapshot.get_removal_interval(
+                start=original_start, end=original_end
+            )
+
+            intervals_to_clear.interval = new_interval
+
+    return snapshot_intervals_to_clear

sqlmesh/core/plan/definition.py

@@ -58,7 +58,18 @@ class Plan(PydanticModel, frozen=True):
     indirectly_modified: t.Dict[SnapshotId, t.Set[SnapshotId]]
 
     deployability_index: DeployabilityIndex
+    selected_models_to_restate: t.Optional[t.Set[str]] = None
+    """Models that have been explicitly selected for restatement by a user"""
     restatements: t.Dict[SnapshotId, Interval]
+    """
+    All models being restated, which are typically the explicitly selected ones + their downstream dependencies.
+
+    Note that dev previews are also considered restatements, so :selected_models_to_restate can be empty
+    while :restatements is still populated with dev previews
+    """
+    restate_all_snapshots: bool
+    """Whether or not to clear intervals from state for other versions of the models listed in :restatements"""
+
     start_override_per_model: t.Optional[t.Dict[str, datetime]]
     end_override_per_model: t.Optional[t.Dict[str, datetime]]
 
@@ -202,8 +213,8 @@ class Plan(PydanticModel, frozen=True):
 
         snapshots_by_name = self.context_diff.snapshots_by_name
         snapshots = [s.table_info for s in self.snapshots.values()]
-        promoted_snapshot_ids = None
-        if self.is_dev and not self.include_unmodified:
+        promotable_snapshot_ids = None
+        if self.is_dev:
             if self.selected_models_to_backfill is not None:
                 # Only promote models that have been explicitly selected for backfill.
                 promotable_snapshot_ids = {
@@ -214,12 +225,14 @@ class Plan(PydanticModel, frozen=True):
                         if m in snapshots_by_name
                     ],
                 }
-            else:
+            elif not self.include_unmodified:
                 promotable_snapshot_ids = self.context_diff.promotable_snapshot_ids.copy()
 
-            promoted_snapshot_ids = [
-                s.snapshot_id for s in snapshots if s.snapshot_id in promotable_snapshot_ids
-            ]
+        promoted_snapshot_ids = (
+            [s.snapshot_id for s in snapshots if s.snapshot_id in promotable_snapshot_ids]
+            if promotable_snapshot_ids is not None
+            else None
+        )
 
         previous_finalized_snapshots = (
             self.context_diff.environment_snapshots
@@ -259,6 +272,7 @@ class Plan(PydanticModel, frozen=True):
             skip_backfill=self.skip_backfill,
             empty_backfill=self.empty_backfill,
             restatements={s.name: i for s, i in self.restatements.items()},
+            restate_all_snapshots=self.restate_all_snapshots,
             is_dev=self.is_dev,
             allow_destructive_models=self.allow_destructive_models,
             allow_additive_models=self.allow_additive_models,
@@ -303,6 +317,7 @@ class EvaluatablePlan(PydanticModel):
     skip_backfill: bool
     empty_backfill: bool
     restatements: t.Dict[str, Interval]
+    restate_all_snapshots: bool
     is_dev: bool
     allow_destructive_models: t.Set[str]
     allow_additive_models: t.Set[str]

sqlmesh/core/plan/evaluator.py

@@ -22,7 +22,7 @@ from sqlmesh.core import constants as c
 from sqlmesh.core.console import Console, get_console
 from sqlmesh.core.environment import EnvironmentNamingInfo, execute_environment_statements
 from sqlmesh.core.macros import RuntimeStage
-from sqlmesh.core.snapshot.definition import Interval, to_view_mapping
+from sqlmesh.core.snapshot.definition import to_view_mapping, SnapshotTableInfo
 from sqlmesh.core.plan import stages
 from sqlmesh.core.plan.definition import EvaluatablePlan
 from sqlmesh.core.scheduler import Scheduler
@@ -33,17 +33,15 @@ from sqlmesh.core.snapshot import (
     SnapshotIntervals,
     SnapshotId,
     SnapshotInfoLike,
-    SnapshotTableInfo,
     SnapshotCreationFailedError,
-    SnapshotNameVersion,
 )
 from sqlmesh.utils import to_snake_case
 from sqlmesh.core.state_sync import StateSync
+from sqlmesh.core.plan.common import identify_restatement_intervals_across_snapshot_versions
 from sqlmesh.utils import CorrelationId
 from sqlmesh.utils.concurrency import NodeExecutionFailedError
-from sqlmesh.utils.errors import PlanError, SQLMeshError
-from sqlmesh.utils.dag import DAG
-from sqlmesh.utils.date import now
+from sqlmesh.utils.errors import PlanError, ConflictingPlanError, SQLMeshError
+from sqlmesh.utils.date import now, to_timestamp
 
 logger = logging.getLogger(__name__)
 
@@ -260,6 +258,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             allow_additive_snapshots=plan.allow_additive_models,
             selected_snapshot_ids=stage.selected_snapshot_ids,
             selected_models=plan.selected_models,
+            is_restatement=bool(plan.restatements),
         )
         if errors:
             raise PlanError("Plan application failed.")
@@ -289,27 +288,78 @@ class BuiltInPlanEvaluator(PlanEvaluator):
     def visit_restatement_stage(
         self, stage: stages.RestatementStage, plan: EvaluatablePlan
     ) -> None:
-        snapshot_intervals_to_restate = {(s, i) for s, i in stage.snapshot_intervals.items()}
-
-        # Restating intervals on prod plans should mean that the intervals are cleared across
-        # all environments, not just the version currently in prod
-        # This ensures that work done in dev environments can still be promoted to prod
-        # by forcing dev environments to re-run intervals that changed in prod
+        # Restating intervals on prod plans means that once the data for the intervals being restated has been backfilled
+        # (which happens in the backfill stage) then we need to clear those intervals *from state* across all other environments.
+        #
+        # This ensures that work done in dev environments can still be promoted to prod by forcing dev environments to
+        # re-run intervals that changed in prod (because after this stage runs they are cleared from state and thus show as missing)
+        #
+        # It also means that any new dev environments created while this restatement plan was running also get the
+        # correct intervals cleared because we look up matching snapshots as at right now and not as at the time the plan
+        # was created, which could have been several hours ago if there was a lot of data to restate.
         #
         # Without this rule, its possible that promoting a dev table to prod will introduce old data to prod
-        snapshot_intervals_to_restate.update(
-            self._restatement_intervals_across_all_environments(
-                prod_restatements=plan.restatements,
-                disable_restatement_models=plan.disabled_restatement_models,
-                loaded_snapshots={s.snapshot_id: s for s in stage.all_snapshots.values()},
-            )
-        )
 
-        self.state_sync.remove_intervals(
-            snapshot_intervals=list(snapshot_intervals_to_restate),
-            remove_shared_versions=plan.is_prod,
+        intervals_to_clear = identify_restatement_intervals_across_snapshot_versions(
+            state_reader=self.state_sync,
+            prod_restatements=plan.restatements,
+            disable_restatement_models=plan.disabled_restatement_models,
+            loaded_snapshots={s.snapshot_id: s for s in stage.all_snapshots.values()},
+            current_ts=to_timestamp(plan.execution_time or now()),
         )
 
+        if not intervals_to_clear:
+            # Nothing to do
+            return
+
+        # While the restatements were being processed, did any of the snapshots being restated get new versions deployed?
+        # If they did, they will not reflect the data that just got restated, so we need to notify the user
+        deployed_during_restatement: t.Dict[
+            str, t.Tuple[SnapshotTableInfo, SnapshotTableInfo]
+        ] = {}  # tuple of (restated_snapshot, current_prod_snapshot)
+
+        if deployed_env := self.state_sync.get_environment(plan.environment.name):
+            promoted_snapshots_by_name = {s.name: s for s in deployed_env.snapshots}
+
+            for name in plan.restatements:
+                snapshot = stage.all_snapshots[name]
+                version = snapshot.table_info.version
+                if (
+                    prod_snapshot := promoted_snapshots_by_name.get(name)
+                ) and prod_snapshot.version != version:
+                    deployed_during_restatement[name] = (
+                        snapshot.table_info,
+                        prod_snapshot.table_info,
+                    )
+
+        # we need to *not* clear the intervals on the snapshots where new versions were deployed while the restatement was running in order to prevent
+        # subsequent plans from having unexpected intervals to backfill.
+        # we instead list the affected models and abort the plan with an error so the user can decide what to do
+        # (either re-attempt the restatement plan or leave things as they are)
+        filtered_intervals_to_clear = [
+            (s.snapshot, s.interval)
+            for s in intervals_to_clear.values()
+            if s.snapshot.name not in deployed_during_restatement
+        ]
+
+        if filtered_intervals_to_clear:
+            # We still clear intervals in other envs for models that were successfully restated without having new versions promoted during restatement
+            self.state_sync.remove_intervals(
+                snapshot_intervals=filtered_intervals_to_clear,
+                remove_shared_versions=plan.is_prod,
+            )
+
+        if deployed_env and deployed_during_restatement:
+            self.console.log_models_updated_during_restatement(
+                list(deployed_during_restatement.values()),
+                plan.environment.naming_info,
+                self.default_catalog,
+            )
+            raise ConflictingPlanError(
+                f"Another plan ({deployed_env.summary.plan_id}) deployed new versions of {len(deployed_during_restatement)} models in the target environment '{plan.environment.name}' while they were being restated by this plan.\n"
+                "Please re-apply your plan if these new versions should be restated."
+            )
+
     def visit_environment_record_update_stage(
         self, stage: stages.EnvironmentRecordUpdateStage, plan: EvaluatablePlan
     ) -> None:
@@ -422,97 +472,6 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             on_complete=on_complete,
         )
 
-    def _restatement_intervals_across_all_environments(
-        self,
-        prod_restatements: t.Dict[str, Interval],
-        disable_restatement_models: t.Set[str],
-        loaded_snapshots: t.Dict[SnapshotId, Snapshot],
-    ) -> t.Set[t.Tuple[SnapshotTableInfo, Interval]]:
-        """
-        Given a map of snapshot names + intervals to restate in prod:
-         - Look up matching snapshots across all environments (match based on name - regardless of version)
-         - For each match, also match downstream snapshots while filtering out models that have restatement disabled
-         - Return all matches mapped to the intervals of the prod snapshot being restated
-
-        The goal here is to produce a list of intervals to invalidate across all environments so that a cadence
-        run in those environments causes the intervals to be repopulated
-        """
-        if not prod_restatements:
-            return set()
-
-        prod_name_versions: t.Set[SnapshotNameVersion] = {
-            s.name_version for s in loaded_snapshots.values()
-        }
-
-        snapshots_to_restate: t.Dict[SnapshotId, t.Tuple[SnapshotTableInfo, Interval]] = {}
-
-        for env_summary in self.state_sync.get_environments_summary():
-            # Fetch the full environment object one at a time to avoid loading all environments into memory at once
-            env = self.state_sync.get_environment(env_summary.name)
-            if not env:
-                logger.warning("Environment %s not found", env_summary.name)
-                continue
-
-            keyed_snapshots = {s.name: s.table_info for s in env.snapshots}
-
-            # We dont just restate matching snapshots, we also have to restate anything downstream of them
-            # so that if A gets restated in prod and dev has A <- B <- C, B and C get restated in dev
-            env_dag = DAG({s.name: {p.name for p in s.parents} for s in env.snapshots})
-
-            for restatement, intervals in prod_restatements.items():
-                if restatement not in keyed_snapshots:
-                    continue
-                affected_snapshot_names = [
-                    x
-                    for x in ([restatement] + env_dag.downstream(restatement))
-                    if x not in disable_restatement_models
-                ]
-                snapshots_to_restate.update(
-                    {
-                        keyed_snapshots[a].snapshot_id: (keyed_snapshots[a], intervals)
-                        for a in affected_snapshot_names
-                        # Don't restate a snapshot if it shares the version with a snapshot in prod
-                        if keyed_snapshots[a].name_version not in prod_name_versions
-                    }
-                )
-
-        # for any affected full_history_restatement_only snapshots, we need to widen the intervals being restated to
-        # include the whole time range for that snapshot. This requires a call to state to load the full snapshot record,
-        # so we only do it if necessary
-        full_history_restatement_snapshot_ids = [
-            # FIXME: full_history_restatement_only is just one indicator that the snapshot can only be fully refreshed, the other one is Model.depends_on_self
-            # however, to figure out depends_on_self, we have to render all the model queries which, alongside having to fetch full snapshots from state,
-            # is problematic in secure environments that are deliberately isolated from arbitrary user code (since rendering a query may require user macros to be present)
-            # So for now, these are not considered
-            s_id
-            for s_id, s in snapshots_to_restate.items()
-            if s[0].full_history_restatement_only
-        ]
-        if full_history_restatement_snapshot_ids:
-            # only load full snapshot records that we havent already loaded
-            additional_snapshots = self.state_sync.get_snapshots(
-                [
-                    s.snapshot_id
-                    for s in full_history_restatement_snapshot_ids
-                    if s.snapshot_id not in loaded_snapshots
-                ]
-            )
-
-            all_snapshots = loaded_snapshots | additional_snapshots
-
-            for full_snapshot_id in full_history_restatement_snapshot_ids:
-                full_snapshot = all_snapshots[full_snapshot_id]
-                _, original_intervals = snapshots_to_restate[full_snapshot_id]
-                original_start, original_end = original_intervals
-
-                # get_removal_interval() widens intervals if necessary
-                new_intervals = full_snapshot.get_removal_interval(
-                    start=original_start, end=original_end
-                )
-                snapshots_to_restate[full_snapshot_id] = (full_snapshot.table_info, new_intervals)
-
-        return set(snapshots_to_restate.values())
-
     def _update_intervals_for_new_snapshots(self, snapshots: t.Collection[Snapshot]) -> None:
         snapshots_intervals: t.List[SnapshotIntervals] = []
         for snapshot in snapshots:

sqlmesh/core/plan/explainer.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 import abc
 import typing as t
 import logging
+from dataclasses import dataclass
+from collections import defaultdict
 
 from rich.console import Console as RichConsole
 from rich.tree import Tree
@@ -8,6 +12,10 @@ from sqlglot.dialects.dialect import DialectType
 from sqlmesh.core import constants as c
 from sqlmesh.core.console import Console, TerminalConsole, get_console
 from sqlmesh.core.environment import EnvironmentNamingInfo
+from sqlmesh.core.plan.common import (
+    SnapshotIntervalClearRequest,
+    identify_restatement_intervals_across_snapshot_versions,
+)
 from sqlmesh.core.plan.definition import EvaluatablePlan, SnapshotIntervals
 from sqlmesh.core.plan import stages
 from sqlmesh.core.plan.evaluator import (
@@ -16,6 +24,8 @@ from sqlmesh.core.plan.evaluator import (
 from sqlmesh.core.state_sync import StateReader
 from sqlmesh.core.snapshot.definition import (
     SnapshotInfoMixin,
+    SnapshotIdAndVersion,
+    model_display_name,
 )
 from sqlmesh.utils import Verbosity, rich as srich, to_snake_case
 from sqlmesh.utils.date import to_ts
@@ -45,6 +55,15 @@ class PlanExplainer(PlanEvaluator):
         explainer_console = _get_explainer_console(
             self.console, plan.environment, self.default_catalog
         )
+
+        # add extra metadata that's only needed at this point for better --explain output
+        plan_stages = [
+            ExplainableRestatementStage.from_restatement_stage(stage, self.state_reader, plan)
+            if isinstance(stage, stages.RestatementStage)
+            else stage
+            for stage in plan_stages
+        ]
+
         explainer_console.explain(plan_stages)
 
 
@@ -54,6 +73,41 @@ class ExplainerConsole(abc.ABC):
         pass
 
 
+@dataclass
+class ExplainableRestatementStage(stages.RestatementStage):
+    """
+    This brings forward some calculations that would usually be done in the evaluator so the user can be given a better indication
+    of what might happen when they ask for the plan to be explained
+    """
+
+    snapshot_intervals_to_clear: t.Dict[str, t.List[SnapshotIntervalClearRequest]]
+    """Which snapshots from other environments would have intervals cleared as part of restatement, grouped by name."""
+
+    @classmethod
+    def from_restatement_stage(
+        cls: t.Type[ExplainableRestatementStage],
+        stage: stages.RestatementStage,
+        state_reader: StateReader,
+        plan: EvaluatablePlan,
+    ) -> ExplainableRestatementStage:
+        all_restatement_intervals = identify_restatement_intervals_across_snapshot_versions(
+            state_reader=state_reader,
+            prod_restatements=plan.restatements,
+            disable_restatement_models=plan.disabled_restatement_models,
+            loaded_snapshots={s.snapshot_id: s for s in stage.all_snapshots.values()},
+        )
+
+        # Group the interval clear requests by snapshot name to make them easier to write to the console
+        snapshot_intervals_to_clear = defaultdict(list)
+        for clear_request in all_restatement_intervals.values():
+            snapshot_intervals_to_clear[clear_request.snapshot.name].append(clear_request)
+
+        return cls(
+            snapshot_intervals_to_clear=snapshot_intervals_to_clear,
+            all_snapshots=stage.all_snapshots,
+        )
+
+
 MAX_TREE_LENGTH = 10
 
 
@@ -146,11 +200,37 @@ class RichExplainerConsole(ExplainerConsole):
             tree.add(display_name)
         return tree
 
-    def visit_restatement_stage(self, stage: stages.RestatementStage) -> Tree:
-        tree = Tree("[bold]Invalidate data intervals as part of restatement[/bold]")
-        for snapshot_table_info, interval in stage.snapshot_intervals.items():
-            display_name = self._display_name(snapshot_table_info)
-            tree.add(f"{display_name} [{to_ts(interval[0])} - {to_ts(interval[1])}]")
+    def visit_explainable_restatement_stage(self, stage: ExplainableRestatementStage) -> Tree:
+        return self.visit_restatement_stage(stage)
+
+    def visit_restatement_stage(
+        self, stage: t.Union[ExplainableRestatementStage, stages.RestatementStage]
+    ) -> Tree:
+        tree = Tree(
+            "[bold]Invalidate data intervals in state for development environments to prevent old data from being promoted[/bold]\n"
+            "This only affects state and will not clear physical data from the tables until the next plan for each environment"
+        )
+
+        if isinstance(stage, ExplainableRestatementStage) and (
+            snapshot_intervals := stage.snapshot_intervals_to_clear
+        ):
+            for name, clear_requests in snapshot_intervals.items():
+                display_name = model_display_name(
+                    name, self.environment_naming_info, self.default_catalog, self.dialect
+                )
+                interval_start = min(cr.interval[0] for cr in clear_requests)
+                interval_end = max(cr.interval[1] for cr in clear_requests)
+
+                if not interval_start or not interval_end:
+                    continue
+
+                node = tree.add(f"{display_name} [{to_ts(interval_start)} - {to_ts(interval_end)}]")
+
+                all_environment_names = sorted(
+                    set(env_name for cr in clear_requests for env_name in cr.environment_names)
+                )
+                node.add("in environments: " + ", ".join(all_environment_names))
+
         return tree
 
     def visit_backfill_stage(self, stage: stages.BackfillStage) -> Tree:
@@ -265,12 +345,14 @@ class RichExplainerConsole(ExplainerConsole):
 
     def _display_name(
         self,
-        snapshot: SnapshotInfoMixin,
+        snapshot: t.Union[SnapshotInfoMixin, SnapshotIdAndVersion],
         environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
     ) -> str:
         return snapshot.display_name(
-            environment_naming_info or self.environment_naming_info,
-            self.default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None,
+            environment_naming_info=environment_naming_info or self.environment_naming_info,
+            default_catalog=self.default_catalog
+            if self.verbosity < Verbosity.VERY_VERBOSE
+            else None,
             dialect=self.dialect,
         )