sqlmesh 0.188.2.dev30__tar.gz → 0.188.2.dev31__tar.gz

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.188.2.dev30
+Version: 0.188.2.dev31
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.188.2.dev30'
-__version_tuple__ = version_tuple = (0, 188, 2, 'dev30')
+__version__ = version = '0.188.2.dev31'
+__version_tuple__ = version_tuple = (0, 188, 2, 'dev31')

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/cli/main.py

@@ -457,7 +457,13 @@ def diff(ctx: click.Context, environment: t.Optional[str] = None) -> None:
 @click.option(
     "--diff-rendered",
     is_flag=True,
-    help="Output text differences for the rendered versions of the models and standalone audits",
+    help="Output text differences for the rendered versions of the models and standalone audits.",
+    default=None,
+)
+@click.option(
+    "--explain",
+    is_flag=True,
+    help="Explain the plan instead of applying it.",
     default=None,
 )
 @opt.verbose

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/context.py

@@ -88,7 +88,7 @@ from sqlmesh.core.notification_target import (
     NotificationTarget,
     NotificationTargetManager,
 )
-from sqlmesh.core.plan import Plan, PlanBuilder, SnapshotIntervals
+from sqlmesh.core.plan import Plan, PlanBuilder, SnapshotIntervals, PlanExplainer
 from sqlmesh.core.plan.definition import UserProvidedFlags
 from sqlmesh.core.reference import ReferenceGraph
 from sqlmesh.core.scheduler import Scheduler, CompletionStatus
@@ -1211,6 +1211,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         run: t.Optional[bool] = None,
         diff_rendered: t.Optional[bool] = None,
         skip_linter: t.Optional[bool] = None,
+        explain: t.Optional[bool] = None,
     ) -> Plan:
         """Interactively creates a plan.
 
@@ -1256,6 +1257,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             run: Whether to run latest intervals as part of the plan application.
             diff_rendered: Whether the diff should compare raw vs rendered models
             skip_linter: Linter runs by default so this will skip it if enabled
+            explain: Whether to explain the plan instead of applying it.
 
         Returns:
             The populated Plan object.
@@ -1283,6 +1285,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             run=run,
             diff_rendered=diff_rendered,
             skip_linter=skip_linter,
+            explain=explain,
         )
 
         plan = plan_builder.build()
@@ -1292,6 +1295,9 @@ class GenericContext(BaseContext, t.Generic[C]):
             # or if there are any uncategorized snapshots in the plan
             no_prompts = False
 
+        if explain:
+            auto_apply = True
+
         self.console.plan(
             plan_builder,
             auto_apply if auto_apply is not None else self.config.plan.auto_apply,
@@ -1328,6 +1334,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         run: t.Optional[bool] = None,
         diff_rendered: t.Optional[bool] = None,
         skip_linter: t.Optional[bool] = None,
+        explain: t.Optional[bool] = None,
     ) -> PlanBuilder:
         """Creates a plan builder.
 
@@ -1544,6 +1551,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             interval_end_per_model=max_interval_end_per_model,
             console=self.console,
             user_provided_flags=user_provided_flags,
+            explain=explain or False,
         )
 
     def apply(
@@ -1568,6 +1576,16 @@ class GenericContext(BaseContext, t.Generic[C]):
             return
         if plan.uncategorized:
             raise UncategorizedPlanError("Can't apply a plan with uncategorized changes.")
+
+        if plan.explain:
+            explainer = PlanExplainer(
+                state_reader=self.state_reader,
+                default_catalog=self.default_catalog,
+                console=self.console,
+            )
+            explainer.evaluate(plan.to_evaluatable())
+            return
+
         self.notification_target_manager.notify(
             NotificationEvent.APPLY_START,
             environment=plan.environment_naming_info.name,

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/environment.py

@@ -229,6 +229,19 @@ class Environment(EnvironmentNamingInfo, EnvironmentSummary):
             finalized_ts=self.finalized_ts,
         )
 
+    def can_partially_promote(self, existing_environment: Environment) -> bool:
+        """Returns True if the existing environment can be partially promoted to the current environment.
+
+        Partial promotion means that we don't need to re-create views for snapshots that are already promoted in the
+        target environment.
+        """
+        return (
+            bool(existing_environment.finalized_ts)
+            and not existing_environment.expired
+            and existing_environment.gateway_managed == self.gateway_managed
+            and existing_environment.name == c.PROD
+        )
+
     def _convert_list_to_models_and_store(
         self, field: str, type_: t.Type[PydanticType]
     ) -> t.Optional[t.List[PydanticType]]:

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/plan/__init__.py

@@ -10,3 +10,4 @@ from sqlmesh.core.plan.evaluator import (
     PlanEvaluator as PlanEvaluator,
     update_intervals_for_new_snapshots as update_intervals_for_new_snapshots,
 )
+from sqlmesh.core.plan.explainer import PlanExplainer as PlanExplainer

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/plan/builder.py

@@ -83,6 +83,7 @@ class PlanBuilder:
             environment state, or to use whatever snapshots are in the current environment state even if
             the environment is not finalized.
         interval_end_per_model: The mapping from model FQNs to target end dates.
+        explain: Whether to explain the plan instead of applying it.
     """
 
     def __init__(
@@ -112,6 +113,7 @@ class PlanBuilder:
         enable_preview: bool = False,
         end_bounded: bool = False,
         ensure_finalized_snapshots: bool = False,
+        explain: bool = False,
         interval_end_per_model: t.Optional[t.Dict[str, int]] = None,
         console: t.Optional[PlanBuilderConsole] = None,
         user_provided_flags: t.Optional[t.Dict[str, UserProvidedFlags]] = None,
@@ -142,6 +144,7 @@ class PlanBuilder:
         self._console = console or get_console()
         self._choices: t.Dict[SnapshotId, SnapshotChangeCategory] = {}
         self._user_provided_flags = user_provided_flags
+        self._explain = explain
 
         self._start = start
         if not self._start and (
@@ -273,6 +276,7 @@ class PlanBuilder:
             empty_backfill=self._empty_backfill,
             no_gaps=self._no_gaps,
             forward_only=self._forward_only,
+            explain=self._explain,
             allow_destructive_models=t.cast(t.Set, self._allow_destructive_models),
             include_unmodified=self._include_unmodified,
             environment_ttl=self._environment_ttl,

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/plan/definition.py

@@ -46,6 +46,7 @@ class Plan(PydanticModel, frozen=True):
     include_unmodified: bool
     end_bounded: bool
     ensure_finalized_snapshots: bool
+    explain: bool
 
     environment_ttl: t.Optional[str] = None
     environment_naming_info: EnvironmentNamingInfo

{sqlmesh-0.188.2.dev30 → sqlmesh-0.188.2.dev31}/sqlmesh/core/plan/evaluator.py

@@ -36,7 +36,7 @@ from sqlmesh.core.snapshot import (
     SnapshotCreationFailedError,
 )
 from sqlmesh.utils import CompletionStatus
-from sqlmesh.core.state_sync import StateSync
+from sqlmesh.core.state_sync import StateSync, StateReader
 from sqlmesh.core.state_sync.base import PromotionResult
 from sqlmesh.utils.concurrency import NodeExecutionFailedError
 from sqlmesh.utils.errors import PlanError
@@ -284,23 +284,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             new_snapshots=plan.new_snapshots, plan_id=plan.plan_id
         )
 
-        promoted_snapshot_ids = (
-            set(plan.environment.promoted_snapshot_ids)
-            if plan.environment.promoted_snapshot_ids is not None
-            else None
-        )
-
-        def _should_create(s: Snapshot) -> bool:
-            if not s.is_model or s.is_symbolic:
-                return False
-            # Only create tables for snapshots that we're planning to promote or that were selected for backfill
-            return (
-                plan.is_selected_for_backfill(s.name)
-                or promoted_snapshot_ids is None
-                or s.snapshot_id in promoted_snapshot_ids
-            )
-
-        snapshots_to_create = [s for s in snapshots.values() if _should_create(s)]
+        snapshots_to_create = get_snapshots_to_create(plan, snapshots)
 
         completion_status = None
         progress_stopped = False
@@ -573,32 +557,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
         plan: EvaluatablePlan,
         new_snapshots: t.Dict[SnapshotId, Snapshot],
     ) -> None:
-        # Filter out snapshots that are not categorized as metadata changes on models
-        metadata_snapshots = []
-        for snapshot in new_snapshots.values():
-            if not snapshot.is_metadata or not snapshot.is_model or not snapshot.evaluatable:
-                continue
-
-            metadata_snapshots.append(snapshot)
-
-        # Bulk load all the previous snapshots
-        previous_snapshots = self.state_sync.get_snapshots(
-            [
-                s.previous_version.snapshot_id(s.name)
-                for s in metadata_snapshots
-                if s.previous_version
-            ]
-        ).values()
-
-        # Check if any of the snapshots have modifications to the audits field by comparing the hashes
-        audit_snapshots = {}
-        for snapshot, previous_snapshot in zip(metadata_snapshots, previous_snapshots):
-            new_audits_hash = snapshot.model.audit_metadata_hash()
-            previous_audit_hash = previous_snapshot.model.audit_metadata_hash()
-
-            if snapshot.model.audits and previous_audit_hash != new_audits_hash:
-                audit_snapshots[snapshot.snapshot_id] = snapshot
-
+        audit_snapshots = get_audit_only_snapshots(new_snapshots, self.state_sync)
         if not audit_snapshots:
             return
 
@@ -636,3 +595,52 @@ def update_intervals_for_new_snapshots(
 
     if snapshots_intervals:
         state_sync.add_snapshots_intervals(snapshots_intervals)
+
+
+def get_audit_only_snapshots(
+    new_snapshots: t.Dict[SnapshotId, Snapshot], state_reader: StateReader
+) -> t.Dict[SnapshotId, Snapshot]:
+    metadata_snapshots = []
+    for snapshot in new_snapshots.values():
+        if not snapshot.is_metadata or not snapshot.is_model or not snapshot.evaluatable:
+            continue
+
+        metadata_snapshots.append(snapshot)
+
+    # Bulk load all the previous snapshots
+    previous_snapshots = state_reader.get_snapshots(
+        [s.previous_version.snapshot_id(s.name) for s in metadata_snapshots if s.previous_version]
+    ).values()
+
+    # Check if any of the snapshots have modifications to the audits field by comparing the hashes
+    audit_snapshots = {}
+    for snapshot, previous_snapshot in zip(metadata_snapshots, previous_snapshots):
+        new_audits_hash = snapshot.model.audit_metadata_hash()
+        previous_audit_hash = previous_snapshot.model.audit_metadata_hash()
+
+        if snapshot.model.audits and previous_audit_hash != new_audits_hash:
+            audit_snapshots[snapshot.snapshot_id] = snapshot
+
+    return audit_snapshots
+
+
+def get_snapshots_to_create(
+    plan: EvaluatablePlan, snapshots: t.Dict[SnapshotId, Snapshot]
+) -> t.List[Snapshot]:
+    promoted_snapshot_ids = (
+        set(plan.environment.promoted_snapshot_ids)
+        if plan.environment.promoted_snapshot_ids is not None
+        else None
+    )
+
+    def _should_create(s: Snapshot) -> bool:
+        if not s.is_model or s.is_symbolic:
+            return False
+        # Only create tables for snapshots that we're planning to promote or that were selected for backfill
+        return (
+            plan.is_selected_for_backfill(s.name)
+            or promoted_snapshot_ids is None
+            or s.snapshot_id in promoted_snapshot_ids
+        )
+
+    return [s for s in snapshots.values() if _should_create(s)]

sqlmesh-0.188.2.dev31/sqlmesh/core/plan/explainer.py

@@ -0,0 +1,281 @@
+import abc
+import typing as t
+import logging
+
+from rich.console import Console as RichConsole
+from rich.tree import Tree
+from sqlglot.dialects.dialect import DialectType
+from sqlmesh.core import constants as c
+from sqlmesh.core.console import Console, TerminalConsole, get_console
+from sqlmesh.core.environment import EnvironmentNamingInfo
+from sqlmesh.core.plan.definition import EvaluatablePlan, SnapshotIntervals
+from sqlmesh.core.plan import stages
+from sqlmesh.core.plan.evaluator import (
+    PlanEvaluator,
+)
+from sqlmesh.core.state_sync import StateReader
+from sqlmesh.core.snapshot.definition import (
+    SnapshotInfoMixin,
+)
+from sqlmesh.utils import Verbosity, rich as srich
+from sqlmesh.utils.date import to_ts
+from sqlmesh.utils.errors import SQLMeshError
+
+
+logger = logging.getLogger(__name__)
+
+
+class PlanExplainer(PlanEvaluator):
+    def __init__(
+        self,
+        state_reader: StateReader,
+        default_catalog: t.Optional[str],
+        console: t.Optional[Console] = None,
+    ):
+        self.state_reader = state_reader
+        self.default_catalog = default_catalog
+        self.console = console or get_console()
+
+    def evaluate(
+        self, plan: EvaluatablePlan, circuit_breaker: t.Optional[t.Callable[[], bool]] = None
+    ) -> None:
+        plan_stages = stages.build_plan_stages(plan, self.state_reader, self.default_catalog)
+        explainer_console = _get_explainer_console(
+            self.console, plan.environment, self.default_catalog
+        )
+        explainer_console.explain(plan_stages)
+
+
+class ExplainerConsole(abc.ABC):
+    @abc.abstractmethod
+    def explain(self, stages: t.List[stages.PlanStage]) -> None:
+        pass
+
+
+MAX_TREE_LENGTH = 10
+
+
+class RichExplainerConsole(ExplainerConsole):
+    def __init__(
+        self,
+        environment_naming_info: EnvironmentNamingInfo,
+        dialect: DialectType,
+        default_catalog: t.Optional[str],
+        verbosity: Verbosity = Verbosity.DEFAULT,
+        console: t.Optional[RichConsole] = None,
+    ):
+        self.environment_naming_info = environment_naming_info
+        self.dialect = dialect
+        self.default_catalog = default_catalog
+        self.verbosity = verbosity
+        self.console: RichConsole = console or srich.console
+
+    def explain(self, stages: t.List[stages.PlanStage]) -> None:
+        tree = Tree("[bold]Explained plan[/bold]")
+        for stage in stages:
+            handler_name = f"visit_{_to_snake_case(stage.__class__.__name__)}"
+            if not hasattr(self, handler_name):
+                logger.error("Unexpected stage: %s", stage.__class__.__name__)
+                continue
+            handler = getattr(self, handler_name)
+            result = handler(stage)
+            if result:
+                tree.add(self._limit_tree(result))
+        self.console.print(tree)
+
+    def visit_before_all_stage(self, stage: stages.BeforeAllStage) -> Tree:
+        return Tree("[bold]Execute before all statements[/bold]")
+
+    def visit_after_all_stage(self, stage: stages.AfterAllStage) -> Tree:
+        return Tree("[bold]Execute after all statements[/bold]")
+
+    def visit_physical_layer_update_stage(self, stage: stages.PhysicalLayerUpdateStage) -> Tree:
+        if not stage.snapshots:
+            return Tree("[bold]SKIP: No physical layer updates to perform[/bold]")
+
+        tree = Tree(
+            "[bold]Validate SQL and create physical layer tables and views if they do not exist[/bold]"
+        )
+        for snapshot in stage.snapshots:
+            is_deployable = (
+                stage.deployability_index.is_deployable(snapshot)
+                if self.environment_naming_info.name != c.PROD
+                else True
+            )
+            display_name = self._display_name(snapshot)
+            table_name = snapshot.table_name(is_deployable)
+            model_tree = Tree(f"{display_name} -> {table_name}")
+
+            if snapshot.is_model:
+                if snapshot.model.pre_statements:
+                    model_tree.add("Run pre-statements")
+                if snapshot.model.annotated:
+                    model_tree.add("Dry run model query without inserting results")
+
+            if snapshot.is_view:
+                create_tree = Tree("Create view if it doesn't exist")
+            elif snapshot.is_forward_only and snapshot.previous_versions:
+                prod_table = snapshot.table_name(True)
+                create_tree = Tree(
+                    f"Clone {prod_table} into {table_name} and then update its schema if it doesn't exist"
+                )
+            else:
+                create_tree = Tree("Create table if it doesn't exist")
+
+            if not is_deployable:
+                create_tree.add("[orange1]preview[/orange1]: data will NOT be reused in production")
+            model_tree.add(create_tree)
+
+            if snapshot.is_model and snapshot.model.post_statements:
+                model_tree.add("Run post-statements")
+
+            tree.add(model_tree)
+        return tree
+
+    def visit_audit_only_run_stage(self, stage: stages.AuditOnlyRunStage) -> Tree:
+        tree = Tree("[bold]Audit-only execution[/bold]")
+        for snapshot in stage.snapshots:
+            display_name = self._display_name(snapshot)
+            tree.add(display_name)
+        return tree
+
+    def visit_restatement_stage(self, stage: stages.RestatementStage) -> Tree:
+        tree = Tree("[bold]Invalidate data intervals as part of restatement[/bold]")
+        for snapshot_table_info, interval in stage.snapshot_intervals.items():
+            display_name = self._display_name(snapshot_table_info)
+            tree.add(f"{display_name} [{to_ts(interval[0])} - {to_ts(interval[1])}]")
+        return tree
+
+    def visit_backfill_stage(self, stage: stages.BackfillStage) -> Tree:
+        if not stage.snapshot_to_intervals:
+            return Tree("[bold]SKIP: No model batches to execute[/bold]")
+
+        tree = Tree(
+            "[bold]Backfill models by running their queries and run standalone audits[/bold]"
+        )
+        for snapshot, intervals in stage.snapshot_to_intervals.items():
+            display_name = self._display_name(snapshot)
+            if snapshot.is_model:
+                table_name = snapshot.table_name(stage.deployability_index.is_deployable(snapshot))
+                model_tree = Tree(f"{display_name} -> {table_name}")
+
+                for signal_name, _ in snapshot.model.signals:
+                    model_tree.add(f"Check '{signal_name}' signal")
+
+                if snapshot.model.pre_statements:
+                    model_tree.add("Run pre-statements")
+
+                if snapshot.is_incremental:
+                    current_intervals = (
+                        snapshot.intervals
+                        if stage.deployability_index.is_deployable(snapshot)
+                        else snapshot.dev_intervals
+                    )
+                    if current_intervals:
+                        formatted_range = SnapshotIntervals(
+                            snapshot_id=snapshot.snapshot_id, intervals=intervals
+                        ).format_intervals(snapshot.node.interval_unit)
+                        model_tree.add(
+                            f"Incrementally insert records within the range [{formatted_range}]"
+                        )
+                    else:
+                        # If there are no intervals, the table will be fully refreshed
+                        model_tree.add("Fully refresh table")
+                elif snapshot.is_view:
+                    model_tree.add("Recreate view")
+                else:
+                    model_tree.add("Fully refresh table")
+
+                if snapshot.model.post_statements:
+                    model_tree.add("Run post-statements")
+
+                if snapshot.model.audits:
+                    for audit_name, _ in snapshot.model.audits:
+                        model_tree.add(f"Run '{audit_name}' audit")
+
+                tree.add(model_tree)
+            else:
+                tree.add(f"{display_name} \[standalone audit]")
+        return tree
+
+    def visit_migrate_schemas_stage(self, stage: stages.MigrateSchemasStage) -> Tree:
+        tree = Tree(
+            "[bold]Update schemas (add, drop, alter columns) of production physical tables to reflect forward-only changes[/bold]"
+        )
+        for snapshot in stage.snapshots:
+            display_name = self._display_name(snapshot)
+            table_name = snapshot.table_name(True)
+            tree.add(f"{display_name} -> {table_name}")
+        return tree
+
+    def visit_virtual_layer_update_stage(self, stage: stages.VirtualLayerUpdateStage) -> Tree:
+        tree = Tree(
+            f"[bold]Update the virtual layer for environment '{self.environment_naming_info.name}'[/bold]"
+        )
+        promote_tree = Tree(
+            "[bold]Create or update views in the virtual layer to point at new physical tables and views[/bold]"
+        )
+        for snapshot in stage.promoted_snapshots:
+            display_name = self._display_name(snapshot)
+            table_name = snapshot.table_name(stage.deployability_index.is_representative(snapshot))
+            promote_tree.add(f"{display_name} -> {table_name}")
+
+        demote_tree = Tree(
+            "[bold]Delete views in the virtual layer for models that were removed[/bold]"
+        )
+        for snapshot in stage.demoted_snapshots:
+            display_name = self._display_name(snapshot)
+            demote_tree.add(display_name)
+
+        if stage.promoted_snapshots:
+            tree.add(self._limit_tree(promote_tree))
+        if stage.demoted_snapshots:
+            tree.add(self._limit_tree(demote_tree))
+        return tree
+
+    def visit_environment_record_update_stage(
+        self, stage: stages.EnvironmentRecordUpdateStage
+    ) -> t.Optional[Tree]:
+        return None
+
+    def _display_name(self, snapshot: SnapshotInfoMixin) -> str:
+        return snapshot.display_name(
+            self.environment_naming_info,
+            self.default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None,
+            dialect=self.dialect,
+        )
+
+    def _limit_tree(self, tree: Tree) -> Tree:
+        tree_length = len(tree.children)
+        if tree_length <= MAX_TREE_LENGTH:
+            return tree
+        if self.verbosity < Verbosity.VERY_VERBOSE:
+            tree.children = [
+                tree.children[0],
+                Tree(f".... {tree_length - 2} more ...."),
+                tree.children[-1],
+            ]
+        return tree
+
+
+def _get_explainer_console(
+    console: t.Optional[Console],
+    environment_naming_info: EnvironmentNamingInfo,
+    default_catalog: t.Optional[str],
+) -> ExplainerConsole:
+    console = console or get_console()
+    if not isinstance(console, TerminalConsole):
+        raise SQLMeshError("Plain explaination is only supported in the terminal.")
+    return RichExplainerConsole(
+        environment_naming_info=environment_naming_info,
+        dialect=console.dialect,
+        default_catalog=default_catalog,
+        verbosity=console.verbosity,
+        console=console.console,
+    )
+
+
+def _to_snake_case(name: str) -> str:
+    return "".join(
+        f"_{c.lower()}" if c.isupper() and idx != 0 else c.lower() for idx, c in enumerate(name)
+    )