sqlmesh 0.209.1.dev63__tar.gz → 0.209.1.dev65__tar.gz

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.209.1.dev63
+Version: 0.209.1.dev65
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/multi/repo_1/models/a.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name bronze.a
+  name bronze.a,
+  kind FULL
 );
 
 SELECT

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/multi/repo_1/models/b.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name bronze.b
+  name bronze.b,
+  kind FULL
 );
 
 SELECT

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/multi/repo_2/models/c.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.c
+  name silver.c,
+  kind FULL
 );
 
 SELECT DISTINCT col_a

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/multi/repo_2/models/d.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.d
+  name silver.d,
+  kind FULL
 );
 
 SELECT

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/multi/repo_2/models/e.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.e
+  name silver.e,
+  kind FULL
 );
 
 SELECT

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/examples/sushi/config.py

@@ -1,6 +1,6 @@
 import os
 
-from sqlmesh.core.config.common import VirtualEnvironmentMode
+from sqlmesh.core.config.common import VirtualEnvironmentMode, TableNamingConvention
 from sqlmesh.core.config import (
     AutoCategorizationMode,
     BigQueryConnectionConfig,
@@ -27,6 +27,11 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
 defaults = {"dialect": "duckdb"}
 model_defaults = ModelDefaultsConfig(**defaults)
 model_defaults_iceberg = ModelDefaultsConfig(**defaults, storage_format="iceberg")
+before_all = [
+    "CREATE SCHEMA IF NOT EXISTS raw",
+    "DROP VIEW IF EXISTS raw.demographics",
+    "CREATE VIEW raw.demographics AS (SELECT 1 AS customer_id, '00000' AS zip)",
+]
 
 
 # A DuckDB config, in-memory by default.
@@ -52,6 +57,7 @@ config = Config(
             "nomissingexternalmodels",
         ],
     ),
+    before_all=before_all,
 )
 
 bigquery_config = Config(
@@ -63,6 +69,7 @@ bigquery_config = Config(
     },
     default_gateway="bq",
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A configuration used for SQLMesh tests.
@@ -75,6 +82,7 @@ test_config = Config(
         )
     ),
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A configuration used for SQLMesh tests with virtual environment mode set to DEV_ONLY.
@@ -84,7 +92,7 @@ test_config_virtual_environment_mode_dev_only = test_config.copy(
         "plan": PlanConfig(
             auto_categorize_changes=CategorizerConfig.all_full(),
         ),
-    }
+    },
 )
 
 # A DuckDB config with a physical schema map.
@@ -92,6 +100,7 @@ map_config = Config(
     default_connection=DuckDBConnectionConfig(),
     physical_schema_mapping={"^sushi$": "company_internal"},
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A config representing isolated systems with a gateway per system
@@ -103,6 +112,7 @@ isolated_systems_config = Config(
     },
     default_gateway="dev",
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 required_approvers_config = Config(
@@ -137,6 +147,7 @@ required_approvers_config = Config(
         ),
     ],
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 
@@ -144,12 +155,13 @@ environment_suffix_table_config = Config(
     default_connection=DuckDBConnectionConfig(),
     model_defaults=model_defaults,
     environment_suffix_target=EnvironmentSuffixTarget.TABLE,
+    before_all=before_all,
 )
 
 environment_suffix_catalog_config = environment_suffix_table_config.model_copy(
     update={
         "environment_suffix_target": EnvironmentSuffixTarget.CATALOG,
-    }
+    },
 )
 
 CATALOGS = {
@@ -161,6 +173,7 @@ local_catalogs = Config(
     default_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
     default_test_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 environment_catalog_mapping_config = Config(
@@ -177,4 +190,13 @@ environment_catalog_mapping_config = Config(
         "^prod$": "prod_catalog",
         ".*": "dev_catalog",
     },
+    before_all=before_all,
+)
+
+hash_md5_naming_config = config.copy(
+    update={"physical_table_naming_convention": TableNamingConvention.HASH_MD5}
+)
+
+table_only_naming_config = config.copy(
+    update={"physical_table_naming_convention": TableNamingConvention.TABLE_ONLY}
 )

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.209.1.dev63'
-__version_tuple__ = version_tuple = (0, 209, 1, 'dev63')
+__version__ = version = '0.209.1.dev65'
+__version_tuple__ = version_tuple = (0, 209, 1, 'dev65')
 
-__commit_id__ = commit_id = 'g31763bf76'
+__commit_id__ = commit_id = 'geb17e2447'

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/console.py

@@ -428,6 +428,7 @@ class Console(
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Updates the snapshot evaluation progress."""
 
@@ -575,6 +576,7 @@ class NoopConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         pass
 
@@ -1056,6 +1058,7 @@ class TerminalConsole(Console):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         """Update the snapshot evaluation progress."""
         if (
@@ -3639,6 +3642,7 @@ class DatabricksMagicConsole(CaptureTerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
         view_name, loaded_batches = self.evaluation_batch_progress[snapshot.snapshot_id]
 
@@ -3808,11 +3812,15 @@ class DebuggerTerminalConsole(TerminalConsole):
         num_audits_passed: int,
         num_audits_failed: int,
         audit_only: bool = False,
+        auto_restatement_triggers: t.Optional[t.List[SnapshotId]] = None,
     ) -> None:
-        message = f"Evaluating {snapshot.name} | batch={batch_idx} | duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
+        message = f"Evaluated {snapshot.name} | batch={batch_idx} | duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
+
+        if auto_restatement_triggers:
+            message += f" | auto_restatement_triggers=[{', '.join(trigger.name for trigger in auto_restatement_triggers)}]"
 
         if audit_only:
-            message = f"Auditing {snapshot.name} duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
+            message = f"Audited {snapshot.name} | duration={duration_ms}ms | num_audits_passed={num_audits_passed} | num_audits_failed={num_audits_failed}"
 
         self._write(message)
 
@@ -4074,8 +4082,8 @@ def _format_node_errors(errors: t.List[NodeExecutionFailedError]) -> t.Dict[str,
         node_name = ""
         if isinstance(error.node, SnapshotId):
             node_name = error.node.name
-        elif isinstance(error.node, tuple):
-            node_name = error.node[0]
+        elif hasattr(error.node, "snapshot_name"):
+            node_name = error.node.snapshot_name
 
         msg = _format_node_error(error)
         msg = "  " + msg.replace("\n", "\n  ")

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/context_diff.py

@@ -222,7 +222,9 @@ class ContextDiff(PydanticModel):
             infer_python_dependencies=infer_python_dependencies,
         )
 
-        previous_environment_statements = state_reader.get_environment_statements(environment)
+        previous_environment_statements = (
+            state_reader.get_environment_statements(env.name) if env else []
+        )
 
         if existing_env and always_recreate_environment:
             previous_plan_id: t.Optional[str] = existing_env.plan_id
@@ -288,7 +290,7 @@ class ContextDiff(PydanticModel):
             previous_finalized_snapshots=env.previous_finalized_snapshots,
             previous_requirements=env.requirements,
             requirements=env.requirements,
-            previous_environment_statements=[],
+            previous_environment_statements=environment_statements,
             environment_statements=environment_statements,
             previous_gateway_managed_virtual_layer=env.gateway_managed,
             gateway_managed_virtual_layer=env.gateway_managed,

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/engine_adapter/base.py

@@ -443,14 +443,20 @@ class EngineAdapter:
             target_table=target_table,
             source_columns=source_columns,
         )
+        if not target_columns_to_types and table_exists:
+            target_columns_to_types = self.columns(target_table)
         query = source_queries[0].query_factory()
-        target_columns_to_types = target_columns_to_types or self.columns(target_table)
         self_referencing = any(
             quote_identifiers(table) == quote_identifiers(target_table)
             for table in query.find_all(exp.Table)
         )
         # If a query references itself then it must have a table created regardless of approach used.
         if self_referencing:
+            if not target_columns_to_types:
+                raise SQLMeshError(
+                    f"Cannot create a self-referencing table {target_table.sql(dialect=self.dialect)} without knowing the column types. "
+                    "Try casting the columns to an expected type or defining the columns in the model metadata. "
+                )
             self._create_table_from_columns(
                 target_table,
                 target_columns_to_types,
@@ -472,6 +478,7 @@ class EngineAdapter:
                 **kwargs,
             )
         if self_referencing:
+            assert target_columns_to_types is not None
             with self.temp_table(
                 self._select_columns(target_columns_to_types).from_(target_table),
                 name=target_table,

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/engine_adapter/base_postgres.py

@@ -53,7 +53,9 @@ class BasePostgresEngineAdapter(EngineAdapter):
         self.execute(sql)
         resp = self.cursor.fetchall()
         if not resp:
-            raise SQLMeshError("Could not get columns for table '%s'. Table not found.", table_name)
+            raise SQLMeshError(
+                f"Could not get columns for table '{table.sql(dialect=self.dialect)}'. Table not found."
+            )
         return {
             column_name: exp.DataType.build(data_type, dialect=self.dialect, udt=True)
             for column_name, data_type in resp

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/plan/builder.py

@@ -16,7 +16,7 @@ from sqlmesh.core.config import (
 )
 from sqlmesh.core.context_diff import ContextDiff
 from sqlmesh.core.environment import EnvironmentNamingInfo
-from sqlmesh.core.plan.common import should_force_rebuild
+from sqlmesh.core.plan.common import should_force_rebuild, is_breaking_kind_change
 from sqlmesh.core.plan.definition import (
     Plan,
     SnapshotMapping,
@@ -597,7 +597,7 @@ class PlanBuilder:
             forward_only = self._forward_only or self._is_forward_only_change(s_id)
             if forward_only and s_id.name in self._context_diff.modified_snapshots:
                 new, old = self._context_diff.modified_snapshots[s_id.name]
-                if should_force_rebuild(old, new) or snapshot.is_seed:
+                if is_breaking_kind_change(old, new) or snapshot.is_seed:
                     # Breaking kind changes and seed changes can't be forward-only.
                     forward_only = False
 
@@ -622,7 +622,7 @@ class PlanBuilder:
         if self._context_diff.directly_modified(s_id.name):
             if self._auto_categorization_enabled:
                 new, old = self._context_diff.modified_snapshots[s_id.name]
-                if should_force_rebuild(old, new):
+                if is_breaking_kind_change(old, new):
                     snapshot.categorize_as(SnapshotChangeCategory.BREAKING, False)
                     return
 
@@ -774,7 +774,7 @@ class PlanBuilder:
         if snapshot.name in self._context_diff.modified_snapshots:
             _, old = self._context_diff.modified_snapshots[snapshot.name]
             # If the model kind has changed in a breaking way, then we can't consider this to be a forward-only change.
-            if snapshot.is_model and should_force_rebuild(old, snapshot):
+            if snapshot.is_model and is_breaking_kind_change(old, snapshot):
                 return False
         return (
             snapshot.is_model and snapshot.model.forward_only and bool(snapshot.previous_versions)

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/plan/common.py

@@ -4,6 +4,13 @@ from sqlmesh.core.snapshot import Snapshot
 
 
 def should_force_rebuild(old: Snapshot, new: Snapshot) -> bool:
+    if new.is_view and new.is_indirect_non_breaking and not new.is_forward_only:
+        # View models always need to be rebuilt to reflect updated upstream dependencies.
+        return True
+    return is_breaking_kind_change(old, new)
+
+
+def is_breaking_kind_change(old: Snapshot, new: Snapshot) -> bool:
     if old.virtual_environment_mode != new.virtual_environment_mode:
         # If the virtual environment mode has changed, then we need to rebuild
         return True

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/plan/evaluator.py

@@ -205,6 +205,16 @@ class BuiltInPlanEvaluator(PlanEvaluator):
                     success=completion_status is not None and completion_status.is_success
                 )
 
+    def visit_physical_layer_schema_creation_stage(
+        self, stage: stages.PhysicalLayerSchemaCreationStage, plan: EvaluatablePlan
+    ) -> None:
+        try:
+            self.snapshot_evaluator.create_physical_schemas(
+                stage.snapshots, stage.deployability_index
+            )
+        except Exception as ex:
+            raise PlanError("Plan application failed.") from ex
+
     def visit_backfill_stage(self, stage: stages.BackfillStage, plan: EvaluatablePlan) -> None:
         if plan.empty_backfill:
             intervals_to_add = []
@@ -243,6 +253,8 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             circuit_breaker=self._circuit_breaker,
             start=plan.start,
             end=plan.end,
+            allow_destructive_snapshots=plan.allow_destructive_models,
+            selected_snapshot_ids=stage.selected_snapshot_ids,
         )
         if errors:
             raise PlanError("Plan application failed.")

{sqlmesh-0.209.1.dev63 → sqlmesh-0.209.1.dev65}/sqlmesh/core/plan/stages.py

@@ -1,6 +1,7 @@
 import typing as t
 
 from dataclasses import dataclass
+from sqlmesh.core import constants as c
 from sqlmesh.core.environment import EnvironmentStatements, EnvironmentNamingInfo, Environment
 from sqlmesh.core.plan.common import should_force_rebuild
 from sqlmesh.core.plan.definition import EvaluatablePlan
@@ -71,6 +72,19 @@ class PhysicalLayerUpdateStage:
     deployability_index: DeployabilityIndex
 
 
+@dataclass
+class PhysicalLayerSchemaCreationStage:
+    """Create the physical schemas for the given snapshots.
+
+    Args:
+        snapshots: Snapshots to create physical schemas for.
+        deployability_index: Deployability index for this stage.
+    """
+
+    snapshots: t.List[Snapshot]
+    deployability_index: DeployabilityIndex
+
+
 @dataclass
 class AuditOnlyRunStage:
     """Run audits only for given snapshots.
@@ -102,12 +116,14 @@ class BackfillStage:
     Args:
         snapshot_to_intervals: Intervals to backfill. This collection can be empty in which case no backfill is needed.
             This can be useful to report the lack of backfills back to the user.
+        selected_snapshot_ids: The snapshots to include in the run DAG.
         all_snapshots: All snapshots in the plan by name.
         deployability_index: Deployability index for this stage.
         before_promote: Whether this stage is before the promotion stage.
     """
 
     snapshot_to_intervals: SnapshotToIntervals
+    selected_snapshot_ids: t.Set[SnapshotId]
     all_snapshots: t.Dict[str, Snapshot]
     deployability_index: DeployabilityIndex
     before_promote: bool = True
@@ -185,6 +201,7 @@ PlanStage = t.Union[
     AfterAllStage,
     CreateSnapshotRecordsStage,
     PhysicalLayerUpdateStage,
+    PhysicalLayerSchemaCreationStage,
     AuditOnlyRunStage,
     RestatementStage,
     BackfillStage,
@@ -236,7 +253,6 @@ class PlanStagesBuilder:
         self._adjust_intervals(snapshots_by_name, plan, existing_environment)
 
         deployability_index = DeployabilityIndex.create(snapshots, start=plan.start)
-        deployability_index_for_creation = deployability_index
         if plan.is_dev:
             before_promote_snapshots = all_selected_for_backfill_snapshots
             after_promote_snapshots = set()
@@ -283,11 +299,23 @@ class PlanStagesBuilder:
         if plan.new_snapshots:
             stages.append(CreateSnapshotRecordsStage(snapshots=plan.new_snapshots))
 
-        stages.append(
-            self._get_physical_layer_update_stage(
-                plan, snapshots, snapshots_to_intervals, deployability_index_for_creation
+        snapshots_to_create = self._get_snapshots_to_create(plan, snapshots)
+        if snapshots_to_create:
+            stages.append(
+                PhysicalLayerSchemaCreationStage(
+                    snapshots=snapshots_to_create, deployability_index=deployability_index
+                )
+            )
+        if not needs_backfill:
+            stages.append(
+                self._get_physical_layer_update_stage(
+                    plan,
+                    snapshots_to_create,
+                    snapshots,
+                    snapshots_to_intervals,
+                    deployability_index,
+                )
             )
-        )
 
         audit_only_snapshots = self._get_audit_only_snapshots(new_snapshots)
         if audit_only_snapshots:
@@ -301,6 +329,11 @@ class PlanStagesBuilder:
             stages.append(
                 BackfillStage(
                     snapshot_to_intervals=missing_intervals_before_promote,
+                    selected_snapshot_ids={
+                        s_id
+                        for s_id in before_promote_snapshots
+                        if plan.is_selected_for_backfill(s_id.name)
+                    },
                     all_snapshots=snapshots_by_name,
                     deployability_index=deployability_index,
                 )
@@ -310,6 +343,7 @@ class PlanStagesBuilder:
             stages.append(
                 BackfillStage(
                     snapshot_to_intervals={},
+                    selected_snapshot_ids=set(),
                     all_snapshots=snapshots_by_name,
                     deployability_index=deployability_index,
                 )
@@ -326,7 +360,7 @@ class PlanStagesBuilder:
                 MigrateSchemasStage(
                     snapshots=snapshots_with_schema_migration,
                     all_snapshots=snapshots,
-                    deployability_index=deployability_index_for_creation,
+                    deployability_index=deployability_index,
                 )
             )
 
@@ -340,6 +374,11 @@ class PlanStagesBuilder:
             stages.append(
                 BackfillStage(
                     snapshot_to_intervals=missing_intervals_after_promote,
+                    selected_snapshot_ids={
+                        s_id
+                        for s_id in after_promote_snapshots
+                        if plan.is_selected_for_backfill(s_id.name)
+                    },
                     all_snapshots=snapshots_by_name,
                     deployability_index=deployability_index,
                 )
@@ -418,13 +457,14 @@ class PlanStagesBuilder:
     def _get_physical_layer_update_stage(
         self,
         plan: EvaluatablePlan,
-        snapshots: t.Dict[SnapshotId, Snapshot],
+        snapshots_to_create: t.List[Snapshot],
+        all_snapshots: t.Dict[SnapshotId, Snapshot],
         snapshots_to_intervals: SnapshotToIntervals,
         deployability_index: DeployabilityIndex,
     ) -> PhysicalLayerUpdateStage:
         return PhysicalLayerUpdateStage(
-            snapshots=self._get_snapshots_to_create(plan, snapshots),
-            all_snapshots=snapshots,
+            snapshots=snapshots_to_create,
+            all_snapshots=all_snapshots,
             snapshots_with_missing_intervals={
                 s.snapshot_id
                 for s in snapshots_to_intervals
@@ -589,6 +629,9 @@ class PlanStagesBuilder:
         # Make sure the intervals are up to date and restatements are reflected
         self.state_reader.refresh_snapshot_intervals(snapshots_by_name.values())
 
+        if not existing_environment:
+            existing_environment = self.state_reader.get_environment(c.PROD)
+
         if existing_environment:
             new_snapshot_ids = set()
             new_snapshot_versions = set()