sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev20__py3-none-any.whl

sqlmesh/core/model/kind.py

@@ -23,7 +23,7 @@ from sqlmesh.utils.pydantic import (
     PydanticModel,
     SQLGlotBool,
     SQLGlotColumn,
-    SQLGlotListOfColumnsOrStar,
+    SQLGlotListOfFieldsOrStar,
     SQLGlotListOfFields,
     SQLGlotPositiveInt,
     SQLGlotString,
@@ -119,6 +119,10 @@ class ModelKindMixin:
     def is_managed(self) -> bool:
         return self.model_kind_name == ModelKindName.MANAGED
 
+    @property
+    def is_dbt_custom(self) -> bool:
+        return self.model_kind_name == ModelKindName.DBT_CUSTOM
+
     @property
     def is_symbolic(self) -> bool:
         """A symbolic model is one that doesn't execute at all."""
@@ -150,6 +154,11 @@ class ModelKindMixin:
     def supports_python_models(self) -> bool:
         return True
 
+    @property
+    def supports_grants(self) -> bool:
+        """Whether this model kind supports grants configuration."""
+        return self.is_materialized or self.is_view
+
 
 class ModelKindName(str, ModelKindMixin, Enum):
     """The kind of model, determining how this data is computed and stored in the warehouse."""
@@ -170,6 +179,7 @@ class ModelKindName(str, ModelKindMixin, Enum):
     EXTERNAL = "EXTERNAL"
     CUSTOM = "CUSTOM"
     MANAGED = "MANAGED"
+    DBT_CUSTOM = "DBT_CUSTOM"
 
     @property
     def model_kind_name(self) -> t.Optional[ModelKindName]:
@@ -842,7 +852,7 @@ class SCDType2ByTimeKind(_SCDType2Kind):
 
 class SCDType2ByColumnKind(_SCDType2Kind):
     name: t.Literal[ModelKindName.SCD_TYPE_2_BY_COLUMN] = ModelKindName.SCD_TYPE_2_BY_COLUMN
-    columns: SQLGlotListOfColumnsOrStar
+    columns: SQLGlotListOfFieldsOrStar
     execution_time_as_valid_from: SQLGlotBool = False
     updated_at_name: t.Optional[SQLGlotColumn] = None
 
@@ -887,6 +897,46 @@ class ManagedKind(_ModelKind):
         return False
 
 
+class DbtCustomKind(_ModelKind):
+    name: t.Literal[ModelKindName.DBT_CUSTOM] = ModelKindName.DBT_CUSTOM
+    materialization: str
+    adapter: str = "default"
+    definition: str
+    dialect: t.Optional[str] = Field(None, validate_default=True)
+
+    _dialect_validator = kind_dialect_validator
+
+    @field_validator("materialization", "adapter", "definition", mode="before")
+    @classmethod
+    def _validate_fields(cls, v: t.Any) -> str:
+        return validate_string(v)
+
+    @property
+    def data_hash_values(self) -> t.List[t.Optional[str]]:
+        return [
+            *super().data_hash_values,
+            self.materialization,
+            self.definition,
+            self.adapter,
+            self.dialect,
+        ]
+
+    def to_expression(
+        self, expressions: t.Optional[t.List[exp.Expression]] = None, **kwargs: t.Any
+    ) -> d.ModelKind:
+        return super().to_expression(
+            expressions=[
+                *(expressions or []),
+                *_properties(
+                    {
+                        "materialization": exp.Literal.string(self.materialization),
+                        "adapter": exp.Literal.string(self.adapter),
+                    }
+                ),
+            ],
+        )
+
+
 class EmbeddedKind(_ModelKind):
     name: t.Literal[ModelKindName.EMBEDDED] = ModelKindName.EMBEDDED
 
@@ -992,6 +1042,7 @@ ModelKind = t.Annotated[
         SCDType2ByColumnKind,
         CustomKind,
         ManagedKind,
+        DbtCustomKind,
     ],
     Field(discriminator="name"),
 ]
@@ -1011,6 +1062,7 @@ MODEL_KIND_NAME_TO_TYPE: t.Dict[str, t.Type[ModelKind]] = {
     ModelKindName.SCD_TYPE_2_BY_COLUMN: SCDType2ByColumnKind,
     ModelKindName.CUSTOM: CustomKind,
     ModelKindName.MANAGED: ManagedKind,
+    ModelKindName.DBT_CUSTOM: DbtCustomKind,
 }
 
 
@@ -1053,6 +1105,18 @@ def create_model_kind(v: t.Any, dialect: str, defaults: t.Dict[str, t.Any]) -> M
                 ):
                     props[on_change_property] = defaults.get(on_change_property)
 
+        # only pass the batch_concurrency user default to models inheriting from _IncrementalBy
+        # that don't explicitly set it in the model definition, but ignore subclasses of _IncrementalBy
+        # that hardcode a specific batch_concurrency
+        if issubclass(kind_type, _IncrementalBy):
+            BATCH_CONCURRENCY: t.Final = "batch_concurrency"
+            if (
+                props.get(BATCH_CONCURRENCY) is None
+                and defaults.get(BATCH_CONCURRENCY) is not None
+                and kind_type.all_field_infos()[BATCH_CONCURRENCY].default is None
+            ):
+                props[BATCH_CONCURRENCY] = defaults.get(BATCH_CONCURRENCY)
+
         if kind_type == CustomKind:
             # load the custom materialization class and check if it uses a custom kind type
             from sqlmesh.core.snapshot.evaluator import get_custom_materialization_type

sqlmesh/core/model/meta.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import typing as t
+from enum import Enum
 from functools import cached_property
 from typing_extensions import Self
 
@@ -13,6 +14,7 @@ from sqlmesh.core import dialect as d
 from sqlmesh.core.config.common import VirtualEnvironmentMode
 from sqlmesh.core.config.linter import LinterConfig
 from sqlmesh.core.dialect import normalize_model_name
+from sqlmesh.utils import classproperty
 from sqlmesh.core.model.common import (
     bool_validator,
     default_catalog_validator,
@@ -46,10 +48,41 @@ from sqlmesh.utils.pydantic import (
 
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import CustomMaterializationProperties, SessionProperties
+    from sqlmesh.core.engine_adapter._typing import GrantsConfig
 
 FunctionCall = t.Tuple[str, t.Dict[str, exp.Expression]]
 
 
+class GrantsTargetLayer(str, Enum):
+    """Target layer(s) where grants should be applied."""
+
+    ALL = "all"
+    PHYSICAL = "physical"
+    VIRTUAL = "virtual"
+
+    @classproperty
+    def default(cls) -> "GrantsTargetLayer":
+        return GrantsTargetLayer.VIRTUAL
+
+    @property
+    def is_all(self) -> bool:
+        return self == GrantsTargetLayer.ALL
+
+    @property
+    def is_physical(self) -> bool:
+        return self == GrantsTargetLayer.PHYSICAL
+
+    @property
+    def is_virtual(self) -> bool:
+        return self == GrantsTargetLayer.VIRTUAL
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
 class ModelMeta(_Node):
     """Metadata for models which can be defined in SQL."""
 
@@ -85,6 +118,8 @@ class ModelMeta(_Node):
     )
     formatting: t.Optional[bool] = Field(default=None, exclude=True)
     virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default
+    grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants")
+    grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default
 
     _bool_validator = bool_validator
     _model_kind_validator = model_kind_validator
@@ -247,11 +282,15 @@ class ModelMeta(_Node):
 
         columns_to_types = info.data.get("columns_to_types_")
         if columns_to_types:
-            for column_name in col_descriptions:
+            from sqlmesh.core.console import get_console
+
+            console = get_console()
+            for column_name in list(col_descriptions):
                 if column_name not in columns_to_types:
-                    raise ConfigError(
+                    console.log_warning(
                         f"In model '{info.data['name']}', a description is provided for column '{column_name}' but it is not a column in the model."
                     )
+                    del col_descriptions[column_name]
 
         return col_descriptions
 
@@ -283,6 +322,14 @@ class ModelMeta(_Node):
     def ignored_rules_validator(cls, vs: t.Any) -> t.Any:
         return LinterConfig._validate_rules(vs)
 
+    @field_validator("grants_target_layer", mode="before")
+    def _grants_target_layer_validator(cls, v: t.Any) -> t.Any:
+        if isinstance(v, exp.Identifier):
+            return v.this
+        if isinstance(v, exp.Literal) and v.is_string:
+            return v.this
+        return v
+
     @field_validator("session_properties_", mode="before")
     def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
         # use the generic properties validator to parse the session properties
@@ -390,6 +437,10 @@ class ModelMeta(_Node):
                 f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead."
             )
 
+        # Validate grants configuration for model kind support
+        if self.grants is not None and not kind.supports_grants:
+            raise ValueError(f"grants cannot be set for {kind.name} models")
+
         return self
 
     @property
@@ -461,6 +512,30 @@ class ModelMeta(_Node):
             return self.kind.materialization_properties
         return {}
 
+    @cached_property
+    def grants(self) -> t.Optional[GrantsConfig]:
+        """A dictionary of grants mapping permission names to lists of grantees."""
+
+        if self.grants_ is None:
+            return None
+
+        if not self.grants_.expressions:
+            return {}
+
+        grants_dict = {}
+        for eq_expr in self.grants_.expressions:
+            try:
+                permission_name = self._validate_config_expression(eq_expr.left)
+                grantee_list = self._validate_nested_config_values(eq_expr.expression)
+                grants_dict[permission_name] = grantee_list
+            except ConfigError as e:
+                permission_name = (
+                    eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left)
+                )
+                raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}")
+
+        return grants_dict if grants_dict else None
+
     @property
     def all_references(self) -> t.List[Reference]:
         """All references including grains."""
@@ -525,3 +600,33 @@ class ModelMeta(_Node):
     @property
     def ignored_rules(self) -> t.Set[str]:
         return self.ignored_rules_ or set()
+
+    def _validate_config_expression(self, expr: exp.Expression) -> str:
+        if isinstance(expr, (d.MacroFunc, d.MacroVar)):
+            raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}")
+
+        if isinstance(expr, exp.Null):
+            raise ConfigError("NULL value")
+
+        if isinstance(expr, exp.Literal):
+            return str(expr.this).strip()
+        if isinstance(expr, (exp.Column, exp.Identifier)):
+            return expr.name
+        return expr.sql(dialect=self.dialect).strip()
+
+    def _validate_nested_config_values(self, value_expr: exp.Expression) -> t.List[str]:
+        result = []
+
+        def flatten_expr(expr: exp.Expression) -> None:
+            if isinstance(expr, exp.Array):
+                for elem in expr.expressions:
+                    flatten_expr(elem)
+            elif isinstance(expr, (exp.Tuple, exp.Paren)):
+                expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions
+                for elem in expressions:
+                    flatten_expr(elem)
+            else:
+                result.append(self._validate_config_expression(expr))
+
+        flatten_expr(value_expr)
+        return result

sqlmesh/core/node.py

@@ -153,6 +153,101 @@ class IntervalUnit(str, Enum):
         return self.seconds * 1000
 
 
+class DbtNodeInfo(PydanticModel):
+    """
+    Represents dbt-specific model information set by the dbt loader and intended to be made available at the Snapshot level
+    (as opposed to hidden within the individual model jinja macro registries).
+
+    This allows for things like injecting implementations of variables / functions into the Jinja context that are compatible with
+    their dbt equivalents but are backed by the sqlmesh snapshots in any given plan / environment
+    """
+
+    unique_id: str
+    """This is the node/resource name/unique_id that's used as the node key in the dbt manifest.
+    It's prefixed by the resource type and is exposed in context variables like {{ selected_resources }}.
+
+    Examples:
+        - test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a
+        - seed.jaffle_shop.raw_payments
+        - model.jaffle_shop.stg_orders
+    """
+
+    name: str
+    """Name of this object in the dbt global namespace, used by things like {{ ref() }} calls.    
+    
+    Examples:
+        - unique_stg_orders_order_id
+        - raw_payments
+        - stg_orders
+    """
+
+    fqn: str
+    """Used for selectors in --select/--exclude.
+    Takes the filesystem into account so may be structured differently to :unique_id.
+    
+    Examples:
+        - jaffle_shop.staging.unique_stg_orders_order_id
+        - jaffle_shop.raw_payments
+        - jaffle_shop.staging.stg_orders
+    """
+
+    alias: t.Optional[str] = None
+    """This is dbt's way of overriding the _physical table_ a model is written to.
+
+    It's used in the following situation:
+     - Say you have two models, "stg_customers" and "customers"
+     - You want "stg_customers" to be written to the "staging" schema as eg "staging.customers" - NOT "staging.stg_customers"
+     - But you cant rename the file to "customers" because it will conflict with your other model file "customers"
+     - Even if you put it in a different folder, eg "staging/customers.sql" - dbt still has a global namespace so it will conflict
+        when you try to do something like "{{ ref('customers') }}"
+     - So dbt's solution to this problem is to keep calling it "stg_customers" at the dbt project/model level,
+        but allow overriding the physical table to "customers" via something like "{{ config(alias='customers', schema='staging') }}"
+
+    Note that if :alias is set, it does *not* replace :name at the model level and cannot be used interchangably with :name.
+    It also does not affect the :fqn or :unique_id. It's just used to override :name when it comes time to generate the physical table name.
+    """
+
+    @model_validator(mode="after")
+    def post_init(self) -> Self:
+        # by default, dbt sets alias to the same as :name
+        # however, we only want to include :alias if it is actually different / actually providing an override
+        if self.alias == self.name:
+            self.alias = None
+        return self
+
+    def to_expression(self) -> exp.Expression:
+        """Produce a SQLGlot expression representing this object, for use in things like the model/audit definition renderers"""
+        return exp.tuple_(
+            *(
+                exp.PropertyEQ(this=exp.var(k), expression=exp.Literal.string(v))
+                for k, v in sorted(self.model_dump(exclude_none=True).items())
+            )
+        )
+
+
+class DbtInfoMixin:
+    """This mixin encapsulates properties that only exist for dbt compatibility and are otherwise not required
+    for native projects"""
+
+    @property
+    def dbt_node_info(self) -> t.Optional[DbtNodeInfo]:
+        raise NotImplementedError()
+
+    @property
+    def dbt_unique_id(self) -> t.Optional[str]:
+        """Used for compatibility with jinja context variables such as {{ selected_resources }}"""
+        if self.dbt_node_info:
+            return self.dbt_node_info.unique_id
+        return None
+
+    @property
+    def dbt_fqn(self) -> t.Optional[str]:
+        """Used in the selector engine for compatibility with selectors that select models by dbt fqn"""
+        if self.dbt_node_info:
+            return self.dbt_node_info.fqn
+        return None
+
+
 # this must be sorted in descending order
 INTERVAL_SECONDS = {
     IntervalUnit.YEAR: 60 * 60 * 24 * 365,
@@ -165,7 +260,7 @@ INTERVAL_SECONDS = {
 }
 
 
-class _Node(PydanticModel):
+class _Node(DbtInfoMixin, PydanticModel):
     """
     Node is the core abstraction for entity that can be executed within the scheduler.
 
@@ -199,7 +294,7 @@ class _Node(PydanticModel):
     interval_unit_: t.Optional[IntervalUnit] = Field(alias="interval_unit", default=None)
     tags: t.List[str] = []
     stamp: t.Optional[str] = None
-    dbt_name: t.Optional[str] = None  # dbt node name
+    dbt_node_info_: t.Optional[DbtNodeInfo] = Field(alias="dbt_node_info", default=None)
     _path: t.Optional[Path] = None
     _data_hash: t.Optional[str] = None
     _metadata_hash: t.Optional[str] = None
@@ -446,6 +541,10 @@ class _Node(PydanticModel):
         """Return True if this is an audit node"""
         return False
 
+    @property
+    def dbt_node_info(self) -> t.Optional[DbtNodeInfo]:
+        return self.dbt_node_info_
+
 
 class NodeType(str, Enum):
     MODEL = "model"

sqlmesh/core/plan/builder.py

@@ -65,6 +65,9 @@ class PlanBuilder:
         restate_models: A list of models for which the data should be restated for the time range
             specified in this plan. Note: models defined outside SQLMesh (external) won't be a part
             of the restatement.
+        restate_all_snapshots: If restatements are present, this flag indicates whether or not the intervals
+            being restated should be cleared from state for other versions of this model (typically, versions that are present in other environments).
+            If set to None, the default behaviour is to not clear anything unless the target environment is prod.
         backfill_models: A list of fully qualified model names for which the data should be backfilled as part of this plan.
         no_gaps:  Whether to ensure that new snapshots for nodes that are already a
             part of the target environment have no data gaps when compared against previous
@@ -103,6 +106,7 @@ class PlanBuilder:
         execution_time: t.Optional[TimeLike] = None,
         apply: t.Optional[t.Callable[[Plan], None]] = None,
         restate_models: t.Optional[t.Iterable[str]] = None,
+        restate_all_snapshots: bool = False,
         backfill_models: t.Optional[t.Iterable[str]] = None,
         no_gaps: bool = False,
         skip_backfill: bool = False,
@@ -154,6 +158,7 @@ class PlanBuilder:
         self._auto_categorization_enabled = auto_categorization_enabled
         self._include_unmodified = include_unmodified
         self._restate_models = set(restate_models) if restate_models is not None else None
+        self._restate_all_snapshots = restate_all_snapshots
         self._effective_from = effective_from
 
         # note: this deliberately doesnt default to now() here.
@@ -277,7 +282,6 @@ class PlanBuilder:
         if self._latest_plan:
             return self._latest_plan
 
-        self._ensure_no_new_snapshots_with_restatements()
         self._ensure_new_env_with_changes()
         self._ensure_valid_date_range()
         self._ensure_no_broken_references()
@@ -338,7 +342,9 @@ class PlanBuilder:
             directly_modified=directly_modified,
             indirectly_modified=indirectly_modified,
             deployability_index=deployability_index,
+            selected_models_to_restate=self._restate_models,
             restatements=restatements,
+            restate_all_snapshots=self._restate_all_snapshots,
             start_override_per_model=self._start_override_per_model,
             end_override_per_model=end_override_per_model,
             selected_models_to_backfill=self._backfill_models,
@@ -674,6 +680,14 @@ class PlanBuilder:
                     if mode == AutoCategorizationMode.FULL:
                         snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only)
         elif self._context_diff.indirectly_modified(snapshot.name):
+            if snapshot.is_materialized_view and not forward_only:
+                # We categorize changes as breaking to allow for instantaneous switches in a virtual layer.
+                # Otherwise, there might be a potentially long downtime during MVs recreation.
+                # In the case of forward-only changes this optimization is not applicable because we want to continue
+                # using the same (existing) table version.
+                snapshot.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING, forward_only)
+                return
+
             all_upstream_forward_only = set()
             all_upstream_categories = set()
             direct_parent_categories = set()
@@ -858,15 +872,6 @@ class PlanBuilder:
                     f"""Removed {broken_references_msg} are referenced in '{snapshot.name}'. Please remove broken references before proceeding."""
                 )
 
-    def _ensure_no_new_snapshots_with_restatements(self) -> None:
-        if self._restate_models is not None and (
-            self._context_diff.new_snapshots or self._context_diff.modified_snapshots
-        ):
-            raise PlanError(
-                "Model changes and restatements can't be a part of the same plan. "
-                "Revert or apply changes before proceeding with restatements."
-            )
-
     def _ensure_new_env_with_changes(self) -> None:
         if (
             self._is_dev