sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/dbt/model.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import datetime
 import typing as t
+import logging
 
 from sqlglot import exp
 from sqlglot.errors import SqlglotError
@@ -24,18 +25,33 @@ from sqlmesh.core.model import (
     ManagedKind,
     create_sql_model,
 )
-from sqlmesh.core.model.kind import SCDType2ByTimeKind, OnDestructiveChange, OnAdditiveChange
+from sqlmesh.core.model.kind import (
+    SCDType2ByTimeKind,
+    OnDestructiveChange,
+    OnAdditiveChange,
+    on_destructive_change_validator,
+    on_additive_change_validator,
+    DbtCustomKind,
+)
 from sqlmesh.dbt.basemodel import BaseModelConfig, Materialization, SnapshotStrategy
-from sqlmesh.dbt.common import SqlStr, extract_jinja_config, sql_str_validator
+from sqlmesh.dbt.common import SqlStr, sql_str_validator
 from sqlmesh.utils.errors import ConfigError
 from sqlmesh.utils.pydantic import field_validator
 
 if t.TYPE_CHECKING:
     from sqlmesh.core.audit.definition import ModelAudit
     from sqlmesh.dbt.context import DbtContext
+    from sqlmesh.dbt.package import MaterializationConfig
+
+logger = logging.getLogger(__name__)
+
+
+logger = logging.getLogger(__name__)
 
 
-INCREMENTAL_BY_TIME_STRATEGIES = set(["delete+insert", "insert_overwrite", "microbatch"])
+INCREMENTAL_BY_TIME_RANGE_STRATEGIES = set(
+    ["delete+insert", "insert_overwrite", "microbatch", "incremental_by_time_range"]
+)
 INCREMENTAL_BY_UNIQUE_KEY_STRATEGIES = set(["merge"])
 
 
@@ -71,16 +87,19 @@ class ModelConfig(BaseModelConfig):
 
     # sqlmesh fields
     sql: SqlStr = SqlStr("")
-    time_column: t.Optional[str] = None
+    time_column: t.Optional[t.Union[str, t.Dict[str, str]]] = None
     cron: t.Optional[str] = None
     interval_unit: t.Optional[str] = None
     batch_concurrency: t.Optional[int] = None
     forward_only: bool = True
     disable_restatement: t.Optional[bool] = None
-    allow_partials: t.Optional[bool] = None
+    allow_partials: bool = True
     physical_version: t.Optional[str] = None
     auto_restatement_cron: t.Optional[str] = None
     auto_restatement_intervals: t.Optional[int] = None
+    partition_by_time_column: t.Optional[bool] = None
+    on_destructive_change: t.Optional[OnDestructiveChange] = None
+    on_additive_change: t.Optional[OnAdditiveChange] = None
 
     # DBT configuration fields
     cluster_by: t.Optional[t.List[str]] = None
@@ -132,11 +151,9 @@ class ModelConfig(BaseModelConfig):
     inserts_only: t.Optional[bool] = None
     incremental_predicates: t.Optional[t.List[str]] = None
 
-    # Private fields
-    _sql_embedded_config: t.Optional[SqlStr] = None
-    _sql_no_config: t.Optional[SqlStr] = None
-
     _sql_validator = sql_str_validator
+    _on_destructive_change_validator = on_destructive_change_validator
+    _on_additive_change_validator = on_additive_change_validator
 
     @field_validator(
         "unique_key",
@@ -155,6 +172,22 @@ class ModelConfig(BaseModelConfig):
             return "*"
         return ensure_list(v)
 
+    @field_validator("updated_at", mode="before")
+    @classmethod
+    def _validate_updated_at(cls, v: t.Optional[str]) -> t.Optional[str]:
+        """
+        Extract column name if updated_at contains a cast.
+
+        SCDType2ByTimeKind and SCDType2ByColumnKind expect a column, and the casting is done later.
+        """
+        if v is None:
+            return None
+        parsed = d.parse_one(v)
+        if isinstance(parsed, exp.Cast) and isinstance(parsed.this, exp.Column):
+            return parsed.this.name
+
+        return v
+
     @field_validator("sql", mode="before")
     @classmethod
     def _validate_sql(cls, v: t.Union[str, SqlStr]) -> SqlStr:
@@ -182,6 +215,14 @@ class ModelConfig(BaseModelConfig):
             ):
                 granularity = v["granularity"]
                 raise ConfigError(f"Unexpected granularity '{granularity}' in partition_by '{v}'.")
+            if "data_type" in v and v["data_type"].lower() not in (
+                "timestamp",
+                "date",
+                "datetime",
+                "int64",
+            ):
+                data_type = v["data_type"]
+                raise ConfigError(f"Unexpected data_type '{data_type}' in partition_by '{v}'.")
             return {"data_type": "date", "granularity": "day", **v}
         raise ConfigError(f"Invalid format for partition_by '{v}'")
 
@@ -228,17 +269,6 @@ class ModelConfig(BaseModelConfig):
     def table_schema(self) -> str:
         return self.target_schema or super().table_schema
 
-    def _get_overlapping_field_value(
-        self, context: DbtContext, dbt_field_name: str, sqlmesh_field_name: str
-    ) -> t.Optional[t.Any]:
-        dbt_field = self._get_field_value(dbt_field_name)
-        sqlmesh_field = getattr(self, sqlmesh_field_name, None)
-        if dbt_field is not None and sqlmesh_field is not None:
-            get_console().log_warning(
-                f"Both '{dbt_field_name}' and '{sqlmesh_field_name}' are set for model '{self.canonical_name(context)}'. '{sqlmesh_field_name}' will be used."
-            )
-        return sqlmesh_field if sqlmesh_field is not None else dbt_field
-
     def model_kind(self, context: DbtContext) -> ModelKind:
         """
         Get the sqlmesh ModelKind
@@ -273,8 +303,12 @@ class ModelConfig(BaseModelConfig):
                 "Valid values are 'ignore', 'fail', 'append_new_columns', 'sync_all_columns'."
             )
 
-        incremental_kind_kwargs["on_destructive_change"] = on_destructive_change
-        incremental_kind_kwargs["on_additive_change"] = on_additive_change
+        incremental_kind_kwargs["on_destructive_change"] = (
+            self._get_field_value("on_destructive_change") or on_destructive_change
+        )
+        incremental_kind_kwargs["on_additive_change"] = (
+            self._get_field_value("on_additive_change") or on_additive_change
+        )
         auto_restatement_cron_value = self._get_field_value("auto_restatement_cron")
         if auto_restatement_cron_value is not None:
             incremental_kind_kwargs["auto_restatement_cron"] = auto_restatement_cron_value
@@ -290,7 +324,8 @@ class ModelConfig(BaseModelConfig):
                 incremental_kind_kwargs["forward_only"] = forward_only_value
 
             is_incremental_by_time_range = self.time_column or (
-                self.incremental_strategy and self.incremental_strategy == "microbatch"
+                self.incremental_strategy
+                and self.incremental_strategy in {"microbatch", "incremental_by_time_range"}
             )
             # Get shared incremental by kwargs
             for field in ("batch_size", "batch_concurrency", "lookback"):
@@ -311,35 +346,35 @@ class ModelConfig(BaseModelConfig):
                     )
             incremental_by_kind_kwargs["disable_restatement"] = disable_restatement
 
-            # Incremental by time range which includes microbatch
             if is_incremental_by_time_range:
                 strategy = self.incremental_strategy or target.default_incremental_strategy(
                     IncrementalByTimeRangeKind
                 )
 
-                if strategy not in INCREMENTAL_BY_TIME_STRATEGIES:
+                if strategy not in INCREMENTAL_BY_TIME_RANGE_STRATEGIES:
                     get_console().log_warning(
                         f"SQLMesh incremental by time strategy is not compatible with '{strategy}' incremental strategy in model '{self.canonical_name(context)}'. "
-                        f"Supported strategies include {collection_to_str(INCREMENTAL_BY_TIME_STRATEGIES)}."
+                        f"Supported strategies include {collection_to_str(INCREMENTAL_BY_TIME_RANGE_STRATEGIES)}."
                     )
 
-                if strategy == "microbatch":
-                    time_column = self._get_overlapping_field_value(
-                        context, "event_time", "time_column"
+                if self.time_column and strategy != "incremental_by_time_range":
+                    get_console().log_warning(
+                        f"Using `time_column` on a model with incremental_strategy '{strategy}' has been deprecated. "
+                        f"Please use `incremental_by_time_range` instead in model '{self.canonical_name(context)}'."
                     )
+
+                if strategy == "microbatch":
+                    if self.time_column:
+                        raise ConfigError(
+                            f"{self.canonical_name(context)}: 'time_column' cannot be used with 'microbatch' incremental strategy. Use 'event_time' instead."
+                        )
+                    time_column = self._get_field_value("event_time")
                     if not time_column:
                         raise ConfigError(
                             f"{self.canonical_name(context)}: 'event_time' is required for microbatch incremental strategy."
                         )
-                    concurrent_batches = self._get_field_value("concurrent_batches")
-                    if concurrent_batches is True:
-                        if incremental_by_kind_kwargs.get("batch_size"):
-                            get_console().log_warning(
-                                f"'concurrent_batches' is set to True and 'batch_size' are defined in '{self.canonical_name(context)}'. The batch size will be set to the value of `batch_size`."
-                            )
-                        incremental_by_kind_kwargs["batch_size"] = incremental_by_kind_kwargs.get(
-                            "batch_size", 1
-                        )
+                    # dbt microbatch always processes batches in a size of 1
+                    incremental_by_kind_kwargs["batch_size"] = 1
                 else:
                     if not self.time_column:
                         raise ConfigError(
@@ -347,11 +382,22 @@ class ModelConfig(BaseModelConfig):
                         )
                     time_column = self.time_column
 
+                incremental_by_time_range_kwargs = {
+                    "time_column": time_column,
+                }
+                if self.auto_restatement_intervals:
+                    incremental_by_time_range_kwargs["auto_restatement_intervals"] = (
+                        self.auto_restatement_intervals
+                    )
+                if self.partition_by_time_column is not None:
+                    incremental_by_time_range_kwargs["partition_by_time_column"] = (
+                        self.partition_by_time_column
+                    )
+
                 return IncrementalByTimeRangeKind(
-                    time_column=time_column,
-                    auto_restatement_intervals=self.auto_restatement_intervals,
                     **incremental_kind_kwargs,
                     **incremental_by_kind_kwargs,
+                    **incremental_by_time_range_kwargs,
                 )
 
             if self.unique_key:
@@ -389,7 +435,7 @@ class ModelConfig(BaseModelConfig):
                 IncrementalUnmanagedKind
             )
             return IncrementalUnmanagedKind(
-                insert_overwrite=strategy in INCREMENTAL_BY_TIME_STRATEGIES,
+                insert_overwrite=strategy in INCREMENTAL_BY_TIME_RANGE_STRATEGIES,
                 disable_restatement=incremental_by_kind_kwargs["disable_restatement"],
                 **incremental_kind_kwargs,
             )
@@ -424,26 +470,20 @@ class ModelConfig(BaseModelConfig):
         if materialization == Materialization.DYNAMIC_TABLE:
             return ManagedKind()
 
-        raise ConfigError(f"{materialization.value} materialization not supported.")
-
-    @property
-    def sql_no_config(self) -> SqlStr:
-        if self._sql_no_config is None:
-            self._sql_no_config = SqlStr("")
-            self._extract_sql_config()
-        return self._sql_no_config
+        if materialization == Materialization.CUSTOM:
+            if custom_materialization := self._get_custom_materialization(context):
+                return DbtCustomKind(
+                    materialization=self.materialized,
+                    adapter=custom_materialization.adapter,
+                    dialect=self.dialect(context),
+                    definition=custom_materialization.definition,
+                )
 
-    @property
-    def sql_embedded_config(self) -> SqlStr:
-        if self._sql_embedded_config is None:
-            self._sql_embedded_config = SqlStr("")
-            self._extract_sql_config()
-        return self._sql_embedded_config
+            raise ConfigError(
+                f"Unknown materialization '{self.materialized}'. Custom materializations must be defined in your dbt project."
+            )
 
-    def _extract_sql_config(self) -> None:
-        no_config, embedded_config = extract_jinja_config(self.sql)
-        self._sql_no_config = SqlStr(no_config)
-        self._sql_embedded_config = SqlStr(embedded_config)
+        raise ConfigError(f"{materialization.value} materialization not supported.")
 
     def _big_query_partition_by_expr(self, context: DbtContext) -> exp.Expression:
         assert isinstance(self.partition_by, dict)
@@ -482,6 +522,18 @@ class ModelConfig(BaseModelConfig):
             dialect="bigquery",
         )
 
+    def _get_custom_materialization(self, context: DbtContext) -> t.Optional[MaterializationConfig]:
+        materializations = context.manifest.materializations()
+        name, target_adapter = self.materialized, context.target.dialect
+
+        adapter_specific_key = f"{name}_{target_adapter}"
+        default_key = f"{name}_default"
+        if adapter_specific_key in materializations:
+            return materializations[adapter_specific_key]
+        if default_key in materializations:
+            return materializations[default_key]
+        return None
+
     @property
     def sqlmesh_config_fields(self) -> t.Set[str]:
         return super().sqlmesh_config_fields | {
@@ -502,35 +554,71 @@ class ModelConfig(BaseModelConfig):
     ) -> Model:
         """Converts the dbt model into a SQLMesh model."""
         model_dialect = self.dialect(context)
-        query = d.jinja_query(self.sql_no_config)
+        query = d.jinja_query(self.sql)
+        kind = self.model_kind(context)
 
         optional_kwargs: t.Dict[str, t.Any] = {}
         physical_properties: t.Dict[str, t.Any] = {}
 
         if self.partition_by:
-            partitioned_by = []
-            if isinstance(self.partition_by, list):
-                for p in self.partition_by:
+            if isinstance(kind, (ViewKind, EmbeddedKind)):
+                logger.warning(
+                    "Ignoring partition_by config for model '%s'; partition_by is not supported for %s.",
+                    self.name,
+                    "views" if isinstance(kind, ViewKind) else "ephemeral models",
+                )
+            elif context.target.dialect == "snowflake":
+                logger.warning(
+                    "Ignoring partition_by config for model '%s' targeting %s. The partition_by config is not supported for Snowflake.",
+                    self.name,
+                    context.target.dialect,
+                )
+            else:
+                partitioned_by = []
+                if isinstance(self.partition_by, list):
+                    for p in self.partition_by:
+                        try:
+                            partitioned_by.append(d.parse_one(p, dialect=model_dialect))
+                        except SqlglotError as e:
+                            raise ConfigError(
+                                f"Failed to parse model '{self.canonical_name(context)}' partition_by field '{p}' in '{self.path}': {e}"
+                            ) from e
+                elif isinstance(self.partition_by, dict):
+                    if context.target.dialect == "bigquery":
+                        partitioned_by.append(self._big_query_partition_by_expr(context))
+                    else:
+                        logger.warning(
+                            "Ignoring partition_by config for model '%s' targeting %s. The format of the config field is only supported for BigQuery.",
+                            self.name,
+                            context.target.dialect,
+                        )
+
+                if partitioned_by:
+                    optional_kwargs["partitioned_by"] = partitioned_by
+
+        if self.cluster_by:
+            if isinstance(kind, (ViewKind, EmbeddedKind)):
+                logger.warning(
+                    "Ignoring cluster_by config for model '%s'; cluster_by is not supported for %s.",
+                    self.name,
+                    "views" if isinstance(kind, ViewKind) else "ephemeral models",
+                )
+            else:
+                clustered_by = []
+                for c in self.cluster_by:
                     try:
-                        partitioned_by.append(d.parse_one(p, dialect=model_dialect))
+                        cluster_expr = exp.maybe_parse(
+                            c, into=exp.Cluster, prefix="CLUSTER BY", dialect=model_dialect
+                        )
+                        for expr in cluster_expr.expressions:
+                            clustered_by.append(
+                                expr.this if isinstance(expr, exp.Ordered) else expr
+                            )
                     except SqlglotError as e:
                         raise ConfigError(
-                            f"Failed to parse model '{self.canonical_name(context)}' partition_by field '{p}' in '{self.path}': {e}"
+                            f"Failed to parse model '{self.canonical_name(context)}' cluster_by field '{c}' in '{self.path}': {e}"
                         ) from e
-            else:
-                partitioned_by.append(self._big_query_partition_by_expr(context))
-            optional_kwargs["partitioned_by"] = partitioned_by
-
-        if self.cluster_by:
-            clustered_by = []
-            for c in self.cluster_by:
-                try:
-                    clustered_by.append(d.parse_one(c, dialect=model_dialect))
-                except SqlglotError as e:
-                    raise ConfigError(
-                        f"Failed to parse model '{self.canonical_name(context)}' cluster_by field '{c}' in '{self.path}': {e}"
-                    ) from e
-            optional_kwargs["clustered_by"] = clustered_by
+                optional_kwargs["clustered_by"] = clustered_by
 
         model_kwargs = self.sqlmesh_model_kwargs(context)
         if self.sql_header:
@@ -628,15 +716,19 @@ class ModelConfig(BaseModelConfig):
                 model_kwargs["physical_properties"] = physical_properties
 
         kind = self.model_kind(context)
+
+        # A falsy grants config (None or {}) is considered as unmanaged per dbt semantics
+        if self.grants and kind.supports_grants:
+            model_kwargs["grants"] = self.grants
+
         allow_partials = model_kwargs.pop("allow_partials", None)
-        if (
-            allow_partials is None
-            and kind.is_materialized
-            and not kind.is_incremental_by_time_range
-        ):
+        if allow_partials is None:
             # Set allow_partials to True for dbt models to preserve the original semantics.
             allow_partials = True
 
+        # pop begin for all models so we don't pass it through for non-incremental materializations
+        # (happens if model config is microbatch but project config overrides)
+        begin = model_kwargs.pop("begin", None)
         if kind.is_incremental:
             if self.batch_size and isinstance(self.batch_size, str):
                 if "interval_unit" in model_kwargs:
@@ -646,13 +738,18 @@ class ModelConfig(BaseModelConfig):
                 else:
                     model_kwargs["interval_unit"] = self.batch_size
                     self.batch_size = None
-            if begin := model_kwargs.pop("begin", None):
+            if begin:
                 if "start" in model_kwargs:
                     get_console().log_warning(
                         f"Both 'begin' and 'start' are set for model '{self.canonical_name(context)}'. 'start' will be used."
                     )
                 else:
                     model_kwargs["start"] = begin
+            # If user explicitly disables concurrent batches then we want to set depends on past to true which we
+            # will do by including the model in the depends_on
+            if self.concurrent_batches is not None and self.concurrent_batches is False:
+                depends_on = model_kwargs.get("depends_on", set())
+                depends_on.add(self.canonical_name(context))
 
         model_kwargs["start"] = model_kwargs.get(
             "start", context.sqlmesh_config.model_defaults.start
@@ -670,6 +767,7 @@ class ModelConfig(BaseModelConfig):
             extract_dependencies_from_query=False,
             allow_partials=allow_partials,
             virtual_environment_mode=virtual_environment_mode,
+            dbt_node_info=self.node_info,
             **optional_kwargs,
             **model_kwargs,
         )

sqlmesh/dbt/package.py

@@ -37,6 +37,16 @@ class HookConfig(PydanticModel):
     dependencies: Dependencies
 
 
+class MaterializationConfig(PydanticModel):
+    """Class to contain custom materialization configuration."""
+
+    name: str
+    adapter: str
+    definition: str
+    dependencies: Dependencies
+    path: Path
+
+
 class Package(PydanticModel):
     """Class to contain package configuration"""
 
@@ -47,6 +57,7 @@ class Package(PydanticModel):
     models: t.Dict[str, ModelConfig]
     variables: t.Dict[str, t.Any]
     macros: t.Dict[str, MacroConfig]
+    materializations: t.Dict[str, MaterializationConfig]
     on_run_start: t.Dict[str, HookConfig]
     on_run_end: t.Dict[str, HookConfig]
     files: t.Set[Path]
@@ -94,6 +105,7 @@ class PackageLoader:
         models = _fix_paths(self._context.manifest.models(package_name), package_root)
         seeds = _fix_paths(self._context.manifest.seeds(package_name), package_root)
         macros = _fix_paths(self._context.manifest.macros(package_name), package_root)
+        materializations = _fix_paths(self._context.manifest.materializations(), package_root)
         on_run_start = _fix_paths(self._context.manifest.on_run_start(package_name), package_root)
         on_run_end = _fix_paths(self._context.manifest.on_run_end(package_name), package_root)
         sources = self._context.manifest.sources(package_name)
@@ -114,13 +126,16 @@ class PackageLoader:
             seeds=seeds,
             variables=package_variables,
             macros=macros,
+            materializations=materializations,
             files=config_paths,
             on_run_start=on_run_start,
             on_run_end=on_run_end,
         )
 
 
-T = t.TypeVar("T", TestConfig, ModelConfig, MacroConfig, SeedConfig, HookConfig)
+T = t.TypeVar(
+    "T", TestConfig, ModelConfig, MacroConfig, MaterializationConfig, SeedConfig, HookConfig
+)
 
 
 def _fix_paths(configs: t.Dict[str, T], package_root: Path) -> t.Dict[str, T]:

sqlmesh/dbt/profile.py

@@ -60,7 +60,7 @@ class Profile:
             if not context.profile_name:
                 raise ConfigError(f"{project_file.stem} must include project name.")
 
-        profile_filepath = cls._find_profile(context.project_root)
+        profile_filepath = cls._find_profile(context.project_root, context.profiles_dir)
         if not profile_filepath:
             raise ConfigError(f"{cls.PROFILE_FILE} not found.")
 
@@ -68,8 +68,8 @@ class Profile:
         return Profile(profile_filepath, target_name, target)
 
     @classmethod
-    def _find_profile(cls, project_root: Path) -> t.Optional[Path]:
-        dir = os.environ.get("DBT_PROFILES_DIR", "")
+    def _find_profile(cls, project_root: Path, profiles_dir: t.Optional[Path]) -> t.Optional[Path]:
+        dir = os.environ.get("DBT_PROFILES_DIR", profiles_dir or "")
         path = Path(project_root, dir, cls.PROFILE_FILE)
         if path.exists():
             return path

sqlmesh/dbt/project.py

@@ -99,16 +99,21 @@ class Project:
             package = package_loader.load(path.parent)
             packages[package.name] = package
 
+        # Variable resolution precedence:
+        # 1. Variable overrides
+        # 2. Package-scoped variables in the root project's dbt_project.yml
+        # 3. Global project variables in the root project's dbt_project.yml
+        # 4. Variables in the package's dbt_project.yml
         all_project_variables = {**(project_yaml.get("vars") or {}), **(variable_overrides or {})}
         for name, package in packages.items():
-            package_vars = all_project_variables.get(name)
-
-            if isinstance(package_vars, dict):
-                package.variables.update(package_vars)
-
-            if name == context.project_name:
-                package.variables.update(all_project_variables)
+            if isinstance(all_project_variables.get(name), dict):
+                project_vars_copy = all_project_variables.copy()
+                package_scoped_vars = project_vars_copy.pop(name)
+                package.variables.update(project_vars_copy)
+                package.variables.update(package_scoped_vars)
             else:
+                package.variables.update(all_project_variables)
+            if variable_overrides:
                 package.variables.update(variable_overrides)
 
         return Project(context, profile, packages)

sqlmesh/dbt/seed.py

@@ -79,7 +79,11 @@ class SeedConfig(BaseModelConfig):
         kwargs["columns"] = new_columns
 
         # dbt treats single whitespace as a null value
-        csv_settings = CsvSettings(na_values=[" "], keep_default_na=True)
+        csv_settings = CsvSettings(
+            delimiter=self.delimiter,
+            na_values=[" "],
+            keep_default_na=True,
+        )
 
         return create_seed_model(
             self.canonical_name(context),
@@ -88,6 +92,7 @@ class SeedConfig(BaseModelConfig):
             audit_definitions=audit_definitions,
             virtual_environment_mode=virtual_environment_mode,
             start=self.start or context.sqlmesh_config.model_defaults.start,
+            dbt_node_info=self.node_info,
             **kwargs,
         )
 

sqlmesh/dbt/source.py

@@ -8,6 +8,7 @@ from sqlmesh.core.config.base import UpdateStrategy
 from sqlmesh.dbt.column import ColumnConfig
 from sqlmesh.dbt.common import GeneralConfig
 from sqlmesh.dbt.relation import RelationType
+from sqlmesh.dbt.util import DBT_VERSION
 from sqlmesh.utils import AttributeDict
 from sqlmesh.utils.errors import ConfigError
 
@@ -35,6 +36,7 @@ class SourceConfig(GeneralConfig):
     # DBT configuration fields
     name: str = ""
     source_name_: str = Field("", alias="source_name")
+    fqn_: t.List[str] = Field(default_factory=list, alias="fqn")
     database: t.Optional[str] = None
     schema_: t.Optional[str] = Field(None, alias="schema")
     identifier: t.Optional[str] = None
@@ -46,6 +48,7 @@ class SourceConfig(GeneralConfig):
     external: t.Optional[t.Dict[str, t.Any]] = {}
     source_meta: t.Optional[t.Dict[str, t.Any]] = {}
     columns: t.Dict[str, ColumnConfig] = {}
+    event_time: t.Optional[str] = None
 
     _canonical_name: t.Optional[str] = None
 
@@ -62,6 +65,10 @@ class SourceConfig(GeneralConfig):
     def config_name(self) -> str:
         return f"{self.source_name_}.{self.name}"
 
+    @property
+    def fqn(self) -> str:
+        return ".".join(self.fqn_)
+
     def canonical_name(self, context: DbtContext) -> str:
         if self._canonical_name is None:
             source = context.get_callable_macro("source")
@@ -72,7 +79,7 @@ class SourceConfig(GeneralConfig):
                 relation = source(self.source_name_, self.name)
             except Exception as e:
                 raise ConfigError(
-                    f"'source' macro failed for '{self.config_name}' with exeception '{e}'."
+                    f"'source' macro failed for '{self.config_name}' with exception '{e}'."
                 )
 
             relation = relation.quote(
@@ -94,6 +101,11 @@ class SourceConfig(GeneralConfig):
         if external_location:
             extras["external"] = external_location.replace("{name}", self.table_name)
 
+        if DBT_VERSION >= (1, 9, 0) and self.event_time:
+            extras["event_time_filter"] = {
+                "field_name": self.event_time,
+            }
+
         return AttributeDict(
             {
                 "database": self.database,

sqlmesh/dbt/target.py

@@ -45,10 +45,24 @@ IncrementalKind = t.Union[
 
 # We only serialize a subset of fields in order to avoid persisting sensitive information
 SERIALIZABLE_FIELDS = {
-    "type",
+    # core
     "name",
-    "database",
     "schema_",
+    "type",
+    "threads",
+    # snowflake
+    "database",
+    "warehouse",
+    "user",
+    "role",
+    "account",
+    # postgres/redshift
+    "dbname",
+    "host",
+    "port",
+    # bigquery
+    "project",
+    "dataset",
 }
 
 SCHEMA_DIFFER_OVERRIDES = {
@@ -587,12 +601,17 @@ class BigQueryConfig(TargetConfig):
         if not isinstance(data, dict):
             return data
 
-        data["schema"] = data.get("schema") or data.get("dataset")
-        if not data["schema"]:
+        # dbt treats schema and dataset interchangeably
+        schema = data.get("schema") or data.get("dataset")
+        if not schema:
             raise ConfigError("Either schema or dataset must be set")
-        data["database"] = data.get("database") or data.get("project")
-        if not data["database"]:
+        data["dataset"] = data["schema"] = schema
+
+        # dbt treats database and project interchangeably
+        database = data.get("database") or data.get("project")
+        if not database:
             raise ConfigError("Either database or project must be set")
+        data["database"] = data["project"] = database
 
         return data