sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev20__py3-none-any.whl

sqlmesh/dbt/basemodel.py

@@ -13,6 +13,8 @@ from sqlmesh.core import dialect as d
 from sqlmesh.core.config.base import UpdateStrategy
 from sqlmesh.core.config.common import VirtualEnvironmentMode
 from sqlmesh.core.model import Model
+from sqlmesh.core.model.common import ParsableSql
+from sqlmesh.core.node import DbtNodeInfo
 from sqlmesh.dbt.column import (
     ColumnConfig,
     column_descriptions_to_sqlmesh,
@@ -56,6 +58,12 @@ class Materialization(str, Enum):
     # Snowflake, https://docs.getdbt.com/reference/resource-configs/snowflake-configs#dynamic-tables
     DYNAMIC_TABLE = "dynamic_table"
 
+    CUSTOM = "custom"
+
+    @classmethod
+    def _missing_(cls, value):  # type: ignore
+        return cls.CUSTOM
+
 
 class SnapshotStrategy(str, Enum):
     """DBT snapshot strategies"""
@@ -80,7 +88,7 @@ class Hook(DbtConfig):
     """
 
     sql: SqlStr
-    transaction: bool = True  # TODO not yet supported
+    transaction: bool = True
 
     _sql_validator = sql_str_validator
 
@@ -120,8 +128,10 @@ class BaseModelConfig(GeneralConfig):
     grain: t.Union[str, t.List[str]] = []
 
     # DBT configuration fields
+    unique_id: str = ""
     name: str = ""
     package_name: str = ""
+    fqn_: t.List[str] = Field(default_factory=list, alias="fqn")
     schema_: str = Field("", alias="schema")
     database: t.Optional[str] = None
     alias: t.Optional[str] = None
@@ -156,7 +166,11 @@ class BaseModelConfig(GeneralConfig):
 
     @field_validator("grants", mode="before")
     @classmethod
-    def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]:
+    def _validate_grants(
+        cls, v: t.Optional[t.Dict[str, str]]
+    ) -> t.Optional[t.Dict[str, t.List[str]]]:
+        if v is None:
+            return None
         return {key: ensure_list(value) for key, value in v.items()}
 
     _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
@@ -268,44 +282,17 @@ class BaseModelConfig(GeneralConfig):
             and all(source in context.sources for source in test.dependencies.sources)
         ]
 
-    def fix_circular_test_refs(self, context: DbtContext) -> None:
-        """
-        Checks for direct circular references between two models and moves the test to the downstream
-        model if found. This addresses the most common circular reference - relationship tests in both
-        directions. In the future, we may want to increase coverage by checking for indirect circular references.
-
-        Args:
-            context: The dbt context this model resides within.
-
-        Returns:
-            None
-        """
-        for test in self.tests.copy():
-            for ref in test.dependencies.refs:
-                if ref == self.name or ref in self.dependencies.refs:
-                    continue
-                model = context.refs[ref]
-                if (
-                    self.name in model.dependencies.refs
-                    or self.name in model.tests_ref_source_dependencies.refs
-                ):
-                    logger.info(
-                        f"Moving test '{test.name}' from model '{self.name}' to '{model.name}' to avoid circular reference."
-                    )
-                    model.tests.append(test)
-                    self.tests.remove(test)
+    @property
+    def fqn(self) -> str:
+        return ".".join(self.fqn_)
 
     @property
     def sqlmesh_config_fields(self) -> t.Set[str]:
         return {"description", "owner", "stamp", "storage_format"}
 
     @property
-    def node_name(self) -> str:
-        resource_type = getattr(self, "resource_type", "model")
-        node_name = f"{resource_type}.{self.package_name}.{self.name}"
-        if self.version:
-            node_name += f".v{self.version}"
-        return node_name
+    def node_info(self) -> DbtNodeInfo:
+        return DbtNodeInfo(unique_id=self.unique_id, name=self.name, fqn=self.fqn, alias=self.alias)
 
     def sqlmesh_model_kwargs(
         self,
@@ -314,7 +301,6 @@ class BaseModelConfig(GeneralConfig):
     ) -> t.Dict[str, t.Any]:
         """Get common sqlmesh model parameters"""
         self.remove_tests_with_invalid_refs(context)
-        self.fix_circular_test_refs(context)
 
         dependencies = self.dependencies.copy()
         if dependencies.has_dynamic_var_names:
@@ -322,7 +308,19 @@ class BaseModelConfig(GeneralConfig):
             # precisely which variables are referenced in the model
             dependencies.variables |= set(context.variables)
 
+        if (
+            getattr(self, "model_materialization", None) == Materialization.CUSTOM
+            and hasattr(self, "_get_custom_materialization")
+            and (custom_mat := self._get_custom_materialization(context))
+        ):
+            # include custom materialization dependencies as they might use macros
+            dependencies = dependencies.union(custom_mat.dependencies)
+
         model_dialect = self.dialect(context)
+
+        # Only keep refs and sources that exist in the context to match dbt behavior
+        dependencies.refs.intersection_update(context.refs)
+        dependencies.sources.intersection_update(context.sources)
         model_context = context.context_for_dependencies(
             dependencies.union(self.tests_ref_source_dependencies)
         )
@@ -332,15 +330,28 @@ class BaseModelConfig(GeneralConfig):
         jinja_macros.add_globals(self._model_jinja_context(model_context, dependencies))
 
         model_kwargs = {
-            "audits": [(test.name, {}) for test in self.tests],
+            "audits": [(test.canonical_name, {}) for test in self.tests],
             "column_descriptions": column_descriptions_to_sqlmesh(self.columns) or None,
             "depends_on": {
                 model.canonical_name(context) for model in model_context.refs.values()
-            }.union({source.canonical_name(context) for source in model_context.sources.values()}),
+            }.union(
+                {
+                    source.canonical_name(context)
+                    for source in model_context.sources.values()
+                    if source.fqn not in context.model_fqns
+                    # Allow dbt projects to reference a model as a source without causing a cycle
+                },
+            ),
             "jinja_macros": jinja_macros,
             "path": self.path,
-            "pre_statements": [d.jinja_statement(hook.sql) for hook in self.pre_hook],
-            "post_statements": [d.jinja_statement(hook.sql) for hook in self.post_hook],
+            "pre_statements": [
+                ParsableSql(sql=d.jinja_statement(hook.sql).sql(), transaction=hook.transaction)
+                for hook in self.pre_hook
+            ],
+            "post_statements": [
+                ParsableSql(sql=d.jinja_statement(hook.sql).sql(), transaction=hook.transaction)
+                for hook in self.post_hook
+            ],
             "tags": self.tags,
             "physical_schema_mapping": context.sqlmesh_config.physical_schema_mapping,
             "default_catalog": context.target.database,
@@ -377,8 +388,8 @@ class BaseModelConfig(GeneralConfig):
     def _model_jinja_context(
         self, context: DbtContext, dependencies: Dependencies
     ) -> t.Dict[str, t.Any]:
-        if context._manifest and self.node_name in context._manifest._manifest.nodes:
-            attributes = context._manifest._manifest.nodes[self.node_name].to_dict()
+        if context._manifest and self.unique_id in context._manifest._manifest.nodes:
+            attributes = context._manifest._manifest.nodes[self.unique_id].to_dict()
             if dependencies.model_attrs.all_attrs:
                 model_node: AttributeDict[str, t.Any] = AttributeDict(attributes)
             else:

sqlmesh/dbt/builtin.py

@@ -25,7 +25,7 @@ from sqlmesh.dbt.target import TARGET_TYPE_TO_CONFIG_CLASS
 from sqlmesh.dbt.util import DBT_VERSION
 from sqlmesh.utils import AttributeDict, debug_mode_enabled, yaml
 from sqlmesh.utils.date import now
-from sqlmesh.utils.errors import ConfigError, MacroEvalError
+from sqlmesh.utils.errors import ConfigError
 from sqlmesh.utils.jinja import JinjaMacroRegistry, MacroReference, MacroReturnVal
 
 logger = logging.getLogger(__name__)
@@ -50,6 +50,22 @@ class Exceptions:
         return ""
 
 
+def try_or_compiler_error(
+    message_if_exception: str, func: t.Callable, *args: t.Any, **kwargs: t.Any
+) -> t.Any:
+    try:
+        return func(*args, **kwargs)
+    except Exception:
+        if DBT_VERSION >= (1, 4, 0):
+            from dbt.exceptions import CompilationError
+
+            raise CompilationError(message_if_exception)
+        else:
+            from dbt.exceptions import CompilationException  # type: ignore
+
+            raise CompilationException(message_if_exception)
+
+
 class Api:
     def __init__(self, dialect: t.Optional[str]) -> None:
         if dialect:
@@ -365,18 +381,16 @@ def do_zip(*args: t.Any, default: t.Optional[t.Any] = None) -> t.Optional[t.Any]
         return default
 
 
-def as_bool(value: str) -> bool:
-    result = _try_literal_eval(value)
-    if isinstance(result, bool):
-        return result
-    raise MacroEvalError(f"Failed to convert '{value}' into boolean.")
+def as_bool(value: t.Any) -> t.Any:
+    # dbt's jinja TEXT_FILTERS just return the input value as is
+    # https://github.com/dbt-labs/dbt-common/blob/main/dbt_common/clients/jinja.py#L559
+    return value
 
 
 def as_number(value: str) -> t.Any:
-    result = _try_literal_eval(value)
-    if isinstance(value, (int, float)) and not isinstance(result, bool):
-        return result
-    raise MacroEvalError(f"Failed to convert '{value}' into number.")
+    # dbt's jinja TEXT_FILTERS just return the input value as is
+    # https://github.com/dbt-labs/dbt-common/blob/main/dbt_common/clients/jinja.py#L559
+    return value
 
 
 def _try_literal_eval(value: str) -> t.Any:
@@ -411,6 +425,7 @@ BUILTIN_GLOBALS = {
     "sqlmesh_incremental": True,
     "tojson": to_json,
     "toyaml": to_yaml,
+    "try_or_compiler_error": try_or_compiler_error,
     "zip": do_zip,
     "zip_strict": lambda *args: list(zip(*args)),
 }
@@ -465,7 +480,7 @@ def create_builtin_globals(
     if variables is not None:
         builtin_globals["var"] = Var(variables)
 
-    builtin_globals["config"] = Config(jinja_globals.pop("config", {}))
+    builtin_globals["config"] = Config(jinja_globals.pop("config", {"tags": []}))
 
     deployability_index = (
         jinja_globals.get("deployability_index") or DeployabilityIndex.all_deployable()
@@ -546,6 +561,7 @@ def create_builtin_globals(
             "statement": sql_execution.statement,
             "graph": adapter.graph,
             "selected_resources": list(jinja_globals.get("selected_models") or []),
+            "write": lambda input: None,  # We don't support writing yet
         }
     )
 

sqlmesh/dbt/column.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import typing as t
+import logging
 
 from sqlglot import exp, parse_one
 from sqlglot.helper import ensure_list
@@ -9,6 +10,8 @@ from sqlmesh.dbt.common import GeneralConfig
 from sqlmesh.utils.conversions import ensure_bool
 from sqlmesh.utils.pydantic import field_validator
 
+logger = logging.getLogger(__name__)
+
 
 def yaml_to_columns(
     yaml: t.Dict[str, ColumnConfig] | t.List[t.Dict[str, ColumnConfig]],
@@ -31,11 +34,20 @@ def column_types_to_sqlmesh(
     Returns:
         A dict of column name to exp.DataType
     """
-    return {
-        name: parse_one(column.data_type, into=exp.DataType, dialect=dialect or "")
-        for name, column in columns.items()
-        if column.enabled and column.data_type
-    }
+    col_types_to_sqlmesh: t.Dict[str, exp.DataType] = {}
+    for name, column in columns.items():
+        if column.enabled and column.data_type:
+            column_def = parse_one(
+                f"{name} {column.data_type}", into=exp.ColumnDef, dialect=dialect or ""
+            )
+            if column_def.args.get("constraints"):
+                logger.warning(
+                    f"Ignoring unsupported constraints for column '{name}' with definition '{column.data_type}'. Please refer to github.com/TobikoData/sqlmesh/issues/4717 for more information."
+                )
+            kind = column_def.kind
+            if kind:
+                col_types_to_sqlmesh[name] = kind
+    return col_types_to_sqlmesh
 
 
 def column_descriptions_to_sqlmesh(columns: t.Dict[str, ColumnConfig]) -> t.Dict[str, str]:

sqlmesh/dbt/common.py

@@ -46,7 +46,9 @@ def load_yaml(source: str | Path) -> t.Dict:
         raise ConfigError(f"{source}: {ex}" if isinstance(source, Path) else f"{ex}")
 
 
-def parse_meta(v: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
+def parse_meta(v: t.Optional[t.Dict[str, t.Any]]) -> t.Dict[str, t.Any]:
+    if v is None:
+        return {}
     for key, value in v.items():
         if isinstance(value, str):
             v[key] = try_str_to_bool(value)
@@ -115,7 +117,7 @@ class GeneralConfig(DbtConfig):
 
     @field_validator("meta", mode="before")
     @classmethod
-    def _validate_meta(cls, v: t.Dict[str, t.Union[str, t.Any]]) -> t.Dict[str, t.Any]:
+    def _validate_meta(cls, v: t.Optional[t.Dict[str, t.Union[str, t.Any]]]) -> t.Dict[str, t.Any]:
         return parse_meta(v)
 
     _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {

sqlmesh/dbt/context.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import typing as t
 from dataclasses import dataclass, field, replace
 from pathlib import Path
@@ -28,12 +29,16 @@ if t.TYPE_CHECKING:
     from sqlmesh.dbt.seed import SeedConfig
     from sqlmesh.dbt.source import SourceConfig
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class DbtContext:
     """Context for DBT environment"""
 
     project_root: Path = Path()
+    profiles_dir: t.Optional[Path] = None
+    """Optional override to specify the directory where profiles.yml is located, if not at the :project_root"""
     target_name: t.Optional[str] = None
     profile_name: t.Optional[str] = None
     project_schema: t.Optional[str] = None
@@ -48,6 +53,7 @@ class DbtContext:
     _project_name: t.Optional[str] = None
     _variables: t.Dict[str, t.Any] = field(default_factory=dict)
     _models: t.Dict[str, ModelConfig] = field(default_factory=dict)
+    _model_fqns: t.Set[str] = field(default_factory=set)
     _seeds: t.Dict[str, SeedConfig] = field(default_factory=dict)
     _sources: t.Dict[str, SourceConfig] = field(default_factory=dict)
     _refs: t.Dict[str, t.Union[ModelConfig, SeedConfig]] = field(default_factory=dict)
@@ -125,7 +131,7 @@ class DbtContext:
             try:
                 rendered_variables[k] = _render_var(v)
             except Exception as ex:
-                raise ConfigError(f"Failed to render variable '{k}', value '{v}': {ex}") from ex
+                logger.warning(f"Failed to render variable '{k}', value '{v}': {ex}")
 
         self.variables = rendered_variables
 
@@ -141,6 +147,7 @@ class DbtContext:
     def models(self, models: t.Dict[str, ModelConfig]) -> None:
         self._models = {}
         self._refs = {}
+        self._model_fqns = set()
         self.add_models(models)
 
     def add_models(self, models: t.Dict[str, ModelConfig]) -> None:
@@ -148,6 +155,12 @@ class DbtContext:
         self._models.update(models)
         self._jinja_environment = None
 
+    @property
+    def model_fqns(self) -> t.Set[str]:
+        if not self._model_fqns:
+            self._model_fqns = {model.fqn for model in self._models.values()}
+        return self._model_fqns
+
     @property
     def seeds(self) -> t.Dict[str, SeedConfig]:
         return self._seeds

sqlmesh/dbt/loader.py

@@ -5,11 +5,13 @@ import sys
 import typing as t
 import sqlmesh.core.dialect as d
 from pathlib import Path
+from collections import defaultdict
 from sqlmesh.core.config import (
     Config,
     ConnectionConfig,
     GatewayConfig,
     ModelDefaultsConfig,
+    DbtConfig as RootDbtConfig,
 )
 from sqlmesh.core.environment import EnvironmentStatements
 from sqlmesh.core.loader import CacheBase, LoadedProject, Loader
@@ -48,11 +50,21 @@ def sqlmesh_config(
     dbt_profile_name: t.Optional[str] = None,
     dbt_target_name: t.Optional[str] = None,
     variables: t.Optional[t.Dict[str, t.Any]] = None,
+    threads: t.Optional[int] = None,
     register_comments: t.Optional[bool] = None,
+    infer_state_schema_name: bool = False,
+    profiles_dir: t.Optional[Path] = None,
     **kwargs: t.Any,
 ) -> Config:
     project_root = project_root or Path()
-    context = DbtContext(project_root=project_root, profile_name=dbt_profile_name)
+    context = DbtContext(
+        project_root=project_root, profiles_dir=profiles_dir, profile_name=dbt_profile_name
+    )
+
+    # note: Profile.load() is called twice with different DbtContext's:
+    # - once here with the above DbtContext (to determine connnection / gateway config which has to be set up before everything else)
+    # - again on the SQLMesh side via GenericContext.load() -> DbtLoader._load_projects() -> Project.load() which constructs a fresh DbtContext and ignores the above one
+    # it's important to ensure that the DbtContext created within the DbtLoader uses the same project root / profiles dir that we use here
     profile = Profile.load(context, target_name=dbt_target_name)
     model_defaults = kwargs.pop("model_defaults", ModelDefaultsConfig())
     if model_defaults.dialect is None:
@@ -66,16 +78,45 @@ def sqlmesh_config(
     if not issubclass(loader, DbtLoader):
         raise ConfigError("The loader must be a DbtLoader.")
 
+    if threads is not None:
+        # the to_sqlmesh() function on TargetConfig maps self.threads -> concurrent_tasks
+        profile.target.threads = threads
+
+    gateway_kwargs = {}
+    if infer_state_schema_name:
+        profile_name = context.profile_name
+
+        # Note: we deliberately isolate state based on the target *schema* and not the target name.
+        # It is assumed that the project will define a target, eg 'dev', and then in each users own ~/.dbt/profiles.yml the schema
+        # for the 'dev' target is overriden to something user-specific, rather than making the target name itself user-specific.
+        # This means that the schema name is the indicator of isolated state, not the target name which may be re-used across multiple schemas.
+        target_schema = profile.target.schema_
+
+        # dbt-core doesnt allow schema to be undefined, but it does allow an empty string, and then just
+        # fails at runtime when `CREATE SCHEMA ""` doesnt work
+        if not target_schema:
+            raise ConfigError(
+                f"Target '{profile.target_name}' does not specify a schema.\n"
+                "A schema is required in order to infer where to store SQLMesh state"
+            )
+
+        inferred_state_schema_name = f"sqlmesh_state_{profile_name}_{target_schema}"
+        logger.info("Inferring state schema: %s", inferred_state_schema_name)
+        gateway_kwargs["state_schema"] = inferred_state_schema_name
+
     return Config(
         loader=loader,
+        loader_kwargs=dict(profiles_dir=profiles_dir),
         model_defaults=model_defaults,
         variables=variables or {},
+        dbt=RootDbtConfig(infer_state_schema_name=infer_state_schema_name),
         **{
             "default_gateway": profile.target_name if "gateways" not in kwargs else "",
             "gateways": {
                 profile.target_name: GatewayConfig(
                     connection=profile.target.to_sqlmesh(**target_to_sqlmesh_args),
                     state_connection=state_connection,
+                    **gateway_kwargs,
                 )
             },  # type: ignore
             **kwargs,
@@ -84,9 +125,12 @@ def sqlmesh_config(
 
 
 class DbtLoader(Loader):
-    def __init__(self, context: GenericContext, path: Path) -> None:
+    def __init__(
+        self, context: GenericContext, path: Path, profiles_dir: t.Optional[Path] = None
+    ) -> None:
         self._projects: t.List[Project] = []
         self._macros_max_mtime: t.Optional[float] = None
+        self._profiles_dir = profiles_dir
         super().__init__(context, path)
 
     def load(self) -> LoadedProject:
@@ -137,16 +181,22 @@ class DbtLoader(Loader):
                 package_context.set_and_render_variables(package.variables, package.name)
                 package_models: t.Dict[str, BaseModelConfig] = {**package.models, **package.seeds}
 
+                package_models_by_path: t.Dict[Path, t.List[BaseModelConfig]] = defaultdict(list)
                 for model in package_models.values():
                     if isinstance(model, ModelConfig) and not model.sql.strip():
                         logger.info(f"Skipping empty model '{model.name}' at path '{model.path}'.")
                         continue
+                    package_models_by_path[model.path].append(model)
 
-                    sqlmesh_model = cache.get_or_load_models(
-                        model.path, loader=lambda: [_to_sqlmesh(model, package_context)]
-                    )[0]
-
-                    models[sqlmesh_model.fqn] = sqlmesh_model
+                for path, path_models in package_models_by_path.items():
+                    sqlmesh_models = cache.get_or_load_models(
+                        path,
+                        loader=lambda: [
+                            _to_sqlmesh(model, package_context) for model in path_models
+                        ],
+                    )
+                    for sqlmesh_model in sqlmesh_models:
+                        models[sqlmesh_model.fqn] = sqlmesh_model
 
             models.update(self._load_external_models(audits, cache))
 
@@ -165,7 +215,8 @@ class DbtLoader(Loader):
                 for test in package.tests.values():
                     logger.debug("Converting '%s' to sqlmesh format", test.name)
                     try:
-                        audits[test.name] = test.to_sqlmesh(package_context)
+                        audits[test.canonical_name] = test.to_sqlmesh(package_context)
+
                     except BaseMissingReferenceError as e:
                         ref_type = "model" if isinstance(e, MissingModelError) else "source"
                         logger.warning(
@@ -186,6 +237,7 @@ class DbtLoader(Loader):
             project = Project.load(
                 DbtContext(
                     project_root=self.config_path,
+                    profiles_dir=self._profiles_dir,
                     target_name=target_name,
                     sqlmesh_config=self.config,
                 ),