sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev20__py3-none-any.whl

sqlmesh/__init__.py

@@ -126,6 +126,8 @@ def is_cicd_environment() -> bool:
 
 
 def is_interactive_environment() -> bool:
+    if sys.stdin is None or sys.stdout is None:
+        return False
     return sys.stdin.isatty() and sys.stdout.isatty()
 
 
@@ -186,6 +188,7 @@ def configure_logging(
     write_to_file: bool = True,
     log_file_dir: t.Optional[t.Union[str, Path]] = None,
     ignore_warnings: bool = False,
+    log_level: t.Optional[t.Union[str, int]] = None,
 ) -> None:
     # Remove noisy grpc logs that are not useful for users
     os.environ["GRPC_VERBOSITY"] = os.environ.get("GRPC_VERBOSITY", "NONE")
@@ -193,8 +196,15 @@ def configure_logging(
     logger = logging.getLogger()
     debug = force_debug or debug_mode_enabled()
 
-    # base logger needs to be the lowest level that we plan to log
-    level = logging.DEBUG if debug else logging.INFO
+    if log_level is not None:
+        if isinstance(log_level, str):
+            level = logging._nameToLevel.get(log_level.upper()) or logging.INFO
+        else:
+            level = log_level
+    else:
+        # base logger needs to be the lowest level that we plan to log
+        level = logging.DEBUG if debug else logging.INFO
+
     logger.setLevel(level)
 
     if debug:

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.217.1.dev1'
-__version_tuple__ = version_tuple = (0, 217, 1, 'dev1')
+__version__ = version = '0.227.2.dev20'
+__version_tuple__ = version_tuple = (0, 227, 2, 'dev20')
 
 __commit_id__ = commit_id = None

sqlmesh/cli/project_init.py

@@ -116,7 +116,15 @@ linter:
     - invalidselectstarexpansion
     - noambiguousprojections
 """,
-        ProjectTemplate.DBT: f"""# --- Virtual Data Environment Mode ---
+        ProjectTemplate.DBT: f"""# --- DBT-specific options ---
+dbt:
+  # This configuration ensures that each dbt target gets its own isolated state.
+  # The inferred state schemas are named "sqlmesh_state_<profile name>_<target schema>", eg "sqlmesh_state_jaffle_shop_dev"
+  # If this is undesirable, you may manually configure the gateway to use a specific state schema name
+  # https://sqlmesh.readthedocs.io/en/stable/integrations/dbt/#selecting-a-different-state-connection
+  infer_state_schema_name: True
+
+# --- Virtual Data Environment Mode ---
 # Enable Virtual Data Environments (VDE) for *development* environments.
 # Note that the production environment in dbt projects is not virtual by default to maintain compatibility with existing tooling.
 # https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#virtual-data-environment-modes
@@ -298,6 +306,7 @@ def init_example_project(
     dlt_path: t.Optional[str] = None,
     schema_name: str = "sqlmesh_example",
     cli_mode: InitCliMode = InitCliMode.DEFAULT,
+    start: t.Optional[str] = None,
 ) -> Path:
     root_path = Path(path)
 
@@ -336,7 +345,6 @@ def init_example_project(
 
     models: t.Set[t.Tuple[str, str]] = set()
     settings = None
-    start = None
     if engine_type and template == ProjectTemplate.DLT:
         project_dialect = dialect or DIALECT_TO_TYPE.get(engine_type)
         if pipeline and project_dialect:

sqlmesh/core/_typing.py

@@ -11,6 +11,7 @@ if t.TYPE_CHECKING:
     SessionProperties = t.Dict[str, t.Union[exp.Expression, str, int, float, bool]]
     CustomMaterializationProperties = t.Dict[str, t.Union[exp.Expression, str, int, float, bool]]
 
+
 if sys.version_info >= (3, 11):
     from typing import Self as Self
 else:

sqlmesh/core/audit/definition.py

@@ -19,7 +19,7 @@ from sqlmesh.core.model.common import (
     sorted_python_env_payloads,
 )
 from sqlmesh.core.model.common import make_python_env, single_value_or_tuple, ParsableSql
-from sqlmesh.core.node import _Node
+from sqlmesh.core.node import _Node, DbtInfoMixin, DbtNodeInfo
 from sqlmesh.core.renderer import QueryRenderer
 from sqlmesh.utils.date import TimeLike
 from sqlmesh.utils.errors import AuditConfigError, SQLMeshError, raise_config_error
@@ -120,7 +120,7 @@ def audit_map_validator(cls: t.Type, v: t.Any, values: t.Any) -> t.Dict[str, t.A
     return {}
 
 
-class ModelAudit(PydanticModel, AuditMixin, frozen=True):
+class ModelAudit(PydanticModel, AuditMixin, DbtInfoMixin, frozen=True):
     """
     Audit is an assertion made about your tables.
 
@@ -137,6 +137,7 @@ class ModelAudit(PydanticModel, AuditMixin, frozen=True):
     expressions_: t.Optional[t.List[ParsableSql]] = Field(default=None, alias="expressions")
     jinja_macros: JinjaMacroRegistry = JinjaMacroRegistry()
     formatting: t.Optional[bool] = Field(default=None, exclude=True)
+    dbt_node_info_: t.Optional[DbtNodeInfo] = Field(alias="dbt_node_info", default=None)
 
     _path: t.Optional[Path] = None
 
@@ -150,6 +151,10 @@ class ModelAudit(PydanticModel, AuditMixin, frozen=True):
         path = f": {self._path.name}" if self._path else ""
         return f"{self.__class__.__name__}<{self.name}{path}>"
 
+    @property
+    def dbt_node_info(self) -> t.Optional[DbtNodeInfo]:
+        return self.dbt_node_info_
+
 
 class StandaloneAudit(_Node, AuditMixin):
     """
@@ -552,4 +557,5 @@ META_FIELD_CONVERTER: t.Dict[str, t.Callable] = {
     "depends_on_": lambda value: exp.Tuple(expressions=sorted(value)),
     "tags": single_value_or_tuple,
     "default_catalog": exp.to_identifier,
+    "dbt_node_info_": lambda value: value.to_expression(),
 }

sqlmesh/core/config/__init__.py

@@ -36,6 +36,6 @@ from sqlmesh.core.config.model import ModelDefaultsConfig as ModelDefaultsConfig
 from sqlmesh.core.config.naming import NameInferenceConfig as NameInferenceConfig
 from sqlmesh.core.config.linter import LinterConfig as LinterConfig
 from sqlmesh.core.config.plan import PlanConfig as PlanConfig
-from sqlmesh.core.config.root import Config as Config
+from sqlmesh.core.config.root import Config as Config, DbtConfig as DbtConfig
 from sqlmesh.core.config.run import RunConfig as RunConfig
 from sqlmesh.core.config.scheduler import BuiltInSchedulerConfig as BuiltInSchedulerConfig

sqlmesh/core/config/connection.py

@@ -58,6 +58,7 @@ FORBIDDEN_STATE_SYNC_ENGINES = {
     "clickhouse",
 }
 MOTHERDUCK_TOKEN_REGEX = re.compile(r"(\?|\&)(motherduck_token=)(\S*)")
+PASSWORD_REGEX = re.compile(r"(password=)(\S+)")
 
 
 def _get_engine_import_validator(
@@ -101,6 +102,7 @@ class ConnectionConfig(abc.ABC, BaseConfig):
     pre_ping: bool
     pretty_sql: bool = False
     schema_differ_overrides: t.Optional[t.Dict[str, t.Any]] = None
+    catalog_type_overrides: t.Optional[t.Dict[str, str]] = None
 
     # Whether to share a  single connection across threads or create a new connection per thread.
     shared_connection: t.ClassVar[bool] = False
@@ -176,6 +178,7 @@ class ConnectionConfig(abc.ABC, BaseConfig):
             pretty_sql=self.pretty_sql,
             shared_connection=self.shared_connection,
             schema_differ_overrides=self.schema_differ_overrides,
+            catalog_type_overrides=self.catalog_type_overrides,
             **self._extra_engine_config,
         )
 
@@ -235,6 +238,7 @@ class DuckDBAttachOptions(BaseConfig):
     data_path: t.Optional[str] = None
     encrypted: bool = False
     data_inlining_row_limit: t.Optional[int] = None
+    metadata_schema: t.Optional[str] = None
 
     def to_sql(self, alias: str) -> str:
         options = []
@@ -256,6 +260,8 @@ class DuckDBAttachOptions(BaseConfig):
                 options.append("ENCRYPTED")
             if self.data_inlining_row_limit is not None:
                 options.append(f"DATA_INLINING_ROW_LIMIT {self.data_inlining_row_limit}")
+            if self.metadata_schema is not None:
+                options.append(f"METADATA_SCHEMA '{self.metadata_schema}'")
 
         options_sql = f" ({', '.join(options)})" if options else ""
         alias_sql = ""
@@ -477,13 +483,13 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
             adapter = BaseDuckDBConnectionConfig._data_file_to_adapter.get(key)
             if adapter is not None:
                 logger.info(
-                    f"Using existing DuckDB adapter due to overlapping data file: {self._mask_motherduck_token(key)}"
+                    f"Using existing DuckDB adapter due to overlapping data file: {self._mask_sensitive_data(key)}"
                 )
                 return adapter
 
         if data_files:
             masked_files = {
-                self._mask_motherduck_token(file if isinstance(file, str) else file.path)
+                self._mask_sensitive_data(file if isinstance(file, str) else file.path)
                 for file in data_files
             }
             logger.info(f"Creating new DuckDB adapter for data files: {masked_files}")
@@ -505,10 +511,14 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
             return list(self.catalogs)[0]
         return None
 
-    def _mask_motherduck_token(self, string: str) -> str:
-        return MOTHERDUCK_TOKEN_REGEX.sub(
-            lambda m: f"{m.group(1)}{m.group(2)}{'*' * len(m.group(3))}", string
+    def _mask_sensitive_data(self, string: str) -> str:
+        # Mask MotherDuck tokens with fixed number of asterisks
+        result = MOTHERDUCK_TOKEN_REGEX.sub(
+            lambda m: f"{m.group(1)}{m.group(2)}{'*' * 8 if m.group(3) else ''}", string
         )
+        # Mask PostgreSQL/MySQL passwords with fixed number of asterisks
+        result = PASSWORD_REGEX.sub(lambda m: f"{m.group(1)}{'*' * 8}", result)
+        return result
 
 
 class MotherDuckConnectionConfig(BaseDuckDBConnectionConfig):
@@ -1753,6 +1763,7 @@ class SparkConnectionConfig(ConnectionConfig):
     config_dir: t.Optional[str] = None
     catalog: t.Optional[str] = None
     config: t.Dict[str, t.Any] = {}
+    wap_enabled: bool = False
 
     concurrent_tasks: int = 4
     register_comments: bool = True
@@ -1799,6 +1810,10 @@ class SparkConnectionConfig(ConnectionConfig):
             .getOrCreate(),
         }
 
+    @property
+    def _extra_engine_config(self) -> t.Dict[str, t.Any]:
+        return {"wap_enabled": self.wap_enabled}
+
 
 class TrinoAuthenticationMethod(str, Enum):
     NO_AUTH = "no-auth"

sqlmesh/core/config/dbt.py

@@ -0,0 +1,13 @@
+from sqlmesh.core.config.base import BaseConfig
+
+
+class DbtConfig(BaseConfig):
+    """
+    Represents dbt-specific options on the SQLMesh root config.
+
+    These options are only taken into account for dbt projects and are ignored on native projects
+    """
+
+    infer_state_schema_name: bool = False
+    """If set, indicates to the dbt loader that the state schema should be inferred based on the profile/target
+    so that each target gets its own isolated state"""

sqlmesh/core/config/janitor.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import typing as t
 
 from sqlmesh.core.config.base import BaseConfig
+from sqlmesh.utils.pydantic import field_validator
 
 
 class JanitorConfig(BaseConfig):
@@ -9,6 +11,16 @@ class JanitorConfig(BaseConfig):
 
     Args:
         warn_on_delete_failure: Whether to warn instead of erroring if the janitor fails to delete the expired environment schema / views.
+        expired_snapshots_batch_size: Maximum number of expired snapshots to clean in a single batch.
     """
 
     warn_on_delete_failure: bool = False
+    expired_snapshots_batch_size: t.Optional[int] = None
+
+    @field_validator("expired_snapshots_batch_size", mode="before")
+    @classmethod
+    def _validate_batch_size(cls, value: int) -> int:
+        batch_size = int(value)
+        if batch_size <= 0:
+            raise ValueError("expired_snapshots_batch_size must be greater than 0")
+        return batch_size

sqlmesh/core/config/loader.py

@@ -172,11 +172,18 @@ def load_config_from_paths(
         if dbt_project_file:
             from sqlmesh.dbt.loader import sqlmesh_config
 
+            infer_state_schema_name = False
+            if dbt := non_python_config.dbt:
+                infer_state_schema_name = dbt.infer_state_schema_name
+
             dbt_python_config = sqlmesh_config(
                 project_root=dbt_project_file.parent,
+                profiles_dir=kwargs.pop("profiles_dir", None),
                 dbt_profile_name=kwargs.pop("profile", None),
                 dbt_target_name=kwargs.pop("target", None),
                 variables=variables,
+                threads=kwargs.pop("threads", None),
+                infer_state_schema_name=infer_state_schema_name,
             )
             if type(dbt_python_config) != config_type:
                 dbt_python_config = convert_config_type(dbt_python_config, config_type)

sqlmesh/core/config/model.py

@@ -45,6 +45,7 @@ class ModelDefaultsConfig(BaseConfig):
         allow_partials: Whether the models can process partial (incomplete) data intervals.
         enabled: Whether the models are enabled.
         interval_unit: The temporal granularity of the models data intervals. By default computed from cron.
+        batch_concurrency: The maximum number of batches that can run concurrently for an incremental model.
         pre_statements: The list of SQL statements that get executed before a model runs.
         post_statements: The list of SQL statements that get executed before a model runs.
         on_virtual_update: The list of SQL statements to be executed after the virtual update.
@@ -69,6 +70,7 @@ class ModelDefaultsConfig(BaseConfig):
     interval_unit: t.Optional[t.Union[str, IntervalUnit]] = None
     enabled: t.Optional[t.Union[str, bool]] = None
     formatting: t.Optional[t.Union[str, bool]] = None
+    batch_concurrency: t.Optional[int] = None
     pre_statements: t.Optional[t.List[t.Union[str, exp.Expression]]] = None
     post_statements: t.Optional[t.List[t.Union[str, exp.Expression]]] = None
     on_virtual_update: t.Optional[t.List[t.Union[str, exp.Expression]]] = None

sqlmesh/core/config/root.py

@@ -36,6 +36,7 @@ from sqlmesh.core.config.naming import NameInferenceConfig as NameInferenceConfi
 from sqlmesh.core.config.linter import LinterConfig as LinterConfig
 from sqlmesh.core.config.plan import PlanConfig
 from sqlmesh.core.config.run import RunConfig
+from sqlmesh.core.config.dbt import DbtConfig
 from sqlmesh.core.config.scheduler import (
     BuiltInSchedulerConfig,
     SchedulerConfig,
@@ -173,6 +174,7 @@ class Config(BaseConfig):
     linter: LinterConfig = LinterConfig()
     janitor: JanitorConfig = JanitorConfig()
     cache_dir: t.Optional[str] = None
+    dbt: t.Optional[DbtConfig] = None
 
     _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
         "gateways": UpdateStrategy.NESTED_UPDATE,
@@ -191,6 +193,7 @@ class Config(BaseConfig):
         "before_all": UpdateStrategy.EXTEND,
         "after_all": UpdateStrategy.EXTEND,
         "linter": UpdateStrategy.NESTED_UPDATE,
+        "dbt": UpdateStrategy.NESTED_UPDATE,
     }
 
     _connection_config_validator = connection_config_validator

sqlmesh/core/console.py

@@ -551,6 +551,22 @@ class Console(
     def log_failed_models(self, errors: t.List[NodeExecutionFailedError]) -> None:
         """Display list of models that failed during evaluation to the user."""
 
+    @abc.abstractmethod
+    def log_models_updated_during_restatement(
+        self,
+        snapshots: t.List[t.Tuple[SnapshotTableInfo, SnapshotTableInfo]],
+        environment_naming_info: EnvironmentNamingInfo,
+        default_catalog: t.Optional[str],
+    ) -> None:
+        """Display a list of models where new versions got deployed to the specified :environment while we were restating data the old versions
+
+        Args:
+            snapshots: a list of (snapshot_we_restated, snapshot_it_got_replaced_with_during_restatement) tuples
+            environment: which environment got updated while we were restating models
+            environment_naming_info: how snapshots are named in that :environment (for display name purposes)
+            default_catalog: the configured default catalog (for display name purposes)
+        """
+
     @abc.abstractmethod
     def loading_start(self, message: t.Optional[str] = None) -> uuid.UUID:
         """Starts loading and returns a unique ID that can be used to stop the loading. Optionally can display a message."""
@@ -771,6 +787,14 @@ class NoopConsole(Console):
     def log_failed_models(self, errors: t.List[NodeExecutionFailedError]) -> None:
         pass
 
+    def log_models_updated_during_restatement(
+        self,
+        snapshots: t.List[t.Tuple[SnapshotTableInfo, SnapshotTableInfo]],
+        environment_naming_info: EnvironmentNamingInfo,
+        default_catalog: t.Optional[str],
+    ) -> None:
+        pass
+
     def log_destructive_change(
         self,
         snapshot_name: str,
@@ -1998,7 +2022,34 @@ class TerminalConsole(Console):
         plan = plan_builder.build()
 
         if plan.restatements:
-            self._print("\n[bold]Restating models\n")
+            # A plan can have restatements for the following reasons:
+            # - The user specifically called `sqlmesh plan` with --restate-model.
+            #   This creates a "restatement plan" which disallows all other changes and simply force-backfills
+            #   the selected models and their downstream dependencies using the versions of the models stored in state.
+            # - There are no specific restatements (so changes are allowed) AND dev previews need to be computed.
+            #   The "restatements" feature is currently reused for dev previews.
+            if plan.selected_models_to_restate:
+                # There were legitimate restatements, no dev previews
+                tree = Tree(
+                    "[bold]Models selected for restatement:[/bold]\n"
+                    "This causes backfill of the model itself as well as affected downstream models"
+                )
+                model_fqn_to_snapshot = {s.name: s for s in plan.snapshots.values()}
+                for model_fqn in plan.selected_models_to_restate:
+                    snapshot = model_fqn_to_snapshot[model_fqn]
+                    display_name = snapshot.display_name(
+                        plan.environment_naming_info,
+                        default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None,
+                        dialect=self.dialect,
+                    )
+                    tree.add(
+                        display_name
+                    )  # note: we deliberately dont show any intervals here; they get shown in the backfill section
+                self._print(tree)
+            else:
+                # We are computing dev previews, do not confuse the user by printing out something to do
+                # with restatements. Dev previews are already highlighted in the backfill step
+                pass
         else:
             self.show_environment_difference_summary(
                 plan.context_diff,
@@ -2225,6 +2276,30 @@ class TerminalConsole(Console):
             for node_name, msg in error_messages.items():
                 self._print(f"  [red]{node_name}[/red]\n\n{msg}")
 
+    def log_models_updated_during_restatement(
+        self,
+        snapshots: t.List[t.Tuple[SnapshotTableInfo, SnapshotTableInfo]],
+        environment_naming_info: EnvironmentNamingInfo,
+        default_catalog: t.Optional[str] = None,
+    ) -> None:
+        if snapshots:
+            tree = Tree(
+                f"[yellow]The following models had new versions deployed while data was being restated:[/yellow]"
+            )
+
+            for restated_snapshot, updated_snapshot in snapshots:
+                display_name = restated_snapshot.display_name(
+                    environment_naming_info,
+                    default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None,
+                    dialect=self.dialect,
+                )
+                current_branch = tree.add(display_name)
+                current_branch.add(f"restated version: '{restated_snapshot.version}'")
+                current_branch.add(f"currently active version: '{updated_snapshot.version}'")
+
+            self._print(tree)
+            self._print("")  # newline spacer
+
     def log_destructive_change(
         self,
         snapshot_name: str,
@@ -3566,7 +3641,10 @@ class MarkdownConsole(CaptureTerminalConsole):
         msg = f"\nLinter {severity} for `{model._path}`:\n{violations_msg}\n"
 
         self._print(msg)
-        self._errors.append(msg)
+        if is_error:
+            self._errors.append(msg)
+        else:
+            self._warnings.append(msg)
 
     @property
     def captured_warnings(self) -> str:

sqlmesh/core/constants.py

@@ -8,7 +8,7 @@ from pathlib import Path
 
 SQLMESH = "sqlmesh"
 SQLMESH_MANAGED = "sqlmesh_managed"
-SQLMESH_PATH = Path.home() / ".sqlmesh"
+SQLMESH_PATH = Path(os.getenv("SQLMESH_HOME") or Path.home() / ".sqlmesh")
 
 PROD = "prod"
 """Prod"""