sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py

@@ -35,9 +35,11 @@ SQLMESH_BLUEPRINT_VARS = "__sqlmesh__blueprint__vars__"
 METADATA_HASH_EXPRESSIONS = {"on_virtual_update", "audits", "signals", "audit_definitions"}
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"

sqlmesh/migrations/v0089_add_virtual_environment_mode.py

@@ -1,5 +1,9 @@
 """Add virtual_environment_mode to the model definition."""
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0090_add_forward_only_column.py

@@ -7,11 +7,7 @@ from sqlglot import exp
 from sqlmesh.utils.migration import index_text_type, blob_text_type
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"
@@ -28,6 +24,14 @@ def migrate(state_sync, **kwargs):  # type: ignore
     )
     engine_adapter.execute(alter_table_exp)
 
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    import pandas as pd
+
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
     new_snapshots = []
 
     for (

sqlmesh/migrations/v0091_on_additive_change.py

@@ -1,5 +1,9 @@
 """Add on_additive_change to incremental model metadata hash."""
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py

@@ -17,9 +17,11 @@ from sqlmesh.core.console import get_console
 SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"

sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py

@@ -1,5 +1,9 @@
 """Use the raw SQL when computing the model fingerprint."""
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py

@@ -0,0 +1,123 @@
+"""Add dev_version and fingerprint columns to the snapshots table."""
+
+import json
+
+from sqlglot import exp
+
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    index_type = index_text_type(engine_adapter.dialect)
+    blob_type = blob_text_type(engine_adapter.dialect)
+
+    add_dev_version_exp = exp.Alter(
+        this=exp.to_table(snapshots_table),
+        kind="TABLE",
+        actions=[
+            exp.ColumnDef(
+                this=exp.to_column("dev_version"),
+                kind=exp.DataType.build(index_type),
+            )
+        ],
+    )
+    engine_adapter.execute(add_dev_version_exp)
+
+    add_fingerprint_exp = exp.Alter(
+        this=exp.to_table(snapshots_table),
+        kind="TABLE",
+        actions=[
+            exp.ColumnDef(
+                this=exp.to_column("fingerprint"),
+                kind=exp.DataType.build(blob_type),
+            )
+        ],
+    )
+    engine_adapter.execute(add_fingerprint_exp)
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    import pandas as pd
+
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    index_type = index_text_type(engine_adapter.dialect)
+    blob_type = blob_text_type(engine_adapter.dialect)
+
+    new_snapshots = []
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+        forward_only,
+        _,
+        _,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+            "forward_only",
+            "dev_version",
+            "fingerprint",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": snapshot,
+                "kind_name": kind_name,
+                "updated_ts": updated_ts,
+                "unpaused_ts": unpaused_ts,
+                "ttl_ms": ttl_ms,
+                "unrestorable": unrestorable,
+                "forward_only": forward_only,
+                "dev_version": parsed_snapshot.get("dev_version"),
+                "fingerprint": json.dumps(parsed_snapshot.get("fingerprint")),
+            }
+        )
+
+    if new_snapshots:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            target_columns_to_types={
+                "name": exp.DataType.build(index_type),
+                "identifier": exp.DataType.build(index_type),
+                "version": exp.DataType.build(index_type),
+                "snapshot": exp.DataType.build(blob_type),
+                "kind_name": exp.DataType.build(index_type),
+                "updated_ts": exp.DataType.build("bigint"),
+                "unpaused_ts": exp.DataType.build("bigint"),
+                "ttl_ms": exp.DataType.build("bigint"),
+                "unrestorable": exp.DataType.build("boolean"),
+                "forward_only": exp.DataType.build("boolean"),
+                "dev_version": exp.DataType.build(index_type),
+                "fingerprint": exp.DataType.build(blob_type),
+            },
+        )

sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py

@@ -0,0 +1,49 @@
+"""
+Warns dbt users about potential diffs due to inclusion of {{ config(...) }} blocks in model SQL.
+
+Prior to this fix, SQLMesh wasn't including the {{ config(...) }} block in the model's SQL payload
+when processing dbt models. Now these config blocks are properly included in the raw SQL, which
+may cause diffs to appear for existing dbt models even though the actual SQL logic hasn't changed.
+
+This is a one-time diff that will appear after upgrading, and applying a plan will resolve it.
+"""
+
+import json
+
+from sqlglot import exp
+
+from sqlmesh.core.console import get_console
+
+SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    warning = (
+        "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
+        "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
+        "command. Please run `sqlmesh diff prod` after the migration has completed, before making any new "
+        "changes. If any unexpected changes are reported, consider running a forward-only plan to apply these "
+        "changes and avoid unnecessary backfills: sqlmesh plan prod --forward-only. "
+        "See https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans for more details.\n"
+    )
+
+    for (snapshot,) in engine_adapter.fetchall(
+        exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+
+        jinja_macros = node.get("jinja_macros") or {}
+        create_builtins_module = jinja_macros.get("create_builtins_module") or ""
+
+        if create_builtins_module == SQLMESH_DBT_PACKAGE:
+            get_console().log_warning(warning)
+            return

sqlmesh/migrations/v0096_remove_plan_dags_table.py

@@ -0,0 +1,13 @@
+"""Remove the obsolete _plan_dags table."""
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    plan_dags_table = "_plan_dags"
+    if schema:
+        plan_dags_table = f"{schema}.{plan_dags_table}"
+
+    engine_adapter.drop_table(plan_dags_table)
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/migrations/v0097_add_dbt_name_in_node.py

@@ -0,0 +1,9 @@
+"""Add 'dbt_name' property to node definition."""
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py}

@@ -1,23 +1,26 @@
-"""Move audits from snapshots to models."""
+"""Replace 'dbt_name' with 'dbt_node_info' in the snapshot definition"""
 
 import json
-
 from sqlglot import exp
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
 
-from sqlmesh.utils.migration import index_text_type
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
 
 
-def migrate(state_sync, **kwargs):  # type: ignore
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     import pandas as pd
 
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
     snapshots_table = "_snapshots"
-    index_type = index_text_type(engine_adapter.dialect)
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"
 
+    index_type = index_text_type(engine_adapter.dialect)
+    blob_type = blob_text_type(engine_adapter.dialect)
+
     new_snapshots = []
+    migration_needed = False
 
     for (
         name,
@@ -29,6 +32,9 @@ def migrate(state_sync, **kwargs):  # type: ignore
         unpaused_ts,
         ttl_ms,
         unrestorable,
+        forward_only,
+        dev_version,
+        fingerprint,
     ) in engine_adapter.fetchall(
         exp.select(
             "name",
@@ -40,17 +46,22 @@ def migrate(state_sync, **kwargs):  # type: ignore
             "unpaused_ts",
             "ttl_ms",
             "unrestorable",
+            "forward_only",
+            "dev_version",
+            "fingerprint",
         ).from_(snapshots_table),
         quote_identifiers=True,
     ):
         parsed_snapshot = json.loads(snapshot)
-
-        audit_definitions = parsed_snapshot.pop("audits", [])
-        node = parsed_snapshot["node"]
-        node.pop("inline_audits", None)
-
-        if audit_definitions:
-            node["audit_definitions"] = {audit["name"]: audit for audit in audit_definitions}
+        if dbt_name := parsed_snapshot["node"].get("dbt_name"):
+            parsed_snapshot["node"].pop("dbt_name")
+            parsed_snapshot["node"]["dbt_node_info"] = {
+                "unique_id": dbt_name,
+                # these will get populated as metadata-only changes on the next plan
+                "name": "",
+                "fqn": "",
+            }
+            migration_needed = True
 
         new_snapshots.append(
             {
@@ -63,10 +74,13 @@ def migrate(state_sync, **kwargs):  # type: ignore
                 "unpaused_ts": unpaused_ts,
                 "ttl_ms": ttl_ms,
                 "unrestorable": unrestorable,
+                "forward_only": forward_only,
+                "dev_version": dev_version,
+                "fingerprint": fingerprint,
             }
         )
 
-    if new_snapshots:
+    if migration_needed and new_snapshots:
         engine_adapter.delete_from(snapshots_table, "TRUE")
 
         engine_adapter.insert_append(
@@ -76,11 +90,14 @@ def migrate(state_sync, **kwargs):  # type: ignore
                 "name": exp.DataType.build(index_type),
                 "identifier": exp.DataType.build(index_type),
                 "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
+                "snapshot": exp.DataType.build(blob_type),
                 "kind_name": exp.DataType.build(index_type),
                 "updated_ts": exp.DataType.build("bigint"),
                 "unpaused_ts": exp.DataType.build("bigint"),
                 "ttl_ms": exp.DataType.build("bigint"),
                 "unrestorable": exp.DataType.build("boolean"),
+                "forward_only": exp.DataType.build("boolean"),
+                "dev_version": exp.DataType.build(index_type),
+                "fingerprint": exp.DataType.build(blob_type),
             },
         )

sqlmesh/migrations/v0099_add_last_altered_to_intervals.py

@@ -0,0 +1,25 @@
+"""Add dev version to the intervals table."""
+
+from sqlglot import exp
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    intervals_table = "_intervals"
+    if schema:
+        intervals_table = f"{schema}.{intervals_table}"
+
+    alter_table_exp = exp.Alter(
+        this=exp.to_table(intervals_table),
+        kind="TABLE",
+        actions=[
+            exp.ColumnDef(
+                this=exp.to_column("last_altered_ts"),
+                kind=exp.DataType.build("BIGINT", dialect=engine_adapter.dialect),
+            )
+        ],
+    )
+    engine_adapter.execute(alter_table_exp)
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py

@@ -0,0 +1,9 @@
+"""Add grants and grants_target_layer to incremental model metadata hash."""
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/utils/__init__.py

@@ -21,6 +21,7 @@ from enum import IntEnum, Enum
 from functools import lru_cache, reduce, wraps
 from pathlib import Path
 
+import unicodedata
 from sqlglot import exp
 from sqlglot.dialects.dialect import Dialects
 
@@ -291,8 +292,14 @@ def sqlglot_dialects() -> str:
 
 NON_ALNUM = re.compile(r"[^a-zA-Z0-9_]")
 
+NON_ALUM_INCLUDE_UNICODE = re.compile(r"\W", flags=re.UNICODE)
 
-def sanitize_name(name: str) -> str:
+
+def sanitize_name(name: str, *, include_unicode: bool = False) -> str:
+    if include_unicode:
+        s = unicodedata.normalize("NFC", name)
+        s = NON_ALUM_INCLUDE_UNICODE.sub("_", s)
+        return s
     return NON_ALNUM.sub("_", name)
 
 

sqlmesh/utils/cache.py

@@ -59,6 +59,10 @@ class FileCache(t.Generic[T]):
         threshold = to_datetime("1 week ago").timestamp()
         # delete all old cache files
         for file in self._path.glob("*"):
+            if IS_WINDOWS:
+                # the file.stat() call below will fail on windows if the :file name is longer than 260 chars
+                file = fix_windows_path(file)
+
             if not file.stem.startswith(self._cache_version) or file.stat().st_atime < threshold:
                 file.unlink(missing_ok=True)
 
@@ -133,7 +137,7 @@ class FileCache(t.Generic[T]):
 
     def _cache_entry_path(self, name: str, entry_id: str = "") -> Path:
         entry_file_name = "__".join(p for p in (self._cache_version, name, entry_id) if p)
-        full_path = self._path / sanitize_name(entry_file_name)
+        full_path = self._path / sanitize_name(entry_file_name, include_unicode=True)
         if IS_WINDOWS:
             # handle paths longer than 260 chars
             full_path = fix_windows_path(full_path)

sqlmesh/utils/connection_pool.py

@@ -227,7 +227,8 @@ class ThreadLocalConnectionPool(_ThreadLocalBase):
                     self._thread_connections.pop(thread_id)
                     self._thread_cursors.pop(thread_id, None)
                     self._discard_transaction(thread_id)
-                self._thread_attributes.pop(thread_id, None)
+
+            self._thread_attributes.clear()
 
 
 class ThreadLocalSharedConnectionPool(_ThreadLocalBase):

sqlmesh/utils/dag.py

@@ -99,6 +99,53 @@ class DAG(t.Generic[T]):
 
         return self._upstream[node]
 
+    def _find_cycle_path(self, nodes_in_cycle: t.Dict[T, t.Set[T]]) -> t.Optional[t.List[T]]:
+        """Find the exact cycle path using DFS when a cycle is detected.
+
+        Args:
+            nodes_in_cycle: Dictionary of nodes that are part of the cycle and their dependencies
+
+        Returns:
+            List of nodes forming the cycle path, or None if no cycle found
+        """
+        if not nodes_in_cycle:
+            return None
+
+        # Use DFS to find a cycle path
+        visited: t.Set[T] = set()
+        path: t.List[T] = []
+
+        def dfs(node: T) -> t.Optional[t.List[T]]:
+            if node in path:
+                # Found a cycle - extract the cycle path
+                cycle_start = path.index(node)
+                return path[cycle_start:] + [node]
+
+            if node in visited:
+                return None
+
+            visited.add(node)
+            path.append(node)
+
+            # Only follow edges to nodes that are still in the unprocessed set
+            for neighbor in nodes_in_cycle.get(node, set()):
+                if neighbor in nodes_in_cycle:
+                    cycle = dfs(neighbor)
+                    if cycle:
+                        return cycle
+
+            path.pop()
+            return None
+
+        # Try starting DFS from each unvisited node
+        for start_node in nodes_in_cycle:
+            if start_node not in visited:
+                cycle = dfs(start_node)
+                if cycle:
+                    return cycle[:-1]  # Remove the duplicate node at the end
+
+        return None
+
     @property
     def roots(self) -> t.Set[T]:
         """Returns all nodes in the graph without any upstream dependencies."""
@@ -125,23 +172,31 @@ class DAG(t.Generic[T]):
                 next_nodes = {node for node, deps in unprocessed_nodes.items() if not deps}
 
                 if not next_nodes:
-                    # Sort cycle candidates to make the order deterministic
-                    cycle_candidates_msg = (
-                        "\nPossible candidates to check for circular references: "
-                        + ", ".join(str(node) for node in sorted(cycle_candidates))
-                    )
+                    # A cycle was detected - find the exact cycle path
+                    cycle_path = self._find_cycle_path(unprocessed_nodes)
 
-                    if last_processed_nodes:
-                        last_processed_msg = "\nLast nodes added to the DAG: " + ", ".join(
-                            str(node) for node in last_processed_nodes
+                    last_processed_msg = ""
+                    if cycle_path:
+                        node_output = " ->\n".join(
+                            str(node) for node in (cycle_path + [cycle_path[0]])
                         )
+                        cycle_msg = f"\nCycle:\n{node_output}"
                     else:
-                        last_processed_msg = ""
+                        # Fallback message in case a cycle can't be found
+                        cycle_candidates_msg = (
+                            "\nPossible candidates to check for circular references: "
+                            + ", ".join(str(node) for node in sorted(cycle_candidates))
+                        )
+                        cycle_msg = cycle_candidates_msg
+                        if last_processed_nodes:
+                            last_processed_msg = "\nLast nodes added to the DAG: " + ", ".join(
+                                str(node) for node in last_processed_nodes
+                            )
 
                     raise SQLMeshError(
                         "Detected a cycle in the DAG. "
                         "Please make sure there are no circular references between nodes."
-                        f"{last_processed_msg}{cycle_candidates_msg}"
+                        f"{last_processed_msg}{cycle_msg}"
                     )
 
                 for node in next_nodes:

sqlmesh/utils/date.py

@@ -343,6 +343,13 @@ def make_exclusive(time: TimeLike) -> datetime:
     return dt
 
 
+def make_ts_exclusive(time: TimeLike, dialect: DialectType) -> datetime:
+    ts = to_datetime(time)
+    if dialect == "tsql":
+        return to_utc_timestamp(ts) - pd.Timedelta(1, unit="ns")
+    return ts + timedelta(microseconds=1)
+
+
 def to_utc_timestamp(time: datetime) -> pd.Timestamp:
     import pandas as pd
 
@@ -437,7 +444,7 @@ def to_time_column(
 
 
 def pandas_timestamp_to_pydatetime(
-    df: pd.DataFrame, columns_to_types: t.Optional[t.Dict[str, exp.DataType]]
+    df: pd.DataFrame, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
 ) -> pd.DataFrame:
     import pandas as pd
     from pandas.api.types import is_datetime64_any_dtype  # type: ignore

sqlmesh/utils/errors.py

@@ -86,6 +86,10 @@ class AuditConfigError(ConfigError):
     pass
 
 
+class StateMigrationError(SQLMeshError):
+    pass
+
+
 class AuditError(SQLMeshError):
     def __init__(
         self,
@@ -147,6 +151,10 @@ class AdditiveChangeError(SQLMeshError):
     pass
 
 
+class MigrationNotSupportedError(SQLMeshError):
+    pass
+
+
 class NotificationTargetError(SQLMeshError):
     pass