sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py

@@ -1,9 +1,9 @@
 """Use the raw SQL when computing the model fingerprint."""
 
 
-def migrate_schemas(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     pass
 
 
-def migrate_rows(state_sync, **kwargs):  # type: ignore
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py

@@ -7,9 +7,7 @@ from sqlglot import exp
 from sqlmesh.utils.migration import index_text_type, blob_text_type
 
 
-def migrate_schemas(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"
@@ -42,11 +40,9 @@ def migrate_schemas(state_sync, **kwargs):  # type: ignore
     engine_adapter.execute(add_fingerprint_exp)
 
 
-def migrate_rows(state_sync, **kwargs):  # type: ignore
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     import pandas as pd
 
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"

sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py

@@ -17,13 +17,11 @@ from sqlmesh.core.console import get_console
 SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
 
 
-def migrate_schemas(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     pass
 
 
-def migrate_rows(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     snapshots_table = "_snapshots"
     if schema:
         snapshots_table = f"{schema}.{snapshots_table}"

sqlmesh/migrations/v0096_remove_plan_dags_table.py

@@ -1,9 +1,7 @@
 """Remove the obsolete _plan_dags table."""
 
 
-def migrate_schemas(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     plan_dags_table = "_plan_dags"
     if schema:
         plan_dags_table = f"{schema}.{plan_dags_table}"
@@ -11,5 +9,5 @@ def migrate_schemas(state_sync, **kwargs):  # type: ignore
     engine_adapter.drop_table(plan_dags_table)
 
 
-def migrate_rows(state_sync, **kwargs):  # type: ignore
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0097_add_dbt_name_in_node.py

@@ -1,9 +1,9 @@
 """Add 'dbt_name' property to node definition."""
 
 
-def migrate_schemas(state_sync, **kwargs):  # type: ignore
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
     pass
 
 
-def migrate_rows(state_sync, **kwargs):  # type: ignore
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
     pass

sqlmesh/migrations/v0098_add_dbt_node_info_in_node.py

@@ -0,0 +1,103 @@
+"""Replace 'dbt_name' with 'dbt_node_info' in the snapshot definition"""
+
+import json
+from sqlglot import exp
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    import pandas as pd
+
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    index_type = index_text_type(engine_adapter.dialect)
+    blob_type = blob_text_type(engine_adapter.dialect)
+
+    new_snapshots = []
+    migration_needed = False
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+        forward_only,
+        dev_version,
+        fingerprint,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+            "forward_only",
+            "dev_version",
+            "fingerprint",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        if dbt_name := parsed_snapshot["node"].get("dbt_name"):
+            parsed_snapshot["node"].pop("dbt_name")
+            parsed_snapshot["node"]["dbt_node_info"] = {
+                "unique_id": dbt_name,
+                # these will get populated as metadata-only changes on the next plan
+                "name": "",
+                "fqn": "",
+            }
+            migration_needed = True
+
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": json.dumps(parsed_snapshot),
+                "kind_name": kind_name,
+                "updated_ts": updated_ts,
+                "unpaused_ts": unpaused_ts,
+                "ttl_ms": ttl_ms,
+                "unrestorable": unrestorable,
+                "forward_only": forward_only,
+                "dev_version": dev_version,
+                "fingerprint": fingerprint,
+            }
+        )
+
+    if migration_needed and new_snapshots:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            target_columns_to_types={
+                "name": exp.DataType.build(index_type),
+                "identifier": exp.DataType.build(index_type),
+                "version": exp.DataType.build(index_type),
+                "snapshot": exp.DataType.build(blob_type),
+                "kind_name": exp.DataType.build(index_type),
+                "updated_ts": exp.DataType.build("bigint"),
+                "unpaused_ts": exp.DataType.build("bigint"),
+                "ttl_ms": exp.DataType.build("bigint"),
+                "unrestorable": exp.DataType.build("boolean"),
+                "forward_only": exp.DataType.build("boolean"),
+                "dev_version": exp.DataType.build(index_type),
+                "fingerprint": exp.DataType.build(blob_type),
+            },
+        )

sqlmesh/migrations/v0099_add_last_altered_to_intervals.py

@@ -0,0 +1,25 @@
+"""Add dev version to the intervals table."""
+
+from sqlglot import exp
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    intervals_table = "_intervals"
+    if schema:
+        intervals_table = f"{schema}.{intervals_table}"
+
+    alter_table_exp = exp.Alter(
+        this=exp.to_table(intervals_table),
+        kind="TABLE",
+        actions=[
+            exp.ColumnDef(
+                this=exp.to_column("last_altered_ts"),
+                kind=exp.DataType.build("BIGINT", dialect=engine_adapter.dialect),
+            )
+        ],
+    )
+    engine_adapter.execute(alter_table_exp)
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py

@@ -0,0 +1,9 @@
+"""Add grants and grants_target_layer to incremental model metadata hash."""
+
+
+def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
+    pass
+
+
+def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
+    pass

sqlmesh/utils/__init__.py

@@ -21,6 +21,7 @@ from enum import IntEnum, Enum
 from functools import lru_cache, reduce, wraps
 from pathlib import Path
 
+import unicodedata
 from sqlglot import exp
 from sqlglot.dialects.dialect import Dialects
 
@@ -291,8 +292,14 @@ def sqlglot_dialects() -> str:
 
 NON_ALNUM = re.compile(r"[^a-zA-Z0-9_]")
 
+NON_ALUM_INCLUDE_UNICODE = re.compile(r"\W", flags=re.UNICODE)
 
-def sanitize_name(name: str) -> str:
+
+def sanitize_name(name: str, *, include_unicode: bool = False) -> str:
+    if include_unicode:
+        s = unicodedata.normalize("NFC", name)
+        s = NON_ALUM_INCLUDE_UNICODE.sub("_", s)
+        return s
     return NON_ALNUM.sub("_", name)
 
 

sqlmesh/utils/cache.py

@@ -59,6 +59,10 @@ class FileCache(t.Generic[T]):
         threshold = to_datetime("1 week ago").timestamp()
         # delete all old cache files
         for file in self._path.glob("*"):
+            if IS_WINDOWS:
+                # the file.stat() call below will fail on windows if the :file name is longer than 260 chars
+                file = fix_windows_path(file)
+
             if not file.stem.startswith(self._cache_version) or file.stat().st_atime < threshold:
                 file.unlink(missing_ok=True)
 
@@ -133,7 +137,7 @@ class FileCache(t.Generic[T]):
 
     def _cache_entry_path(self, name: str, entry_id: str = "") -> Path:
         entry_file_name = "__".join(p for p in (self._cache_version, name, entry_id) if p)
-        full_path = self._path / sanitize_name(entry_file_name)
+        full_path = self._path / sanitize_name(entry_file_name, include_unicode=True)
         if IS_WINDOWS:
             # handle paths longer than 260 chars
             full_path = fix_windows_path(full_path)

sqlmesh/utils/date.py

@@ -444,7 +444,7 @@ def to_time_column(
 
 
 def pandas_timestamp_to_pydatetime(
-    df: pd.DataFrame, columns_to_types: t.Optional[t.Dict[str, exp.DataType]]
+    df: pd.DataFrame, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
 ) -> pd.DataFrame:
     import pandas as pd
     from pandas.api.types import is_datetime64_any_dtype  # type: ignore

sqlmesh/utils/errors.py

@@ -151,6 +151,10 @@ class AdditiveChangeError(SQLMeshError):
     pass
 
 
+class MigrationNotSupportedError(SQLMeshError):
+    pass
+
+
 class NotificationTargetError(SQLMeshError):
     pass
 

sqlmesh/utils/jinja.py

@@ -133,6 +133,12 @@ def find_call_names(node: nodes.Node, vars_in_scope: t.Set[str]) -> t.Iterator[C
     vars_in_scope = vars_in_scope.copy()
     for child_node in node.iter_child_nodes():
         if "target" in child_node.fields:
+            # For nodes with assignment targets (Assign, AssignBlock, For, Import),
+            # the target name could shadow a reference in the right hand side.
+            # So we need to process the RHS before adding the target to scope.
+            # For example: {% set model = model.path %} should track model.path.
+            yield from find_call_names(child_node, vars_in_scope)
+
             target = getattr(child_node, "target")
             if isinstance(target, nodes.Name):
                 vars_in_scope.add(target.name)
@@ -149,7 +155,9 @@ def find_call_names(node: nodes.Node, vars_in_scope: t.Set[str]) -> t.Iterator[C
             name = call_name(child_node)
             if name[0][0] != "'" and name[0] not in vars_in_scope:
                 yield (name, child_node)
-        yield from find_call_names(child_node, vars_in_scope)
+
+        if "target" not in child_node.fields:
+            yield from find_call_names(child_node, vars_in_scope)
 
 
 def extract_call_names(
@@ -206,6 +214,20 @@ def extract_macro_references_and_variables(
     return macro_references, variables
 
 
+def sort_dict_recursive(
+    item: t.Dict[str, t.Any],
+) -> t.Dict[str, t.Any]:
+    sorted_dict: t.Dict[str, t.Any] = {}
+    for k, v in sorted(item.items()):
+        if isinstance(v, list):
+            sorted_dict[k] = sorted(v)
+        elif isinstance(v, dict):
+            sorted_dict[k] = sort_dict_recursive(v)
+        else:
+            sorted_dict[k] = v
+    return sorted_dict
+
+
 JinjaGlobalAttribute = t.Union[str, int, float, bool, AttributeDict]
 
 
@@ -355,6 +377,7 @@ class JinjaMacroRegistry(PydanticModel):
         context.update(builtin_globals)
         context.update(root_macros)
         context.update(package_macros)
+        context["render"] = lambda input: env.from_string(input).render()
 
         env.globals.update(context)
         env.filters.update(self._environment.filters)
@@ -440,7 +463,7 @@ class JinjaMacroRegistry(PydanticModel):
                 d.PythonCode(
                     expressions=[
                         f"{k} = '{v}'" if isinstance(v, str) else f"{k} = {v}"
-                        for k, v in sorted(filtered_objs.items())
+                        for k, v in sort_dict_recursive(filtered_objs).items()
                     ]
                 )
             )

sqlmesh/utils/pydantic.py

@@ -289,13 +289,13 @@ def column_validator(v: t.Any, values: t.Any) -> exp.Column:
     return expression
 
 
-def list_of_columns_or_star_validator(
+def list_of_fields_or_star_validator(
     v: t.Any, values: t.Any
-) -> t.Union[exp.Star, t.List[exp.Column]]:
+) -> t.Union[exp.Star, t.List[exp.Expression]]:
     expressions = _get_fields(v, values)
     if len(expressions) == 1 and isinstance(expressions[0], exp.Star):
         return t.cast(exp.Star, expressions[0])
-    return t.cast(t.List[exp.Column], expressions)
+    return t.cast(t.List[exp.Expression], expressions)
 
 
 def cron_validator(v: t.Any) -> str:
@@ -339,7 +339,7 @@ if t.TYPE_CHECKING:
     SQLGlotPositiveInt = int
     SQLGlotColumn = exp.Column
     SQLGlotListOfFields = t.List[exp.Expression]
-    SQLGlotListOfColumnsOrStar = t.Union[t.List[exp.Column], exp.Star]
+    SQLGlotListOfFieldsOrStar = t.Union[SQLGlotListOfFields, exp.Star]
     SQLGlotCron = str
 else:
     from pydantic.functional_validators import BeforeValidator
@@ -352,7 +352,7 @@ else:
     SQLGlotListOfFields = t.Annotated[
         t.List[exp.Expression], BeforeValidator(list_of_fields_validator)
     ]
-    SQLGlotListOfColumnsOrStar = t.Annotated[
-        t.Union[t.List[exp.Column], exp.Star], BeforeValidator(list_of_columns_or_star_validator)
+    SQLGlotListOfFieldsOrStar = t.Annotated[
+        t.Union[SQLGlotListOfFields, exp.Star], BeforeValidator(list_of_fields_or_star_validator)
     ]
     SQLGlotCron = t.Annotated[str, BeforeValidator(cron_validator)]

sqlmesh/utils/windows.py

@@ -3,12 +3,22 @@ from pathlib import Path
 
 IS_WINDOWS = platform.system() == "Windows"
 
+WINDOWS_LONGPATH_PREFIX = "\\\\?\\"
+
 
 def fix_windows_path(path: Path) -> Path:
     """
     Windows paths are limited to 260 characters: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
     Users can change this by updating a registry entry but we cant rely on that.
-    We can quite commonly generate a cache file path that exceeds 260 characters which causes a FileNotFound error.
-    If we prefix the path with "\\?\" then we can have paths up to 32,767 characters
+
+    SQLMesh quite commonly generates cache file paths that exceed 260 characters and thus cause a FileNotFound error.
+    If we prefix paths with "\\?\" then we can have paths up to 32,767 characters.
+
+    Note that this prefix also means that relative paths no longer work. From the above docs:
+     > Because you cannot use the "\\?\" prefix with a relative path, relative paths are always limited to a total of MAX_PATH characters.
+
+    So we also call path.resolve() to resolve the relative sections so that operations like `path.read_text()` continue to work
     """
-    return Path("\\\\?\\" + str(path.absolute()))
+    if path.parts and not path.parts[0].startswith(WINDOWS_LONGPATH_PREFIX):
+        path = Path(WINDOWS_LONGPATH_PREFIX + str(path.absolute()))
+    return path.resolve()

{sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.217.1.dev1
+Version: 0.227.2.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -235,7 +235,7 @@ Requires-Dist: python-dotenv
 Requires-Dist: requests
 Requires-Dist: rich[jupyter]
 Requires-Dist: ruamel.yaml
-Requires-Dist: sqlglot[rs]~=27.13.2
+Requires-Dist: sqlglot[rs]~=27.28.0
 Requires-Dist: tenacity
 Requires-Dist: time-machine
 Requires-Dist: json-stream
@@ -258,7 +258,7 @@ Provides-Extra: dev
 Requires-Dist: agate; extra == "dev"
 Requires-Dist: beautifulsoup4; extra == "dev"
 Requires-Dist: clickhouse-connect; extra == "dev"
-Requires-Dist: cryptography; extra == "dev"
+Requires-Dist: cryptography<46.0.0; extra == "dev"
 Requires-Dist: databricks-sql-connector; extra == "dev"
 Requires-Dist: dbt-bigquery; extra == "dev"
 Requires-Dist: dbt-core; extra == "dev"
@@ -275,6 +275,7 @@ Requires-Dist: google-cloud-bigquery; extra == "dev"
 Requires-Dist: google-cloud-bigquery-storage; extra == "dev"
 Requires-Dist: httpx; extra == "dev"
 Requires-Dist: mypy~=1.13.0; extra == "dev"
+Requires-Dist: numpy; extra == "dev"
 Requires-Dist: pandas-stubs; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: psycopg2-binary; extra == "dev"
@@ -330,7 +331,7 @@ Requires-Dist: redshift_connector; extra == "redshift"
 Provides-Extra: slack
 Requires-Dist: slack_sdk; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: cryptography; extra == "snowflake"
+Requires-Dist: cryptography<46.0.0; extra == "snowflake"
 Requires-Dist: snowflake-connector-python[pandas,secure-local-storage]; extra == "snowflake"
 Requires-Dist: snowflake-snowpark-python; extra == "snowflake"
 Provides-Extra: trino
@@ -344,7 +345,6 @@ Requires-Dist: pyarrow; extra == "web"
 Provides-Extra: lsp
 Requires-Dist: fastapi==0.115.5; extra == "lsp"
 Requires-Dist: watchfiles>=0.19.0; extra == "lsp"
-Requires-Dist: uvicorn[standard]==0.22.0; extra == "lsp"
 Requires-Dist: sse-starlette>=0.2.2; extra == "lsp"
 Requires-Dist: pyarrow; extra == "lsp"
 Requires-Dist: pygls<2.0.0,>=1.2.0; extra == "lsp"