sqlmesh 0.206.2.dev1__py3-none-any.whl → 0.206.2.dev3__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.206.2.dev1'
-__version_tuple__ = version_tuple = (0, 206, 2, 'dev1')
+__version__ = version = '0.206.2.dev3'
+__version_tuple__ = version_tuple = (0, 206, 2, 'dev3')

sqlmesh/core/linter/rule.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import abc
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 
 from sqlmesh.core.model import Model
@@ -49,12 +49,21 @@ class TextEdit:
     new_text: str
 
 
+@dataclass(frozen=True)
+class CreateFile:
+    """Create a new file with the provided text."""
+
+    path: Path
+    text: str
+
+
 @dataclass(frozen=True)
 class Fix:
     """A fix that can be applied to resolve a rule violation."""
 
     title: str
-    edits: t.List[TextEdit]
+    edits: t.List[TextEdit] = field(default_factory=list)
+    create_files: t.List[CreateFile] = field(default_factory=list)
 
 
 class _Rule(abc.ABCMeta):

sqlmesh/core/linter/rules/builtin.py

@@ -14,7 +14,15 @@ from sqlmesh.core.linter.helpers import (
     get_range_of_model_block,
     read_range_from_string,
 )
-from sqlmesh.core.linter.rule import Rule, RuleViolation, Range, Fix, TextEdit, Position
+from sqlmesh.core.linter.rule import (
+    Rule,
+    RuleViolation,
+    Range,
+    Fix,
+    TextEdit,
+    Position,
+    CreateFile,
+)
 from sqlmesh.core.linter.definition import RuleSet
 from sqlmesh.core.model import Model, SqlModel, ExternalModel
 from sqlmesh.utils.lineage import extract_references_from_query, ExternalModelReference
@@ -227,7 +235,16 @@ class NoMissingExternalModels(Rule):
 
         external_models_path = root / EXTERNAL_MODELS_YAML
         if not external_models_path.exists():
-            return None
+            return Fix(
+                title="Add external model file",
+                edits=[],
+                create_files=[
+                    CreateFile(
+                        path=external_models_path,
+                        text=f"- name: '{model_name}'\n",
+                    )
+                ],
+            )
 
         # Figure out the position to insert the new external model at the end of the file, whether
         # needs new line or not.

sqlmesh/core/model/common.py

@@ -157,7 +157,7 @@ def _add_variables_to_python_env(
 
     if blueprint_variables:
         blueprint_variables = {
-            k: SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
+            k.lower(): SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
             for k, v in blueprint_variables.items()
         }
         python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(

sqlmesh/lsp/context.py

@@ -273,13 +273,41 @@ class LSPContext:
             if found_violation is not None and found_violation.fixes:
                 # Create code actions for each fix
                 for fix in found_violation.fixes:
-                    # Convert our Fix to LSP TextEdits
                     changes: t.Dict[str, t.List[types.TextEdit]] = {}
+                    document_changes: t.List[
+                        t.Union[
+                            types.TextDocumentEdit,
+                            types.CreateFile,
+                            types.RenameFile,
+                            types.DeleteFile,
+                        ]
+                    ] = []
+
+                    for create in fix.create_files:
+                        create_uri = URI.from_path(create.path).value
+                        document_changes.append(types.CreateFile(uri=create_uri))
+                        document_changes.append(
+                            types.TextDocumentEdit(
+                                text_document=types.OptionalVersionedTextDocumentIdentifier(
+                                    uri=create_uri,
+                                    version=None,
+                                ),
+                                edits=[
+                                    types.TextEdit(
+                                        range=types.Range(
+                                            start=types.Position(line=0, character=0),
+                                            end=types.Position(line=0, character=0),
+                                        ),
+                                        new_text=create.text,
+                                    )
+                                ],
+                            )
+                        )
+
                     for edit in fix.edits:
                         uri_key = URI.from_path(edit.path).value
                         if uri_key not in changes:
                             changes[uri_key] = []
-                        # Create a TextEdit for the LSP
                         changes[uri_key].append(
                             types.TextEdit(
                                 range=types.Range(
@@ -296,12 +324,15 @@ class LSPContext:
                             )
                         )
 
-                    # Create the code action
+                    workspace_edit = types.WorkspaceEdit(
+                        changes=changes if changes else None,
+                        document_changes=document_changes if document_changes else None,
+                    )
                     code_action = types.CodeAction(
                         title=fix.title,
                         kind=types.CodeActionKind.QuickFix,
                         diagnostics=[diagnostic],
-                        edit=types.WorkspaceEdit(changes=changes),
+                        edit=workspace_edit,
                     )
                     code_actions.append(code_action)
 

sqlmesh/migrations/v0087_normalize_blueprint_variables.py

@@ -0,0 +1,138 @@
+"""
+Normalizes blueprint variables, so Customer_Field is stored as customer_field in the `python_env`:
+
+MODEL (
+  ...
+  blueprints (
+    Customer_Field := 1
+  )
+);
+
+SELECT
+  @customer_field AS col
+"""
+
+import json
+import logging
+from dataclasses import dataclass
+
+from sqlglot import exp
+from sqlmesh.core.console import get_console
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
+
+logger = logging.getLogger(__name__)
+
+
+SQLMESH_BLUEPRINT_VARS = "__sqlmesh__blueprint__vars__"
+
+
+# Make sure `SqlValue` is defined so it can be used by `eval` call in the migration
+@dataclass
+class SqlValue:
+    """A SQL string representing a generated SQLGlot AST."""
+
+    sql: str
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    import pandas as pd
+
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    migration_needed = False
+    new_snapshots = []
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+        python_env = node.get("python_env") or {}
+
+        migrate_snapshot = False
+
+        if blueprint_vars_executable := python_env.get(SQLMESH_BLUEPRINT_VARS):
+            blueprint_vars = eval(blueprint_vars_executable["payload"])
+
+            for var, value in dict(blueprint_vars).items():
+                lowercase_var = var.lower()
+                if var != lowercase_var:
+                    if lowercase_var in blueprint_vars:
+                        get_console().log_warning(
+                            "SQLMesh is unable to fully migrate the state database, because the "
+                            f"model '{node['name']}' contains two blueprint variables ('{var}' and "
+                            f"'{lowercase_var}') that resolve to the same value ('{lowercase_var}'). "
+                            "This may result in unexpected changes being reported by the next "
+                            "`sqlmesh plan` command. If this happens, consider renaming either variable, "
+                            "so that the lowercase version of their names are different."
+                        )
+                    else:
+                        del blueprint_vars[var]
+                        blueprint_vars[lowercase_var] = value
+                        migrate_snapshot = True
+
+            if migrate_snapshot:
+                migration_needed = True
+                blueprint_vars_executable["payload"] = repr(blueprint_vars)
+
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": json.dumps(parsed_snapshot),
+                "kind_name": kind_name,
+                "updated_ts": updated_ts,
+                "unpaused_ts": unpaused_ts,
+                "ttl_ms": ttl_ms,
+                "unrestorable": unrestorable,
+            }
+        )
+
+    if migration_needed and new_snapshots:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        index_type = index_text_type(engine_adapter.dialect)
+        blob_type = blob_text_type(engine_adapter.dialect)
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            columns_to_types={
+                "name": exp.DataType.build(index_type),
+                "identifier": exp.DataType.build(index_type),
+                "version": exp.DataType.build(index_type),
+                "snapshot": exp.DataType.build(blob_type),
+                "kind_name": exp.DataType.build("text"),
+                "updated_ts": exp.DataType.build("bigint"),
+                "unpaused_ts": exp.DataType.build("bigint"),
+                "ttl_ms": exp.DataType.build("bigint"),
+                "unrestorable": exp.DataType.build("boolean"),
+            },
+        )

{sqlmesh-0.206.2.dev1.dist-info → sqlmesh-0.206.2.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.206.2.dev1
+Version: 0.206.2.dev3
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.206.2.dev1.dist-info → sqlmesh-0.206.2.dev3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=S4s0lANCdCA8rxuKgeIztkdnzaNgM5vUafG7dP6Q4hU,528
+sqlmesh/_version.py,sha256=81qxsWTVH02pMJYGoBBAKEIHMBdl80V7JtlCq12vfoQ,528
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -78,15 +78,15 @@ sqlmesh/core/engine_adapter/trino.py,sha256=Tws3GxqnC6BmXFohqru1TDxsgPH-21qV6Oky
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=JDJTkhByUw4Tr3nF-BOHKFAx_eASQqEqzX6AUVQUz8I,5659
 sqlmesh/core/linter/helpers.py,sha256=Bv11aGRIrnv9Ym9fudy3XQB0k-FvchkLtBTkGI91SEw,6504
-sqlmesh/core/linter/rule.py,sha256=MgFwfvlGEvh4nijsxc7Bbk_S9ctJ_m-nKCom9LIN4W4,3734
+sqlmesh/core/linter/rule.py,sha256=nB3o1rHyN44ZOg5ImICP16SeUHimf-12ObdXJjkTGyM,3964
 sqlmesh/core/linter/rules/__init__.py,sha256=gevzfb67vFqckTCoVAe_TBGf6hQ-YtE1_YuGuXyh1L0,77
-sqlmesh/core/linter/rules/builtin.py,sha256=2H4yV4eVUCY25uFgO2M62czpZt5dXdK5mW6XGGOc12Q,9857
+sqlmesh/core/linter/rules/builtin.py,sha256=BVBoBr1Ru-DA4VPnwfaodZdVq84afCc9qBgPLmwrfiA,10204
 sqlmesh/core/metric/__init__.py,sha256=H1HmoD5IwN4YWe9iJXyueLYNmTQFZwok5nSWNJcZIBQ,237
 sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw9x9M,7201
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
-sqlmesh/core/model/common.py,sha256=mUVJZ1otbcY9IPqeJQWxZOgv_2QnYEEKdFwcPvZGe9c,17961
+sqlmesh/core/model/common.py,sha256=Gt1nBdbILfR52Rhq2KxF0EMGkB3J5zfk6UH7ecLpMZI,17969
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
 sqlmesh/core/model/definition.py,sha256=UsRFDlO3A5so4HM2_x6M0sdTR-VlTIxw1d-T_gxS_9Q,113897
 sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
@@ -166,7 +166,7 @@ sqlmesh/integrations/github/cicd/controller.py,sha256=0I5mHPz3HBJR5lkR-2J9Bj06Yn
 sqlmesh/lsp/api.py,sha256=_lA_buSPZFIoLJuINbS53LbNOOaHp3orb0uFTE3aCqI,2328
 sqlmesh/lsp/commands.py,sha256=7tZPePSH-IwBYmXJPIlqGM7pi4rOCLEtc3fKJglAxZs,72
 sqlmesh/lsp/completions.py,sha256=7Lhboh6xyoMJ3kkHG3aZz1xVbDwKiXeQKdIRj5xlUOA,6674
-sqlmesh/lsp/context.py,sha256=xaKZycAycH3-Ugzyg0dLni3shcClOEo1xq5OV2cxccU,19423
+sqlmesh/lsp/context.py,sha256=VIL_Xl3II6I79ILZ67UTY9u6OzgkLHDTEpTK91DJxAA,20892
 sqlmesh/lsp/custom.py,sha256=3gfN_z9IXKOhTGkaZX7V5I8WUmI4z1GWe1sViOo92pw,3980
 sqlmesh/lsp/errors.py,sha256=3NMim_5J00Eypz7t8b7XbkBfy8gIsRkeq-VcjD4COtc,1489
 sqlmesh/lsp/helpers.py,sha256=EFc1u3-b7kSv5-tNwmKUDxId72RCLDBnN2lLTgRSzzQ,1020
@@ -263,6 +263,7 @@ sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py,sha256=0r-S
 sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py,sha256=w1TbhvB98czuq5VZxsfcQE2w7o-nd96aKEIG-M1dn6A,329
 sqlmesh/migrations/v0085_deterministic_repr.py,sha256=mkbZV0rzXJ5bmN1Y75kFI9T6gbnx_sy-zn_UzbOL1Sg,4265
 sqlmesh/migrations/v0086_check_deterministic_bug.py,sha256=WiYcmhQD434GXWiiakQtcy9ESmB35spXMltV43nvU7Q,2953
+sqlmesh/migrations/v0087_normalize_blueprint_variables.py,sha256=GI6KKA-6PtONUoOnyvLpRTUmPjbpa-CKhDa0iqJW01U,4421
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
@@ -287,7 +288,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.206.2.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.206.2.dev3.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=SXykdSRNgrT1k6I2dPZPEfpxPjIsi8-yVFijt-naSA4,5472
 web/client/dist/assets/Audits-Jm353fhk.js,sha256=ry6b5IKT3hLAfD5C3T2BqxulfauEV0n0YZ2vsP4eQ_U,1041
@@ -405,8 +406,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.206.2.dev1.dist-info/METADATA,sha256=v7lvQZd0_0twHOAcdcG3OnLLxO9Bf45E6gUP6ADSlHM,26685
-sqlmesh-0.206.2.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.206.2.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.206.2.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.206.2.dev1.dist-info/RECORD,,
+sqlmesh-0.206.2.dev3.dist-info/METADATA,sha256=5Y9CyunF4DqlvQ-HoC44EjxiboczgoA4ha-fz_fv3F0,26685
+sqlmesh-0.206.2.dev3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.206.2.dev3.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.206.2.dev3.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.206.2.dev3.dist-info/RECORD,,