sqlmesh 0.206.2.dev2__py3-none-any.whl → 0.206.2.dev4__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.206.2.dev2'
-__version_tuple__ = version_tuple = (0, 206, 2, 'dev2')
+__version__ = version = '0.206.2.dev4'
+__version_tuple__ = version_tuple = (0, 206, 2, 'dev4')

sqlmesh/core/dialect.py

@@ -13,6 +13,7 @@ from functools import lru_cache
 from sqlglot import Dialect, Generator, ParseError, Parser, Tokenizer, TokenType, exp
 from sqlglot.dialects.dialect import DialectType
 from sqlglot.dialects import DuckDB, Snowflake
+import sqlglot.dialects.athena as athena
 from sqlglot.helper import seq_get
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 from sqlglot.optimizer.qualify_columns import quote_identifiers
@@ -1014,6 +1015,14 @@ def extend_sqlglot() -> None:
     generators = {Generator}
 
     for dialect in Dialect.classes.values():
+        # Athena picks a different Tokenizer / Parser / Generator depending on the query
+        # so this ensures that the extra ones it defines are also extended
+        if dialect == athena.Athena:
+            tokenizers.add(athena._TrinoTokenizer)
+            parsers.add(athena._TrinoParser)
+            generators.add(athena._TrinoGenerator)
+            generators.add(athena._HiveGenerator)
+
         if hasattr(dialect, "Tokenizer"):
             tokenizers.add(dialect.Tokenizer)
         if hasattr(dialect, "Parser"):

sqlmesh/core/model/common.py

@@ -157,7 +157,7 @@ def _add_variables_to_python_env(
 
     if blueprint_variables:
         blueprint_variables = {
-            k: SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
+            k.lower(): SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
             for k, v in blueprint_variables.items()
         }
         python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(

sqlmesh/migrations/v0087_normalize_blueprint_variables.py

@@ -0,0 +1,138 @@
+"""
+Normalizes blueprint variables, so Customer_Field is stored as customer_field in the `python_env`:
+
+MODEL (
+  ...
+  blueprints (
+    Customer_Field := 1
+  )
+);
+
+SELECT
+  @customer_field AS col
+"""
+
+import json
+import logging
+from dataclasses import dataclass
+
+from sqlglot import exp
+from sqlmesh.core.console import get_console
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
+
+logger = logging.getLogger(__name__)
+
+
+SQLMESH_BLUEPRINT_VARS = "__sqlmesh__blueprint__vars__"
+
+
+# Make sure `SqlValue` is defined so it can be used by `eval` call in the migration
+@dataclass
+class SqlValue:
+    """A SQL string representing a generated SQLGlot AST."""
+
+    sql: str
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    import pandas as pd
+
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    migration_needed = False
+    new_snapshots = []
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+        python_env = node.get("python_env") or {}
+
+        migrate_snapshot = False
+
+        if blueprint_vars_executable := python_env.get(SQLMESH_BLUEPRINT_VARS):
+            blueprint_vars = eval(blueprint_vars_executable["payload"])
+
+            for var, value in dict(blueprint_vars).items():
+                lowercase_var = var.lower()
+                if var != lowercase_var:
+                    if lowercase_var in blueprint_vars:
+                        get_console().log_warning(
+                            "SQLMesh is unable to fully migrate the state database, because the "
+                            f"model '{node['name']}' contains two blueprint variables ('{var}' and "
+                            f"'{lowercase_var}') that resolve to the same value ('{lowercase_var}'). "
+                            "This may result in unexpected changes being reported by the next "
+                            "`sqlmesh plan` command. If this happens, consider renaming either variable, "
+                            "so that the lowercase version of their names are different."
+                        )
+                    else:
+                        del blueprint_vars[var]
+                        blueprint_vars[lowercase_var] = value
+                        migrate_snapshot = True
+
+            if migrate_snapshot:
+                migration_needed = True
+                blueprint_vars_executable["payload"] = repr(blueprint_vars)
+
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": json.dumps(parsed_snapshot),
+                "kind_name": kind_name,
+                "updated_ts": updated_ts,
+                "unpaused_ts": unpaused_ts,
+                "ttl_ms": ttl_ms,
+                "unrestorable": unrestorable,
+            }
+        )
+
+    if migration_needed and new_snapshots:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        index_type = index_text_type(engine_adapter.dialect)
+        blob_type = blob_text_type(engine_adapter.dialect)
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            columns_to_types={
+                "name": exp.DataType.build(index_type),
+                "identifier": exp.DataType.build(index_type),
+                "version": exp.DataType.build(index_type),
+                "snapshot": exp.DataType.build(blob_type),
+                "kind_name": exp.DataType.build("text"),
+                "updated_ts": exp.DataType.build("bigint"),
+                "unpaused_ts": exp.DataType.build("bigint"),
+                "ttl_ms": exp.DataType.build("bigint"),
+                "unrestorable": exp.DataType.build("boolean"),
+            },
+        )

{sqlmesh-0.206.2.dev2.dist-info → sqlmesh-0.206.2.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.206.2.dev2
+Version: 0.206.2.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.206.2.dev2.dist-info → sqlmesh-0.206.2.dev4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=ODmuGfXnsHOG7yOzSoA-XJjd_1OkCpWgVFsH_K0ggDM,528
+sqlmesh/_version.py,sha256=UjOy4yonGrNWHc4ceruDSYcy8OBbaR2th5VIqKYuU8E,528
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,7 +15,7 @@ sqlmesh/core/console.py,sha256=ovjM75l1A2rAi-ud3ovsIKSGwF9TsGYTK8NvokbCdL4,16611
 sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
 sqlmesh/core/context.py,sha256=SkTNrp9TUNx469_mWkemhMk4leGDtmblBIWE7S64sk8,127598
 sqlmesh/core/context_diff.py,sha256=_Ur55fGLJ58Icsp44I4pVhdi7_JHiHq45mWIco9oFzk,21792
-sqlmesh/core/dialect.py,sha256=ELBtB8ft0OyalhgqQ1CUnn26xJxLDah2SP1v8kh_FB8,50903
+sqlmesh/core/dialect.py,sha256=lJHcE7D6GvYs1tzbexJb1mi4gltsMUzHeOrhN4ObIdo,51343
 sqlmesh/core/environment.py,sha256=odus40R-YzdQdS_IiUq_tFouUXIhnpwmNwo8dE8Aw1k,13041
 sqlmesh/core/lineage.py,sha256=zjB0Zfamo2Fja2r5SSZPMbrEKTXr1WjozZPVqvxdybI,3143
 sqlmesh/core/loader.py,sha256=N9KPfa-R4BCId7BCLujMkIKwXgxnftC-dD0rP0cpv1w,41088
@@ -86,7 +86,7 @@ sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
-sqlmesh/core/model/common.py,sha256=mUVJZ1otbcY9IPqeJQWxZOgv_2QnYEEKdFwcPvZGe9c,17961
+sqlmesh/core/model/common.py,sha256=Gt1nBdbILfR52Rhq2KxF0EMGkB3J5zfk6UH7ecLpMZI,17969
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
 sqlmesh/core/model/definition.py,sha256=UsRFDlO3A5so4HM2_x6M0sdTR-VlTIxw1d-T_gxS_9Q,113897
 sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
@@ -263,6 +263,7 @@ sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py,sha256=0r-S
 sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py,sha256=w1TbhvB98czuq5VZxsfcQE2w7o-nd96aKEIG-M1dn6A,329
 sqlmesh/migrations/v0085_deterministic_repr.py,sha256=mkbZV0rzXJ5bmN1Y75kFI9T6gbnx_sy-zn_UzbOL1Sg,4265
 sqlmesh/migrations/v0086_check_deterministic_bug.py,sha256=WiYcmhQD434GXWiiakQtcy9ESmB35spXMltV43nvU7Q,2953
+sqlmesh/migrations/v0087_normalize_blueprint_variables.py,sha256=GI6KKA-6PtONUoOnyvLpRTUmPjbpa-CKhDa0iqJW01U,4421
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
@@ -287,7 +288,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.206.2.dev2.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.206.2.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=SXykdSRNgrT1k6I2dPZPEfpxPjIsi8-yVFijt-naSA4,5472
 web/client/dist/assets/Audits-Jm353fhk.js,sha256=ry6b5IKT3hLAfD5C3T2BqxulfauEV0n0YZ2vsP4eQ_U,1041
@@ -405,8 +406,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.206.2.dev2.dist-info/METADATA,sha256=qajqCyJXBIv3CUyhI_xZDP41Mo7zRlMflgxEW0Hs1fs,26685
-sqlmesh-0.206.2.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.206.2.dev2.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.206.2.dev2.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.206.2.dev2.dist-info/RECORD,,
+sqlmesh-0.206.2.dev4.dist-info/METADATA,sha256=PoP8Fc1vGMvf1E7QwezSWn_BL8ZUVCRuTJjdIVu6lcU,26685
+sqlmesh-0.206.2.dev4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.206.2.dev4.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.206.2.dev4.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.206.2.dev4.dist-info/RECORD,,