sqlmesh 0.201.2.dev1__py3-none-any.whl → 0.201.2.dev3__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.201.2.dev1'
-__version_tuple__ = version_tuple = (0, 201, 2, 'dev1')
+__version__ = version = '0.201.2.dev3'
+__version_tuple__ = version_tuple = (0, 201, 2, 'dev3')

sqlmesh/core/context.py

@@ -2649,13 +2649,16 @@ class GenericContext(BaseContext, t.Generic[C]):
 
     @cached_property
     def engine_adapters(self) -> t.Dict[str, EngineAdapter]:
-        """Returns all the engine adapters for the gateways defined in the configuration."""
+        """Returns all the engine adapters for the gateways defined in the configurations."""
         adapters: t.Dict[str, EngineAdapter] = {self.selected_gateway: self.engine_adapter}
-        for gateway_name in self.config.gateways:
-            if gateway_name != self.selected_gateway:
-                connection = self.config.get_connection(gateway_name)
-                adapter = connection.create_engine_adapter(concurrent_tasks=self.concurrent_tasks)
-                adapters[gateway_name] = adapter
+        for config in self.configs.values():
+            for gateway_name in config.gateways:
+                if gateway_name not in adapters:
+                    connection = config.get_connection(gateway_name)
+                    adapter = connection.create_engine_adapter(
+                        concurrent_tasks=self.concurrent_tasks,
+                    )
+                    adapters[gateway_name] = adapter
         return adapters
 
     @cached_property

sqlmesh/core/model/common.py

@@ -153,14 +153,16 @@ def _add_variables_to_python_env(
 
     variables = {k: v for k, v in (variables or {}).items() if k in used_variables}
     if variables:
-        python_env[c.SQLMESH_VARS] = Executable.value(variables)
+        python_env[c.SQLMESH_VARS] = Executable.value(variables, sort_root_dict=True)
 
     if blueprint_variables:
         blueprint_variables = {
             k: SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
             for k, v in blueprint_variables.items()
         }
-        python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(blueprint_variables)
+        python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(
+            blueprint_variables, sort_root_dict=True
+        )
 
     return python_env
 

sqlmesh/migrations/v0085_deterministic_repr.py

@@ -4,6 +4,7 @@ and therefore this migration applies deterministic sorting to the keys of the di
 """
 
 import json
+import logging
 import typing as t
 from dataclasses import dataclass
 
@@ -12,6 +13,12 @@ from sqlglot import exp
 from sqlmesh.utils.migration import index_text_type, blob_text_type
 
 
+logger = logging.getLogger(__name__)
+
+
+KEYS_TO_MAKE_DETERMINISTIC = ["__sqlmesh__vars__", "__sqlmesh__blueprint__vars__"]
+
+
 # Make sure `SqlValue` is defined so it can be used by `eval` call in the migration
 @dataclass
 class SqlValue:
@@ -20,25 +27,13 @@ class SqlValue:
     sql: str
 
 
-def _deterministic_repr(obj: t.Any) -> str:
-    """
-    This is a copy of the function from utils.metaprogramming
-    """
-
-    def _normalize_for_repr(o: t.Any) -> t.Any:
-        if isinstance(o, dict):
-            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
-            return {k: _normalize_for_repr(v) for k, v in sorted_items}
-        if isinstance(o, (list, tuple)):
-            # Recursively normalize nested structures
-            normalized = [_normalize_for_repr(item) for item in o]
-            return type(o)(normalized)
-        return o
-
+def _dict_sort(obj: t.Any) -> str:
     try:
-        return repr(_normalize_for_repr(obj))
+        if isinstance(obj, dict):
+            obj = dict(sorted(obj.items(), key=lambda x: str(x[0])))
     except Exception:
-        return repr(obj)
+        logger.warning("Failed to sort non-recursive dict", exc_info=True)
+    return repr(obj)
 
 
 def migrate(state_sync, **kwargs):  # type: ignore
@@ -82,12 +77,14 @@ def migrate(state_sync, **kwargs):  # type: ignore
 
         if python_env:
             for key, executable in python_env.items():
+                if key not in KEYS_TO_MAKE_DETERMINISTIC:
+                    continue
                 if isinstance(executable, dict) and executable.get("kind") == "value":
                     old_payload = executable["payload"]
                     try:
                         # Try to parse the old payload and re-serialize it deterministically
                         parsed_value = eval(old_payload)
-                        new_payload = _deterministic_repr(parsed_value)
+                        new_payload = _dict_sort(parsed_value)
 
                         # Only update if the representation changed
                         if old_payload != new_payload:
@@ -95,7 +92,7 @@ def migrate(state_sync, **kwargs):  # type: ignore
                             migration_needed = True
                     except Exception:
                         # If we still can't eval it, leave it as-is
-                        pass
+                        logger.warning("Exception trying to eval payload", exc_info=True)
 
         new_snapshots.append(
             {

sqlmesh/migrations/v0086_check_deterministic_bug.py

@@ -0,0 +1,82 @@
+import json
+import logging
+
+from sqlglot import exp
+
+from sqlmesh.core.console import get_console
+
+
+logger = logging.getLogger(__name__)
+KEYS_TO_MAKE_DETERMINISTIC = ["__sqlmesh__vars__", "__sqlmesh__blueprint__vars__"]
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    versions_table = "_versions"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+        versions_table = f"{schema}.{versions_table}"
+
+    result = engine_adapter.fetchone(
+        exp.select("schema_version").from_(versions_table), quote_identifiers=True
+    )
+    if not result:
+        # This must be the first migration, so we can skip the check since the project was not exposed to 85 migration bug
+        return
+    schema_version = result[0]
+    if schema_version < 85:
+        # The project was not exposed to the bugged 85 migration, so we can skip it.
+        return
+
+    warning = (
+        "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
+        "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
+        "command. Please run `sqlmesh diff prod` after the migration has completed, before making any new "
+        "changes. If any unexpected changes are reported, consider running a forward-only plan to apply these "
+        "changes and avoid unnecessary backfills: sqlmesh plan prod --forward-only. "
+        "See https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans for more details.\n"
+    )
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        python_env = parsed_snapshot["node"].get("python_env")
+
+        if python_env:
+            for key, executable in python_env.items():
+                if (
+                    key not in KEYS_TO_MAKE_DETERMINISTIC
+                    and isinstance(executable, dict)
+                    and executable.get("kind") == "value"
+                ):
+                    try:
+                        parsed_value = eval(executable["payload"])
+                        if isinstance(parsed_value, dict):
+                            get_console().log_warning(warning)
+                            return
+                    except Exception:
+                        logger.warning("Exception trying to eval payload", exc_info=True)

sqlmesh/utils/metaprogramming.py

@@ -5,6 +5,7 @@ import dis
 import importlib
 import inspect
 import linecache
+import logging
 import os
 import re
 import sys
@@ -23,6 +24,9 @@ from sqlmesh.utils import format_exception, unique
 from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.utils.pydantic import PydanticModel
 
+logger = logging.getLogger(__name__)
+
+
 IGNORE_DECORATORS = {"macro", "model", "signal"}
 SERIALIZABLE_CALLABLES = (type, types.FunctionType)
 LITERALS = (Number, str, bytes, tuple, list, dict, set, bool)
@@ -424,10 +428,11 @@ class Executable(PydanticModel):
         return self.kind == ExecutableKind.VALUE
 
     @classmethod
-    def value(cls, v: t.Any, is_metadata: t.Optional[bool] = None) -> Executable:
-        return Executable(
-            payload=_deterministic_repr(v), kind=ExecutableKind.VALUE, is_metadata=is_metadata
-        )
+    def value(
+        cls, v: t.Any, is_metadata: t.Optional[bool] = None, sort_root_dict: bool = False
+    ) -> Executable:
+        payload = _dict_sort(v) if sort_root_dict else repr(v)
+        return Executable(payload=payload, kind=ExecutableKind.VALUE, is_metadata=is_metadata)
 
 
 def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
@@ -635,36 +640,13 @@ def print_exception(
     out.write(tb)
 
 
-def _deterministic_repr(obj: t.Any) -> str:
-    """Create a deterministic representation by ensuring consistent ordering before repr().
-
-    For dictionaries, ensures consistent key ordering to prevent non-deterministic
-    serialization that affects fingerprinting. Uses Python's native repr() logic
-    for all formatting to handle edge cases properly.
-
-    Note that this function assumes list/tuple order is significant and therefore does not sort them.
-
-    Args:
-        obj: The object to represent as a string.
-
-    Returns:
-        A deterministic string representation of the object.
-    """
-
-    def _normalize_for_repr(o: t.Any) -> t.Any:
-        if isinstance(o, dict):
-            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
-            return {k: _normalize_for_repr(v) for k, v in sorted_items}
-        if isinstance(o, (list, tuple)):
-            # Recursively normalize nested structures
-            normalized = [_normalize_for_repr(item) for item in o]
-            return type(o)(normalized)
-        return o
-
+def _dict_sort(obj: t.Any) -> str:
     try:
-        return repr(_normalize_for_repr(obj))
+        if isinstance(obj, dict):
+            obj = dict(sorted(obj.items(), key=lambda x: str(x[0])))
     except Exception:
-        return repr(obj)
+        logger.warning("Failed to sort non-recursive dict", exc_info=True)
+    return repr(obj)
 
 
 def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:

{sqlmesh-0.201.2.dev1.dist-info → sqlmesh-0.201.2.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.201.2.dev1
+Version: 0.201.2.dev3
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.201.2.dev1.dist-info → sqlmesh-0.201.2.dev3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=hQHazrZwjilrrZX8wL1OKnB-tUikQtLSNpN9LZPiJ2c,528
+sqlmesh/_version.py,sha256=WiIT14sA7aFc6g1yecX7SEN0IfRMBpNW8l6mQIEj-2E,528
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,7 +13,7 @@ sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
 sqlmesh/core/console.py,sha256=lb6Zq6BhimfQkcSd19t35owSmaGCqw0WHMhGK-WZ-y4,165491
 sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
-sqlmesh/core/context.py,sha256=0EH5FEoPEoxgr3_yu2pasH_Itei7daInEL1uEJgpgwM,127360
+sqlmesh/core/context.py,sha256=34hhZPwqxdjLBuZ-kSlOen07CtlmD3KocQlQdXY8KeI,127454
 sqlmesh/core/context_diff.py,sha256=_Ur55fGLJ58Icsp44I4pVhdi7_JHiHq45mWIco9oFzk,21792
 sqlmesh/core/dialect.py,sha256=ELBtB8ft0OyalhgqQ1CUnn26xJxLDah2SP1v8kh_FB8,50903
 sqlmesh/core/environment.py,sha256=odus40R-YzdQdS_IiUq_tFouUXIhnpwmNwo8dE8Aw1k,13041
@@ -86,7 +86,7 @@ sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
-sqlmesh/core/model/common.py,sha256=2kXT6DhDAOR59x3EVYrhyoU4QxKv_ScG0zzp98wU8m0,17897
+sqlmesh/core/model/common.py,sha256=mUVJZ1otbcY9IPqeJQWxZOgv_2QnYEEKdFwcPvZGe9c,17961
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
 sqlmesh/core/model/definition.py,sha256=PvE6EYZrPFQHZgSw7kn-V85qpwBf07TOGMnjdz03CE0,112811
 sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
@@ -260,7 +260,8 @@ sqlmesh/migrations/v0081_update_partitioned_by.py,sha256=Rj9RDd7WBonbGfTJx3vAqSv
 sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py,sha256=K-Z0ghTCFSZuAYzNSUXF4L8axihPemM15EOXHcqkK3s,2489
 sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py,sha256=0r-SlncPec4TBpnQTeblPsiIjIOVKc10GjBVbFPGcFA,148
 sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py,sha256=w1TbhvB98czuq5VZxsfcQE2w7o-nd96aKEIG-M1dn6A,329
-sqlmesh/migrations/v0085_deterministic_repr.py,sha256=gedfV5hoq62mp9HK3eiJ1p9aYAfJ4Cku_cnQHUrEinE,4388
+sqlmesh/migrations/v0085_deterministic_repr.py,sha256=mkbZV0rzXJ5bmN1Y75kFI9T6gbnx_sy-zn_UzbOL1Sg,4265
+sqlmesh/migrations/v0086_check_deterministic_bug.py,sha256=WiYcmhQD434GXWiiakQtcy9ESmB35spXMltV43nvU7Q,2953
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
@@ -276,7 +277,7 @@ sqlmesh/utils/errors.py,sha256=8FE33Eo3KN3Y4s5EZM5t92jZi9BAVGzkr3QCGlrLeDk,5679
 sqlmesh/utils/git.py,sha256=v1MD4Zwn52UBn_tTfoKn8SAPJVGC6SEFrJ4WEJFbgF0,1868
 sqlmesh/utils/hashing.py,sha256=4Bi0NvRctBHzg58btNxXwSNgcvj8doIVVvOjJ40mHBg,512
 sqlmesh/utils/jinja.py,sha256=zulSPIICdFOjN__xkAiMpxbCS7S0--xAFK9d8VG0KvU,27844
-sqlmesh/utils/metaprogramming.py,sha256=5KW6LyjD509I2wi8pZmz2V0fXlzYq5EQ6C42ApWLJxc,24311
+sqlmesh/utils/metaprogramming.py,sha256=yDWS3_tkm6Yd5TNHgii2x_VdZ2pVM0vL6oFuwFMMs0A,23554
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=FytTSLcRNtO0-YMvpoDBkkut3yoQJR9mBSKpif8lxug,2497
 sqlmesh/utils/process.py,sha256=vpXcpJ1eIbivs7gZdwTKSJI7OS6kKBebnaBzz75ykvs,2428
@@ -284,7 +285,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.201.2.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.201.2.dev3.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -402,8 +403,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.201.2.dev1.dist-info/METADATA,sha256=k-puQ_Tjflc39pgRlT-XmHBmN8RfmgdYBwdaLZCIqZ0,26678
-sqlmesh-0.201.2.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.201.2.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.201.2.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.201.2.dev1.dist-info/RECORD,,
+sqlmesh-0.201.2.dev3.dist-info/METADATA,sha256=o8uHF5DQnR5z4OfzBTgjDRT0UEbEMhKGhJdI3Zvplhw,26678
+sqlmesh-0.201.2.dev3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.201.2.dev3.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.201.2.dev3.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.201.2.dev3.dist-info/RECORD,,