sqlmesh 0.199.1.dev45__py3-none-any.whl → 0.200.1.dev1__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.199.1.dev45'
-__version_tuple__ = version_tuple = (0, 199, 1, 'dev45')
+__version__ = version = '0.200.1.dev1'
+__version_tuple__ = version_tuple = (0, 200, 1, 'dev1')

sqlmesh/migrations/v0085_deterministic_repr.py

@@ -0,0 +1,134 @@
+"""
+When serializing some objects, like `__sqlmesh__vars__`, the order of keys in the dictionary were not deterministic
+and therefore this migration applies deterministic sorting to the keys of the dictionary.
+"""
+
+import json
+import typing as t
+from dataclasses import dataclass
+
+from sqlglot import exp
+
+from sqlmesh.utils.migration import index_text_type, blob_text_type
+
+
+# Make sure `SqlValue` is defined so it can be used by `eval` call in the migration
+@dataclass
+class SqlValue:
+    """A SQL string representing a generated SQLGlot AST."""
+
+    sql: str
+
+
+def _deterministic_repr(obj: t.Any) -> str:
+    """
+    This is a copy of the function from utils.metaprogramming
+    """
+
+    def _normalize_for_repr(o: t.Any) -> t.Any:
+        if isinstance(o, dict):
+            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
+            return {k: _normalize_for_repr(v) for k, v in sorted_items}
+        if isinstance(o, (list, tuple)):
+            # Recursively normalize nested structures
+            normalized = [_normalize_for_repr(item) for item in o]
+            return type(o)(normalized)
+        return o
+
+    try:
+        return repr(_normalize_for_repr(obj))
+    except Exception:
+        return repr(obj)
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    import pandas as pd
+
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    migration_needed = False
+    new_snapshots = []
+
+    for (
+        name,
+        identifier,
+        version,
+        snapshot,
+        kind_name,
+        updated_ts,
+        unpaused_ts,
+        ttl_ms,
+        unrestorable,
+    ) in engine_adapter.fetchall(
+        exp.select(
+            "name",
+            "identifier",
+            "version",
+            "snapshot",
+            "kind_name",
+            "updated_ts",
+            "unpaused_ts",
+            "ttl_ms",
+            "unrestorable",
+        ).from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        python_env = parsed_snapshot["node"].get("python_env")
+
+        if python_env:
+            for key, executable in python_env.items():
+                if isinstance(executable, dict) and executable.get("kind") == "value":
+                    old_payload = executable["payload"]
+                    try:
+                        # Try to parse the old payload and re-serialize it deterministically
+                        parsed_value = eval(old_payload)
+                        new_payload = _deterministic_repr(parsed_value)
+
+                        # Only update if the representation changed
+                        if old_payload != new_payload:
+                            executable["payload"] = new_payload
+                            migration_needed = True
+                    except Exception:
+                        # If we still can't eval it, leave it as-is
+                        pass
+
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": json.dumps(parsed_snapshot),
+                "kind_name": kind_name,
+                "updated_ts": updated_ts,
+                "unpaused_ts": unpaused_ts,
+                "ttl_ms": ttl_ms,
+                "unrestorable": unrestorable,
+            }
+        )
+
+    if migration_needed and new_snapshots:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        index_type = index_text_type(engine_adapter.dialect)
+        blob_type = blob_text_type(engine_adapter.dialect)
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            columns_to_types={
+                "name": exp.DataType.build(index_type),
+                "identifier": exp.DataType.build(index_type),
+                "version": exp.DataType.build(index_type),
+                "snapshot": exp.DataType.build(blob_type),
+                "kind_name": exp.DataType.build("text"),
+                "updated_ts": exp.DataType.build("bigint"),
+                "unpaused_ts": exp.DataType.build("bigint"),
+                "ttl_ms": exp.DataType.build("bigint"),
+                "unrestorable": exp.DataType.build("boolean"),
+            },
+        )

sqlmesh/utils/metaprogramming.py

@@ -425,7 +425,9 @@ class Executable(PydanticModel):
 
     @classmethod
     def value(cls, v: t.Any, is_metadata: t.Optional[bool] = None) -> Executable:
-        return Executable(payload=repr(v), kind=ExecutableKind.VALUE, is_metadata=is_metadata)
+        return Executable(
+            payload=_deterministic_repr(v), kind=ExecutableKind.VALUE, is_metadata=is_metadata
+        )
 
 
 def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
@@ -633,6 +635,38 @@ def print_exception(
     out.write(tb)
 
 
+def _deterministic_repr(obj: t.Any) -> str:
+    """Create a deterministic representation by ensuring consistent ordering before repr().
+
+    For dictionaries, ensures consistent key ordering to prevent non-deterministic
+    serialization that affects fingerprinting. Uses Python's native repr() logic
+    for all formatting to handle edge cases properly.
+
+    Note that this function assumes list/tuple order is significant and therefore does not sort them.
+
+    Args:
+        obj: The object to represent as a string.
+
+    Returns:
+        A deterministic string representation of the object.
+    """
+
+    def _normalize_for_repr(o: t.Any) -> t.Any:
+        if isinstance(o, dict):
+            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
+            return {k: _normalize_for_repr(v) for k, v in sorted_items}
+        if isinstance(o, (list, tuple)):
+            # Recursively normalize nested structures
+            normalized = [_normalize_for_repr(item) for item in o]
+            return type(o)(normalized)
+        return o
+
+    try:
+        return repr(_normalize_for_repr(obj))
+    except Exception:
+        return repr(obj)
+
+
 def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:
     relative_path = path.absolute().relative_to(relative_base.absolute())
     module_name = str(relative_path.with_suffix("")).replace(os.path.sep, ".")

{sqlmesh-0.199.1.dev45.dist-info → sqlmesh-0.200.1.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.199.1.dev45
+Version: 0.200.1.dev1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.199.1.dev45.dist-info → sqlmesh-0.200.1.dev1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=HL_ihNgGoramFkQE1PrHObtLhsJZGjVBQK7cM0fe3O4,530
+sqlmesh/_version.py,sha256=KslZWeWoEUEoAhNTAI5BzM2M3yOb3IJN3RCD9jDFfNY,528
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -259,6 +259,7 @@ sqlmesh/migrations/v0081_update_partitioned_by.py,sha256=Rj9RDd7WBonbGfTJx3vAqSv
 sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py,sha256=K-Z0ghTCFSZuAYzNSUXF4L8axihPemM15EOXHcqkK3s,2489
 sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py,sha256=0r-SlncPec4TBpnQTeblPsiIjIOVKc10GjBVbFPGcFA,148
 sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py,sha256=w1TbhvB98czuq5VZxsfcQE2w7o-nd96aKEIG-M1dn6A,329
+sqlmesh/migrations/v0085_deterministic_repr.py,sha256=gedfV5hoq62mp9HK3eiJ1p9aYAfJ4Cku_cnQHUrEinE,4388
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
@@ -274,7 +275,7 @@ sqlmesh/utils/errors.py,sha256=HtpLcs1IN0aq65G372flV9kqqJUULLC3Ci7wvgoBANk,5392
 sqlmesh/utils/git.py,sha256=v1MD4Zwn52UBn_tTfoKn8SAPJVGC6SEFrJ4WEJFbgF0,1868
 sqlmesh/utils/hashing.py,sha256=4Bi0NvRctBHzg58btNxXwSNgcvj8doIVVvOjJ40mHBg,512
 sqlmesh/utils/jinja.py,sha256=zulSPIICdFOjN__xkAiMpxbCS7S0--xAFK9d8VG0KvU,27844
-sqlmesh/utils/metaprogramming.py,sha256=ebN5p4Boi9VmsoNVsoUIaW3obgsrtZ6Gt1FlgbA-9xM,23126
+sqlmesh/utils/metaprogramming.py,sha256=5KW6LyjD509I2wi8pZmz2V0fXlzYq5EQ6C42ApWLJxc,24311
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=FytTSLcRNtO0-YMvpoDBkkut3yoQJR9mBSKpif8lxug,2497
 sqlmesh/utils/process.py,sha256=vpXcpJ1eIbivs7gZdwTKSJI7OS6kKBebnaBzz75ykvs,2428
@@ -282,7 +283,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.199.1.dev45.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.200.1.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -400,8 +401,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.199.1.dev45.dist-info/METADATA,sha256=NL8VQrIbMzzmLYAeFcIgiKFZUofxtSDUsBBZKgTJTHU,26658
-sqlmesh-0.199.1.dev45.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.199.1.dev45.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.199.1.dev45.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.199.1.dev45.dist-info/RECORD,,
+sqlmesh-0.200.1.dev1.dist-info/METADATA,sha256=zG4lNQXIkIYODLR_cZogwPgZhYbu8g67qxsqQLTIo2Y,26657
+sqlmesh-0.200.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.200.1.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.200.1.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.200.1.dev1.dist-info/RECORD,,