sqlmesh 0.186.1.dev3__py3-none-any.whl → 0.186.1.dev5__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.186.1.dev3'
-__version_tuple__ = version_tuple = (0, 186, 1, 'dev3')
+__version__ = version = '0.186.1.dev5'
+__version_tuple__ = version_tuple = (0, 186, 1, 'dev5')

sqlmesh/core/config/connection.py

@@ -13,6 +13,7 @@ from functools import partial
 
 import pydantic
 from pydantic import Field
+from pydantic_core import from_json
 from packaging import version
 from sqlglot import exp
 from sqlglot.helper import subclasses
@@ -33,6 +34,7 @@ from sqlmesh.utils.pydantic import (
     field_validator,
     model_validator,
     validation_error_message,
+    get_concrete_types_from_typehint,
 )
 from sqlmesh.utils.aws import validate_s3_uri
 
@@ -177,6 +179,42 @@ class ConnectionConfig(abc.ABC, BaseConfig):
             return self.db
         return None
 
+    @model_validator(mode="before")
+    @classmethod
+    def _expand_json_strings_to_concrete_types(cls, data: t.Any) -> t.Any:
+        """
+        There are situations where a connection config class has a field that is some kind of complex type
+        (eg a list of strings or a dict) but the value is being supplied from a source such as an environment variable
+
+        When this happens, the value is supplied as a string rather than a Python object. We need some way
+        of turning this string into the corresponding Python list or dict.
+
+        Rather than doing this piecemeal on every config subclass, this provides a generic implementatation
+        to identify fields that may be be supplied as JSON strings and handle them transparently
+        """
+        if data and isinstance(data, dict):
+            for maybe_json_field_name in cls._get_list_and_dict_field_names():
+                if (value := data.get(maybe_json_field_name)) and isinstance(value, str):
+                    # crude JSON check as we dont want to try and parse every string we get
+                    value = value.strip()
+                    if value.startswith("{") or value.startswith("["):
+                        data[maybe_json_field_name] = from_json(value)
+
+        return data
+
+    @classmethod
+    def _get_list_and_dict_field_names(cls) -> t.Set[str]:
+        field_names = set()
+        for name, field in cls.model_fields.items():
+            if field.annotation:
+                field_types = get_concrete_types_from_typehint(field.annotation)
+
+                # check if the field type is something that could concievably be supplied as a json string
+                if any(ft is t for t in (list, tuple, set, dict) for ft in field_types):
+                    field_names.add(name)
+
+        return field_names
+
 
 class DuckDBAttachOptions(BaseConfig):
     type: str

sqlmesh/core/plan/evaluator.py

@@ -116,6 +116,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
                 after_promote_snapshots = all_names - before_promote_snapshots
                 deployability_index_for_evaluation = DeployabilityIndex.all_deployable()
 
+            # Step 1: Run before_all environment statements before doing anything else
             execute_environment_statements(
                 adapter=self.snapshot_evaluator.adapter,
                 environment_statements=plan.environment_statements or [],
@@ -128,15 +129,24 @@ class BuiltInPlanEvaluator(PlanEvaluator):
                 execution_time=plan.execution_time,
             )
 
-            self._run_audits_for_metadata_snapshots(plan, new_snapshots)
-
+            # Step 2:Store new snapshot records in the state and create physical tables for them
             push_completion_status = self._push(plan, snapshots, deployability_index_for_creation)
             if push_completion_status.is_nothing_to_do:
                 self.console.log_success(
                     "" if plan.restatements else "\nSKIP: No physical layer updates to perform"
                 )
+
+            # Step 3: Update the intervals for the new forward-only snapshots
             update_intervals_for_new_snapshots(plan.new_snapshots, self.state_sync)
+
+            # Step 4: Run audits without evaluations for snapshots that capture audit metadata changes
+            self._run_audits_for_metadata_snapshots(plan, new_snapshots)
+
+            # Step 5: Remove intervals for snapshots that need to be restated
             self._restate(plan, snapshots_by_name)
+
+            # Step 6: Backfill missing intervals for snapshots that can be backfilled before updating
+            # the schema of production tables
             first_bf_completion_status = self._backfill(
                 plan,
                 snapshots_by_name,
@@ -144,9 +154,15 @@ class BuiltInPlanEvaluator(PlanEvaluator):
                 deployability_index_for_evaluation,
                 circuit_breaker=circuit_breaker,
             )
+
+            # Step 7: Update the target environment record in the state and migrate table schemas for forward-only
+            # snapshots if deploying to production
             promotion_result = self._promote(
                 plan, snapshots, before_promote_snapshots, deployability_index_for_creation
             )
+
+            # Step 8: Backfill missing intervals for snapshots that can be backfilled only after updating
+            # the schema of production tables
             second_bf_completion_status = self._backfill(
                 plan,
                 snapshots_by_name,
@@ -154,15 +170,20 @@ class BuiltInPlanEvaluator(PlanEvaluator):
                 deployability_index_for_evaluation,
                 circuit_breaker=circuit_breaker,
             )
+
             if (
                 first_bf_completion_status.is_nothing_to_do
                 and second_bf_completion_status.is_nothing_to_do
             ):
                 self.console.log_success("SKIP: No model batches to execute")
+
+            # Step 9: Update environment views to point at new physical tables and finalize the environment
+            # record in the state
             self._update_views(
                 plan, snapshots, promotion_result, deployability_index_for_evaluation
             )
 
+            # Step 10: Run after_all environment statements
             execute_environment_statements(
                 adapter=self.snapshot_evaluator.adapter,
                 environment_statements=plan.environment_statements or [],

sqlmesh/utils/pydantic.py

@@ -306,6 +306,24 @@ def cron_validator(v: t.Any) -> str:
     return v
 
 
+def get_concrete_types_from_typehint(typehint: type[t.Any]) -> set[type[t.Any]]:
+    concrete_types = set()
+    unpacked = t.get_origin(typehint)
+    if unpacked is None:
+        if type(typehint) == type(type):
+            return {typehint}
+    elif unpacked is t.Union:
+        for item in t.get_args(typehint):
+            if str(item).startswith("typing."):
+                concrete_types |= get_concrete_types_from_typehint(item)
+            else:
+                concrete_types.add(item)
+    else:
+        concrete_types.add(unpacked)
+
+    return concrete_types
+
+
 if t.TYPE_CHECKING:
     SQLGlotListOfStrings = t.List[str]
     SQLGlotString = str

{sqlmesh-0.186.1.dev3.dist-info → sqlmesh-0.186.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.186.1.dev3
+Version: 0.186.1.dev5
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.186.1.dev3.dist-info → sqlmesh-0.186.1.dev5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=69xw6bTmi4cW90M-sl3g6wLDgxVcLHcKg1q8GS3lqyA,7466
-sqlmesh/_version.py,sha256=q0wHKOOCel53puhXCfdhzyK_CaoMrYy0za8-7qWbl-I,528
+sqlmesh/_version.py,sha256=3v3CElNhp7edOXQqADP-xzMSjCYE9ldTFk_BbXg7f8U,528
 sqlmesh/magics.py,sha256=JHorAZNtezf123jmTyNSoUC9yHmv-08hMP205PJmmIo,39750
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,7 +41,7 @@ sqlmesh/core/config/__init__.py,sha256=b4gFb04tpeZpowqJD-7h3U3TLgxrpQhZahVEAo6VW
 sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
 sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
 sqlmesh/core/config/common.py,sha256=-o_HoRFtR4u84frg_TLno7z7vS9II6ASiOayb59MKHo,2806
-sqlmesh/core/config/connection.py,sha256=scyhg-gq0czu7-2y9n1tBKnLfM0jT51QaYR_hFCeQU8,73917
+sqlmesh/core/config/connection.py,sha256=NlTFCqRVA9S18EfdXELH3Qb79Qfj84Tt-_eW0-4P0Eg,75808
 sqlmesh/core/config/feature_flag.py,sha256=JRcD31OG_t2cYc24meKbHDGB0iVID0P07V2oOeB_mgA,203
 sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ,1355
 sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
@@ -95,7 +95,7 @@ sqlmesh/core/model/seed.py,sha256=ctUHH6CmeEnBVJyIAZznx0xJBm09W7VIZ-76Cfwfq28,50
 sqlmesh/core/plan/__init__.py,sha256=ctpnu9mK3uTujs0It8qRMqcj9GdObXDnvSgn4rD2zqc,450
 sqlmesh/core/plan/builder.py,sha256=Fd5raxq6r6b_3mOZndiI2U7SmliVYevcrpxWNqRV_OE,38471
 sqlmesh/core/plan/definition.py,sha256=SI3zi2poPkKJKTZogYv60Q7eIOhksaAcFUcu-J2-XqI,13218
-sqlmesh/core/plan/evaluator.py,sha256=ocRqGy36RKZyfkGBzaXDnCA1-p8JPkb5IF__ZrmEhL4,25724
+sqlmesh/core/plan/evaluator.py,sha256=RAHJgQoV5Y5SIG_pV2O_Bn0VFlD1yzMC0UrUU6eaB_E,26842
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=MRYi_koDvp2yeQ5NDGoNFzhuJTw7Vh7mI0oJyeVbqio,4022
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
@@ -264,11 +264,11 @@ sqlmesh/utils/jinja.py,sha256=6Iob2Pv88L2rCaHyl_V90b5GKsG9hI10Ua_FRWhQIl0,23628
 sqlmesh/utils/metaprogramming.py,sha256=i1rxWi-oxdh1VwKoBh5Qvy0GgHm9GaDZMvIrv3QXGt4,23083
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=uKI-ygEtMcGDRzXdBu0RkGmMBkYxwu5O2Wabho841Ls,2196
-sqlmesh/utils/pydantic.py,sha256=dywc9i-diWLA3vB-X_28I8NX3BPoSWAjcaP31zkzr0k,11115
+sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,11685
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.186.1.dev3.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.186.1.dev5.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=xQIU9G6dbuTnmbHZ61bSYbh0gYz9A2lb4BRwLlXL94Y,5472
 web/client/dist/assets/Audits-Dcl7Y5yf.js,sha256=D2CfXzrdXFdD_-7c51tBJCKNwUY9yGeIpDEaRae7V8I,1041
@@ -386,8 +386,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.186.1.dev3.dist-info/METADATA,sha256=Z5edNTAmrWqwHkPo1u9p_deLywLwvoyqe8j4knOuZuo,25523
-sqlmesh-0.186.1.dev3.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-sqlmesh-0.186.1.dev3.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.186.1.dev3.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.186.1.dev3.dist-info/RECORD,,
+sqlmesh-0.186.1.dev5.dist-info/METADATA,sha256=NiKDb3CP2KKR8HBi_lw8Gpwnx38auqfQg0rNmy3yifg,25523
+sqlmesh-0.186.1.dev5.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+sqlmesh-0.186.1.dev5.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.186.1.dev5.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.186.1.dev5.dist-info/RECORD,,