sqlmesh 0.186.1.dev4__py3-none-any.whl → 0.186.1.dev5__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.186.1.dev4'
-__version_tuple__ = version_tuple = (0, 186, 1, 'dev4')
+__version__ = version = '0.186.1.dev5'
+__version_tuple__ = version_tuple = (0, 186, 1, 'dev5')

sqlmesh/core/config/connection.py

@@ -13,6 +13,7 @@ from functools import partial
 
 import pydantic
 from pydantic import Field
+from pydantic_core import from_json
 from packaging import version
 from sqlglot import exp
 from sqlglot.helper import subclasses
@@ -33,6 +34,7 @@ from sqlmesh.utils.pydantic import (
     field_validator,
     model_validator,
     validation_error_message,
+    get_concrete_types_from_typehint,
 )
 from sqlmesh.utils.aws import validate_s3_uri
 
@@ -177,6 +179,42 @@ class ConnectionConfig(abc.ABC, BaseConfig):
             return self.db
         return None
 
+    @model_validator(mode="before")
+    @classmethod
+    def _expand_json_strings_to_concrete_types(cls, data: t.Any) -> t.Any:
+        """
+        There are situations where a connection config class has a field that is some kind of complex type
+        (eg a list of strings or a dict) but the value is being supplied from a source such as an environment variable
+
+        When this happens, the value is supplied as a string rather than a Python object. We need some way
+        of turning this string into the corresponding Python list or dict.
+
+        Rather than doing this piecemeal on every config subclass, this provides a generic implementatation
+        to identify fields that may be be supplied as JSON strings and handle them transparently
+        """
+        if data and isinstance(data, dict):
+            for maybe_json_field_name in cls._get_list_and_dict_field_names():
+                if (value := data.get(maybe_json_field_name)) and isinstance(value, str):
+                    # crude JSON check as we dont want to try and parse every string we get
+                    value = value.strip()
+                    if value.startswith("{") or value.startswith("["):
+                        data[maybe_json_field_name] = from_json(value)
+
+        return data
+
+    @classmethod
+    def _get_list_and_dict_field_names(cls) -> t.Set[str]:
+        field_names = set()
+        for name, field in cls.model_fields.items():
+            if field.annotation:
+                field_types = get_concrete_types_from_typehint(field.annotation)
+
+                # check if the field type is something that could concievably be supplied as a json string
+                if any(ft is t for t in (list, tuple, set, dict) for ft in field_types):
+                    field_names.add(name)
+
+        return field_names
+
 
 class DuckDBAttachOptions(BaseConfig):
     type: str

sqlmesh/utils/pydantic.py

@@ -306,6 +306,24 @@ def cron_validator(v: t.Any) -> str:
     return v
 
 
+def get_concrete_types_from_typehint(typehint: type[t.Any]) -> set[type[t.Any]]:
+    concrete_types = set()
+    unpacked = t.get_origin(typehint)
+    if unpacked is None:
+        if type(typehint) == type(type):
+            return {typehint}
+    elif unpacked is t.Union:
+        for item in t.get_args(typehint):
+            if str(item).startswith("typing."):
+                concrete_types |= get_concrete_types_from_typehint(item)
+            else:
+                concrete_types.add(item)
+    else:
+        concrete_types.add(unpacked)
+
+    return concrete_types
+
+
 if t.TYPE_CHECKING:
     SQLGlotListOfStrings = t.List[str]
     SQLGlotString = str

{sqlmesh-0.186.1.dev4.dist-info → sqlmesh-0.186.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.186.1.dev4
+Version: 0.186.1.dev5
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.186.1.dev4.dist-info → sqlmesh-0.186.1.dev5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=69xw6bTmi4cW90M-sl3g6wLDgxVcLHcKg1q8GS3lqyA,7466
-sqlmesh/_version.py,sha256=nZNgbGbDzUCLzSZHhV7u6sUFSQ-y2tAU1zZN-q70hlk,528
+sqlmesh/_version.py,sha256=3v3CElNhp7edOXQqADP-xzMSjCYE9ldTFk_BbXg7f8U,528
 sqlmesh/magics.py,sha256=JHorAZNtezf123jmTyNSoUC9yHmv-08hMP205PJmmIo,39750
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,7 +41,7 @@ sqlmesh/core/config/__init__.py,sha256=b4gFb04tpeZpowqJD-7h3U3TLgxrpQhZahVEAo6VW
 sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
 sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
 sqlmesh/core/config/common.py,sha256=-o_HoRFtR4u84frg_TLno7z7vS9II6ASiOayb59MKHo,2806
-sqlmesh/core/config/connection.py,sha256=scyhg-gq0czu7-2y9n1tBKnLfM0jT51QaYR_hFCeQU8,73917
+sqlmesh/core/config/connection.py,sha256=NlTFCqRVA9S18EfdXELH3Qb79Qfj84Tt-_eW0-4P0Eg,75808
 sqlmesh/core/config/feature_flag.py,sha256=JRcD31OG_t2cYc24meKbHDGB0iVID0P07V2oOeB_mgA,203
 sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ,1355
 sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
@@ -264,11 +264,11 @@ sqlmesh/utils/jinja.py,sha256=6Iob2Pv88L2rCaHyl_V90b5GKsG9hI10Ua_FRWhQIl0,23628
 sqlmesh/utils/metaprogramming.py,sha256=i1rxWi-oxdh1VwKoBh5Qvy0GgHm9GaDZMvIrv3QXGt4,23083
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=uKI-ygEtMcGDRzXdBu0RkGmMBkYxwu5O2Wabho841Ls,2196
-sqlmesh/utils/pydantic.py,sha256=dywc9i-diWLA3vB-X_28I8NX3BPoSWAjcaP31zkzr0k,11115
+sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,11685
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.186.1.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.186.1.dev5.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=xQIU9G6dbuTnmbHZ61bSYbh0gYz9A2lb4BRwLlXL94Y,5472
 web/client/dist/assets/Audits-Dcl7Y5yf.js,sha256=D2CfXzrdXFdD_-7c51tBJCKNwUY9yGeIpDEaRae7V8I,1041
@@ -386,8 +386,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.186.1.dev4.dist-info/METADATA,sha256=QZQyCEcO0vO_kXhiN3InAityt4DOunO6_rSocKbkfHk,25523
-sqlmesh-0.186.1.dev4.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-sqlmesh-0.186.1.dev4.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.186.1.dev4.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.186.1.dev4.dist-info/RECORD,,
+sqlmesh-0.186.1.dev5.dist-info/METADATA,sha256=NiKDb3CP2KKR8HBi_lw8Gpwnx38auqfQg0rNmy3yifg,25523
+sqlmesh-0.186.1.dev5.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+sqlmesh-0.186.1.dev5.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.186.1.dev5.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.186.1.dev5.dist-info/RECORD,,