sqlmesh 0.183.1.dev11__py3-none-any.whl → 0.183.1.dev12__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.183.1.dev11'
-__version_tuple__ = version_tuple = (0, 183, 1, 'dev11')
+__version__ = version = '0.183.1.dev12'
+__version_tuple__ = version_tuple = (0, 183, 1, 'dev12')

sqlmesh/core/engine_adapter/base.py

@@ -1008,6 +1008,8 @@ class EngineAdapter:
         if materialized_properties and not materialized:
             raise SQLMeshError("Materialized properties are only supported for materialized views")
 
+        query_or_df = self._native_df_to_pandas_df(query_or_df)
+
         if isinstance(query_or_df, pd.DataFrame):
             values: t.List[t.Tuple[t.Any, ...]] = list(
                 query_or_df.itertuples(index=False, name=None)
@@ -2002,6 +2004,19 @@ class EngineAdapter:
             self.execute(query, quote_identifiers=quote_identifiers)
             return self.cursor.fetchdf()
 
+    def _native_df_to_pandas_df(
+        self,
+        query_or_df: QueryOrDF,
+    ) -> t.Union[Query, pd.DataFrame]:
+        """
+        Take a "native" DataFrame (eg Pyspark, Bigframe, Snowpark etc) and convert it to Pandas
+        """
+        if isinstance(query_or_df, (exp.Query, exp.DerivedTable, pd.DataFrame)):
+            return query_or_df
+
+        # EngineAdapter subclasses that have native DataFrame types should override this
+        raise NotImplementedError(f"Unable to convert {type(query_or_df)} to Pandas")
+
     def fetchdf(
         self, query: t.Union[exp.Expression, str], quote_identifiers: bool = False
     ) -> pd.DataFrame:

sqlmesh/core/engine_adapter/bigquery.py

@@ -26,6 +26,7 @@ from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import optional_import
 from sqlmesh.utils.date import to_datetime
 from sqlmesh.utils.errors import SQLMeshError
+from sqlmesh.utils.pandas import columns_to_types_from_dtypes
 
 if t.TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -1107,6 +1108,39 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
     def _normalize_nested_value(self, col: exp.Expression) -> exp.Expression:
         return exp.func("TO_JSON_STRING", col, dialect=self.dialect)
 
+    @t.overload
+    def _columns_to_types(
+        self, query_or_df: DF, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Dict[str, exp.DataType]: ...
+
+    @t.overload
+    def _columns_to_types(
+        self, query_or_df: Query, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Optional[t.Dict[str, exp.DataType]]: ...
+
+    def _columns_to_types(
+        self, query_or_df: QueryOrDF, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Optional[t.Dict[str, exp.DataType]]:
+        if (
+            not columns_to_types
+            and bigframes
+            and isinstance(query_or_df, bigframes.dataframe.DataFrame)
+        ):
+            # using dry_run=True attempts to prevent the DataFrame from being materialized just to read the column types from it
+            dtypes = query_or_df.to_pandas(dry_run=True).columnDtypes
+            return columns_to_types_from_dtypes(dtypes.items())
+
+        return super()._columns_to_types(query_or_df, columns_to_types)
+
+    def _native_df_to_pandas_df(
+        self,
+        query_or_df: QueryOrDF,
+    ) -> t.Union[Query, pd.DataFrame]:
+        if bigframes and isinstance(query_or_df, bigframes.dataframe.DataFrame):
+            return query_or_df.to_pandas()
+
+        return super()._native_df_to_pandas_df(query_or_df)
+
     @property
     def _query_data(self) -> t.Any:
         return self._connection_pool.get_attribute("query_data")

sqlmesh/core/engine_adapter/snowflake.py

@@ -27,6 +27,7 @@ from sqlmesh.core.engine_adapter.shared import (
 from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils import optional_import
 from sqlmesh.utils.errors import SQLMeshError
+from sqlmesh.utils.pandas import columns_to_types_from_dtypes
 
 logger = logging.getLogger(__name__)
 snowpark = optional_import("snowflake.snowpark")
@@ -298,13 +299,23 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
             if is_snowpark_dataframe:
                 temp_table.set("catalog", database)
-                df_renamed = df.rename(
-                    {
-                        col: exp.to_identifier(col).sql(dialect=self.dialect, identify=True)
-                        for col in columns_to_types
-                    }
-                )  # type: ignore
-                df_renamed.createOrReplaceTempView(
+
+                # only quote columns if they arent already quoted
+                # if the Snowpark dataframe was created from a Pandas dataframe via snowpark.create_dataframe(pandas_df),
+                # then they will be quoted already. But if the Snowpark dataframe was created manually by the user, then the
+                # columns may not be quoted
+                columns_already_quoted = all(
+                    col.startswith('"') and col.endswith('"') for col in df.columns
+                )
+                local_df = df
+                if not columns_already_quoted:
+                    local_df = df.rename(
+                        {
+                            col: exp.to_identifier(col).sql(dialect=self.dialect, identify=True)
+                            for col in columns_to_types
+                        }
+                    )  # type: ignore
+                local_df.createOrReplaceTempView(
                     temp_table.sql(dialect=self.dialect, identify=True)
                 )  # type: ignore
             elif isinstance(df, pd.DataFrame):
@@ -356,6 +367,11 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
 
         def cleanup() -> None:
             if is_snowpark_dataframe:
+                if hasattr(df, "table_name"):
+                    if isinstance(df.table_name, str):
+                        # created by the Snowpark library if the Snowpark DataFrame was created from a Pandas DataFrame
+                        # (if the Snowpark DataFrame was created via native means then there is no 'table_name' property and no temp table)
+                        self.drop_table(df.table_name)
                 self.drop_view(temp_table)
             else:
                 self.drop_table(temp_table)
@@ -381,6 +397,15 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             columns = self.cursor._result_set.batches[0].column_names
             return pd.DataFrame([dict(zip(columns, row)) for row in rows])
 
+    def _native_df_to_pandas_df(
+        self,
+        query_or_df: QueryOrDF,
+    ) -> t.Union[Query, pd.DataFrame]:
+        if snowpark and isinstance(query_or_df, snowpark.DataFrame):
+            return query_or_df.to_pandas()
+
+        return super()._native_df_to_pandas_df(query_or_df)
+
     def _get_data_objects(
         self, schema_name: SchemaName, object_names: t.Optional[t.Set[str]] = None
     ) -> t.List[DataObject]:
@@ -426,6 +451,10 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         if object_names:
             query = query.where(exp.column("TABLE_NAME").isin(*object_names))
 
+        # exclude SNOWPARK_TEMP_TABLE tables that are managed by the Snowpark library and are an implementation
+        # detail of dealing with DataFrame's
+        query = query.where(exp.column("TABLE_NAME").like("SNOWPARK_TEMP_TABLE%").not_())
+
         df = self.fetchdf(query, quote_identifiers=True)
         if df.empty:
             return []
@@ -537,3 +566,21 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
             clone_kwargs=clone_kwargs,
             **kwargs,
         )
+
+    @t.overload
+    def _columns_to_types(
+        self, query_or_df: DF, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Dict[str, exp.DataType]: ...
+
+    @t.overload
+    def _columns_to_types(
+        self, query_or_df: Query, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Optional[t.Dict[str, exp.DataType]]: ...
+
+    def _columns_to_types(
+        self, query_or_df: QueryOrDF, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
+    ) -> t.Optional[t.Dict[str, exp.DataType]]:
+        if not columns_to_types and snowpark and isinstance(query_or_df, snowpark.DataFrame):
+            return columns_to_types_from_dtypes(query_or_df.sample(n=1).to_pandas().dtypes.items())
+
+        return super()._columns_to_types(query_or_df, columns_to_types)

sqlmesh/core/engine_adapter/spark.py

@@ -382,51 +382,14 @@ class SparkEngineAdapter(
             partitioned_by=[exp.column(x) for x in primary_key] if primary_key else None,
         )
 
-    def create_view(
+    def _native_df_to_pandas_df(
         self,
-        view_name: TableName,
         query_or_df: QueryOrDF,
-        columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
-        replace: bool = True,
-        materialized: bool = False,
-        materialized_properties: t.Optional[t.Dict[str, t.Any]] = None,
-        table_description: t.Optional[str] = None,
-        column_descriptions: t.Optional[t.Dict[str, str]] = None,
-        view_properties: t.Optional[t.Dict[str, exp.Expression]] = None,
-        **create_kwargs: t.Any,
-    ) -> None:
-        """Create a view with a query or dataframe.
-
-        If a dataframe is passed in, it will be converted into a literal values statement.
-        This should only be done if the dataframe is very small!
-
-        Args:
-            view_name: The view name.
-            query_or_df: A query or dataframe.
-            columns_to_types: Columns to use in the view statement.
-            replace: Whether or not to replace an existing view - defaults to True.
-            materialized: Whether or not the view should be materialized - defaults to False.
-            materialized_properties: Optional materialized view properties to add to the view.
-            table_description: Optional table description from MODEL DDL.
-            column_descriptions: Optional column descriptions from model query.
-            create_kwargs: Additional kwargs to pass into the Create expression
-            view_properties: Optional view properties to add to the view.
-        """
-        pyspark_df = self.try_get_pyspark_df(query_or_df)
-        if pyspark_df:
-            query_or_df = pyspark_df.toPandas()
-        super().create_view(
-            view_name,
-            query_or_df,
-            columns_to_types,
-            replace,
-            materialized,
-            materialized_properties,
-            table_description,
-            column_descriptions,
-            view_properties=view_properties,
-            **create_kwargs,
-        )
+    ) -> t.Union[Query, pd.DataFrame]:
+        if pyspark_df := self.try_get_pyspark_df(query_or_df):
+            return pyspark_df.toPandas()
+
+        return super()._native_df_to_pandas_df(query_or_df)
 
     def _create_table(
         self,

sqlmesh/utils/pandas.py

@@ -27,13 +27,20 @@ PANDAS_TYPE_MAPPINGS = {
     pd.Float32Dtype(): exp.DataType.build("float"),
     pd.Float64Dtype(): exp.DataType.build("double"),
     pd.StringDtype(): exp.DataType.build("text"),  # type: ignore
+    pd.StringDtype("pyarrow"): exp.DataType.build("text"),
     pd.BooleanDtype(): exp.DataType.build("boolean"),
 }
 
 
 def columns_to_types_from_df(df: pd.DataFrame) -> t.Dict[str, exp.DataType]:
+    return columns_to_types_from_dtypes(df.dtypes.items())
+
+
+def columns_to_types_from_dtypes(
+    dtypes: t.Iterable[t.Tuple[t.Hashable, t.Any]],
+) -> t.Dict[str, exp.DataType]:
     result = {}
-    for column_name, column_type in df.dtypes.items():
+    for column_name, column_type in dtypes:
         exp_type: t.Optional[exp.DataType] = None
         if hasattr(pd, "DatetimeTZDtype") and isinstance(column_type, pd.DatetimeTZDtype):
             exp_type = exp.DataType.build("timestamptz")

{sqlmesh-0.183.1.dev11.dist-info → sqlmesh-0.183.1.dev12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.183.1.dev11
+Version: 0.183.1.dev12
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.183.1.dev11.dist-info → sqlmesh-0.183.1.dev12.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=DRovDXI-9D9xeYDP7t8Fq7mI31S2btlN-LHPOK3y7PA,7240
-sqlmesh/_version.py,sha256=kTLIe0mLJLAvUOHpn79satDKVGT_ads4OrzArp14Yk8,530
+sqlmesh/_version.py,sha256=PNcoRr1Tz4tr1llNJk86ORpVx483m6Xj_6wKHJhtX0E,530
 sqlmesh/magics.py,sha256=9oY1Uz8uwsuTtPRg9HTYDxnGiJ7-OVRCpmCgAhbKHUU,39431
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,9 +59,9 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=BAM_U7Vsi0k37XHu_nNnu3whXxuTVDey_uZe3aelKYY,1033
 sqlmesh/core/engine_adapter/athena.py,sha256=kVkGhDhjVd_8zsvq7B94kr_CHD0bct6XXFkhzrOjQOY,26158
-sqlmesh/core/engine_adapter/base.py,sha256=Ni9kF_ms6s5xKGPgroMAsxjqTvJmX1relpGaDAxYsuI,99947
+sqlmesh/core/engine_adapter/base.py,sha256=XQeJi4Rr5VllnQKFZBYXNs1G142T16ZOVU_l9UYvzoE,100539
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
-sqlmesh/core/engine_adapter/bigquery.py,sha256=mdgXlFOdyz6Qvno_io_9hm5QLEOQncLnT5itnOU7Ock,47845
+sqlmesh/core/engine_adapter/bigquery.py,sha256=gnd2cFc7v95tT_E9XMHHgccS-QwSF2sJHsL8xfcSguk,49249
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=4ACsYsZ8-gvyCozwoEMEayTTUdMjmNea4xUkxtXw6R4,34907
 sqlmesh/core/engine_adapter/databricks.py,sha256=AbNbnt9nQdOd7lTov2zsKROZlzgV2fLvOYXam3cowvk,14507
 sqlmesh/core/engine_adapter/duckdb.py,sha256=FfNmBzLySDDi--nTlsSavTwdperCzhpYzDoroCilPcA,4900
@@ -72,8 +72,8 @@ sqlmesh/core/engine_adapter/postgres.py,sha256=XWnV7MDndkCRokb7ih2yxXodw4W8E61yH
 sqlmesh/core/engine_adapter/redshift.py,sha256=_rZWv4rchDAzB6m6idoV1NLHFBKK_gM4nlfeErqXEcE,16691
 sqlmesh/core/engine_adapter/risingwave.py,sha256=NXXuiPfbB5ZSp7igMqZpaX2c-5TYreOlIouPU7iCVJ8,936
 sqlmesh/core/engine_adapter/shared.py,sha256=NyF2osSJDWPp7KlJUBR4GZcS_cJFr4daUPvhGyp4XcQ,12909
-sqlmesh/core/engine_adapter/snowflake.py,sha256=M0--yVpl8S9x1kjz7t-lcIORewtnA7dEPEd4oBehb7Q,23967
-sqlmesh/core/engine_adapter/spark.py,sha256=NwtVkdvy39bD6gnrIvIzITa0g510KZOrP3jz0aMKjwI,22711
+sqlmesh/core/engine_adapter/snowflake.py,sha256=UrfoKY6Z4S9rOgtcJqzYW84Z3XN4h-HGrJnftZCxPHo,26403
+sqlmesh/core/engine_adapter/spark.py,sha256=nDJXH1su08wv3m9X0Mz-x98xhZblq1BvM3EtRYjZhrI,21012
 sqlmesh/core/engine_adapter/trino.py,sha256=pZ2E1owyC6DPmkU3RkibR3CIE1gE-8fNd016YHh52II,14915
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=UL05OjFBfoLSYGWXNf4zjZYzIKr-OyFIMIcIh7RhR8s,5237
@@ -262,11 +262,11 @@ sqlmesh/utils/hashing.py,sha256=4Bi0NvRctBHzg58btNxXwSNgcvj8doIVVvOjJ40mHBg,512
 sqlmesh/utils/jinja.py,sha256=6Iob2Pv88L2rCaHyl_V90b5GKsG9hI10Ua_FRWhQIl0,23628
 sqlmesh/utils/metaprogramming.py,sha256=i1rxWi-oxdh1VwKoBh5Qvy0GgHm9GaDZMvIrv3QXGt4,23083
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
-sqlmesh/utils/pandas.py,sha256=j_tYhuPcWpzuPceZZ6IhdTVbmxYyGJW7l4FHr4bQqjM,1824
+sqlmesh/utils/pandas.py,sha256=WmNlTLuv_qLelrcTnwb05QMaW53AYM6dpOW0o3dz6qM,2051
 sqlmesh/utils/pydantic.py,sha256=ioaAzYF42PutEJRyAS11-AHguiCG3yWI6VRcTVa2G-g,10470
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.183.1.dev11.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.183.1.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=nPqIDCJmonOaEAp4hhwrwgPdwt_F-5zzw-WW1kJqq3I,5472
 web/client/dist/assets/Audits-DiRd7NXk.js,sha256=Q5dIFqsHps-vVYD8037NpTPW_1juQgn_Ov45smEm7Y8,1041
@@ -384,8 +384,8 @@ web/server/api/endpoints/models.py,sha256=SzS5bshzdXyM6hb948sZ_3k-xhpGCHkxkb7pHz
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.183.1.dev11.dist-info/METADATA,sha256=MSoP9L1VSUlqF9Vm_cwRuqAbgsdELDnaCSK-sZ3DN1U,25253
-sqlmesh-0.183.1.dev11.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-sqlmesh-0.183.1.dev11.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.183.1.dev11.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.183.1.dev11.dist-info/RECORD,,
+sqlmesh-0.183.1.dev12.dist-info/METADATA,sha256=3GDwXQ8VnIg--Q9fGYfL2wKdBNSBPGtIbSFzUrYINLI,25253
+sqlmesh-0.183.1.dev12.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+sqlmesh-0.183.1.dev12.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.183.1.dev12.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.183.1.dev12.dist-info/RECORD,,