sqlmesh 0.181.2.dev12__py3-none-any.whl → 0.181.2.dev14__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.181.2.dev12'
-__version_tuple__ = version_tuple = (0, 181, 2, 'dev12')
+__version__ = version = '0.181.2.dev14'
+__version_tuple__ = version_tuple = (0, 181, 2, 'dev14')

sqlmesh/core/engine_adapter/base.py

@@ -246,7 +246,12 @@ class EngineAdapter:
         assert isinstance(df, pd.DataFrame)
         num_rows = len(df.index)
         batch_size = sys.maxsize if batch_size == 0 else batch_size
+
+        # we need to ensure that the order of the columns in columns_to_types columns matches the order of the values
+        # they can differ if a user specifies columns() on a python model in a different order than what's in the DataFrame's emitted by that model
+        df = df[list(columns_to_types)]
         values = list(df.itertuples(index=False, name=None))
+
         return [
             SourceQuery(
                 query_factory=partial(

sqlmesh/core/engine_adapter/mssql.py

@@ -218,10 +218,13 @@ class MSSQLEngineAdapter(
             # as later calls.
             if not self.table_exists(temp_table):
                 columns_to_types_create = columns_to_types.copy()
-                self._convert_df_datetime(df, columns_to_types_create)
+                ordered_df = df[
+                    list(columns_to_types_create)
+                ]  # reorder DataFrame so it matches columns_to_types
+                self._convert_df_datetime(ordered_df, columns_to_types_create)
                 self.create_table(temp_table, columns_to_types_create)
                 rows: t.List[t.Tuple[t.Any, ...]] = list(
-                    df.replace({np.nan: None}).itertuples(index=False, name=None)  # type: ignore
+                    ordered_df.replace({np.nan: None}).itertuples(index=False, name=None)  # type: ignore
                 )
                 conn = self._connection_pool.get()
                 conn.bulk_copy(temp_table.sql(dialect=self.dialect), rows)

sqlmesh/core/engine_adapter/snowflake.py

@@ -288,8 +288,25 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         is_snowpark_dataframe = snowpark and isinstance(df, snowpark.dataframe.DataFrame)
 
         def query_factory() -> Query:
+            # The catalog needs to be normalized before being passed to Snowflake's library functions because they
+            # just wrap whatever they are given in quotes without checking if its already quoted
+            database = (
+                normalize_identifiers(temp_table.catalog, dialect=self.dialect)
+                if temp_table.catalog
+                else None
+            )
+
             if is_snowpark_dataframe:
-                df.createOrReplaceTempView(temp_table.sql(dialect=self.dialect, identify=True))  # type: ignore
+                temp_table.set("catalog", database)
+                df_renamed = df.rename(
+                    {
+                        col: exp.to_identifier(col).sql(dialect=self.dialect, identify=True)
+                        for col in columns_to_types
+                    }
+                )  # type: ignore
+                df_renamed.createOrReplaceTempView(
+                    temp_table.sql(dialect=self.dialect, identify=True)
+                )  # type: ignore
             elif isinstance(df, pd.DataFrame):
                 from snowflake.connector.pandas_tools import write_pandas
 
@@ -325,11 +342,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
                     df,
                     temp_table.name,
                     schema=temp_table.db or None,
-                    database=normalize_identifiers(temp_table.catalog, dialect=self.dialect).sql(
-                        dialect=self.dialect
-                    )
-                    if temp_table.catalog
-                    else None,
+                    database=database.sql(dialect=self.dialect) if database else None,
                     chunk_size=self.DEFAULT_BATCH_SIZE,
                     overwrite=True,
                     table_type="temp",

sqlmesh/core/engine_adapter/spark.py

@@ -279,10 +279,16 @@ class SparkEngineAdapter(
     ) -> PySparkDataFrame:
         pyspark_df = self.try_get_pyspark_df(generic_df)
         if pyspark_df:
+            if columns_to_types:
+                # ensure Spark dataframe column order matches columns_to_types
+                pyspark_df = pyspark_df.select(*columns_to_types)
             return pyspark_df
         df = self.try_get_pandas_df(generic_df)
         if df is None:
             raise SQLMeshError("Ensure PySpark DF can only be run on a PySpark or Pandas DataFrame")
+        if columns_to_types:
+            # ensure Pandas dataframe column order matches columns_to_types
+            df = df[list(columns_to_types)]
         kwargs = (
             dict(schema=self.sqlglot_to_spark_types(columns_to_types)) if columns_to_types else {}
         )

{sqlmesh-0.181.2.dev12.dist-info → sqlmesh-0.181.2.dev14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.181.2.dev12
+Version: 0.181.2.dev14
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -243,8 +243,7 @@ Requires-Dist: pymssql; extra == "azuresql"
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery[pandas]; extra == "bigquery"
 Requires-Dist: google-cloud-bigquery-storage; extra == "bigquery"
-Provides-Extra: bigframes
-Requires-Dist: bigframes>=1.32.0; extra == "bigframes"
+Requires-Dist: bigframes>=1.32.0; extra == "bigquery"
 Provides-Extra: clickhouse
 Requires-Dist: clickhouse-connect; extra == "clickhouse"
 Provides-Extra: databricks
@@ -322,7 +321,7 @@ Requires-Dist: slack_sdk; extra == "slack"
 Provides-Extra: snowflake
 Requires-Dist: cryptography; extra == "snowflake"
 Requires-Dist: snowflake-connector-python[pandas,secure-local-storage]; extra == "snowflake"
-Requires-Dist: snowflake-snowpark-python; python_version < "3.12" and extra == "snowflake"
+Requires-Dist: snowflake-snowpark-python; extra == "snowflake"
 Provides-Extra: trino
 Requires-Dist: trino; extra == "trino"
 Provides-Extra: web

{sqlmesh-0.181.2.dev12.dist-info → sqlmesh-0.181.2.dev14.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=DRovDXI-9D9xeYDP7t8Fq7mI31S2btlN-LHPOK3y7PA,7240
-sqlmesh/_version.py,sha256=KPw1hSYQp86AT8JW7TewN7S1v-4tSdmS9J0O5gcznyI,530
+sqlmesh/_version.py,sha256=A7qEcETvhMQJ2QoWJ2lEMm0p8KrDJNetI0_lN-rRkiY,530
 sqlmesh/magics.py,sha256=9oY1Uz8uwsuTtPRg9HTYDxnGiJ7-OVRCpmCgAhbKHUU,39431
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,21 +59,21 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=BAM_U7Vsi0k37XHu_nNnu3whXxuTVDey_uZe3aelKYY,1033
 sqlmesh/core/engine_adapter/athena.py,sha256=kVkGhDhjVd_8zsvq7B94kr_CHD0bct6XXFkhzrOjQOY,26158
-sqlmesh/core/engine_adapter/base.py,sha256=b6JU8b54JMi-Hnn6POlsg1t8IO_srKM2HFcyq6PXegU,99639
+sqlmesh/core/engine_adapter/base.py,sha256=Ni9kF_ms6s5xKGPgroMAsxjqTvJmX1relpGaDAxYsuI,99947
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
 sqlmesh/core/engine_adapter/bigquery.py,sha256=mdgXlFOdyz6Qvno_io_9hm5QLEOQncLnT5itnOU7Ock,47845
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=4ACsYsZ8-gvyCozwoEMEayTTUdMjmNea4xUkxtXw6R4,34907
 sqlmesh/core/engine_adapter/databricks.py,sha256=AbNbnt9nQdOd7lTov2zsKROZlzgV2fLvOYXam3cowvk,14507
 sqlmesh/core/engine_adapter/duckdb.py,sha256=FfNmBzLySDDi--nTlsSavTwdperCzhpYzDoroCilPcA,4900
 sqlmesh/core/engine_adapter/mixins.py,sha256=HMB0Z9O3p5jaTZMolKiWKqpVDZokf5GoI99YXwEJ8bc,22470
-sqlmesh/core/engine_adapter/mssql.py,sha256=cp5JRO-Nkb0Bf0LqU3ApKVJw6d8Ko03sgjoSB8LPGag,11619
+sqlmesh/core/engine_adapter/mssql.py,sha256=N21kjLrvloxLPMCGloQx5Jew5SIddrCVq3lTrnl8coc,11788
 sqlmesh/core/engine_adapter/mysql.py,sha256=HnMtNjxnCVz3X3-uK8Im-BNgLo_viX6QH3Uo8rMfPeY,6935
 sqlmesh/core/engine_adapter/postgres.py,sha256=XWnV7MDndkCRokb7ih2yxXodw4W8E61yHbuWtRTFE3k,4570
 sqlmesh/core/engine_adapter/redshift.py,sha256=_rZWv4rchDAzB6m6idoV1NLHFBKK_gM4nlfeErqXEcE,16691
 sqlmesh/core/engine_adapter/risingwave.py,sha256=NXXuiPfbB5ZSp7igMqZpaX2c-5TYreOlIouPU7iCVJ8,936
 sqlmesh/core/engine_adapter/shared.py,sha256=NyF2osSJDWPp7KlJUBR4GZcS_cJFr4daUPvhGyp4XcQ,12909
-sqlmesh/core/engine_adapter/snowflake.py,sha256=LKiQq43JNukLcDq5OnKALpiZ0xjFK7fVJYJvlXHAWxo,23361
-sqlmesh/core/engine_adapter/spark.py,sha256=VOXStkBBZIzebPo_ivdQFvZtRMiWJHHo7h118Lh5Yt8,22384
+sqlmesh/core/engine_adapter/snowflake.py,sha256=M0--yVpl8S9x1kjz7t-lcIORewtnA7dEPEd4oBehb7Q,23967
+sqlmesh/core/engine_adapter/spark.py,sha256=NwtVkdvy39bD6gnrIvIzITa0g510KZOrP3jz0aMKjwI,22711
 sqlmesh/core/engine_adapter/trino.py,sha256=pZ2E1owyC6DPmkU3RkibR3CIE1gE-8fNd016YHh52II,14915
 sqlmesh/core/linter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/linter/definition.py,sha256=UL05OjFBfoLSYGWXNf4zjZYzIKr-OyFIMIcIh7RhR8s,5237
@@ -266,7 +266,7 @@ sqlmesh/utils/pandas.py,sha256=j_tYhuPcWpzuPceZZ6IhdTVbmxYyGJW7l4FHr4bQqjM,1824
 sqlmesh/utils/pydantic.py,sha256=ioaAzYF42PutEJRyAS11-AHguiCG3yWI6VRcTVa2G-g,10470
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.181.2.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.181.2.dev14.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=yO5MoJN_pmodwLqC5Uq89JoZSis5xxeHTmEw-TlVwUE,5472
 web/client/dist/assets/Audits-Dt3QUYCw.js,sha256=26GaGzCP6oOh5tsDrjvkgDgVJrYnsenF_GRK_XF__8Q,997
@@ -383,8 +383,8 @@ web/server/api/endpoints/models.py,sha256=SzS5bshzdXyM6hb948sZ_3k-xhpGCHkxkb7pHz
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.181.2.dev12.dist-info/METADATA,sha256=mIQR9ev4V0g9_roCCc8ywjVDaG9HIeuvOfGHMt7xlwo,25308
-sqlmesh-0.181.2.dev12.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-sqlmesh-0.181.2.dev12.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.181.2.dev12.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.181.2.dev12.dist-info/RECORD,,
+sqlmesh-0.181.2.dev14.dist-info/METADATA,sha256=sCSPUZ4INnhRLdVQkjK-5Oj2mvgwU_AQOTSbaWRF42U,25253
+sqlmesh-0.181.2.dev14.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+sqlmesh-0.181.2.dev14.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.181.2.dev14.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.181.2.dev14.dist-info/RECORD,,