sqlframe 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.0'
-__version_tuple__ = version_tuple = (1, 0, 0)
+__version__ = version = '1.1.0'
+__version_tuple__ = version_tuple = (1, 1, 0)

sqlframe/base/catalog.py

@@ -8,7 +8,7 @@ from sqlglot import MappingSchema, exp
 
 from sqlframe.base.decorators import normalize
 from sqlframe.base.exceptions import TableSchemaError
-from sqlframe.base.util import to_schema
+from sqlframe.base.util import ensure_column_mapping, to_schema
 
 if t.TYPE_CHECKING:
     from sqlglot.schema import ColumnMapping
@@ -82,6 +82,7 @@ class _BaseCatalog(t.Generic[SESSION, DF]):
                 raise TableSchemaError(
                     "This session does not have access to a catalog that can lookup column information. See docs for explicitly defining columns or using a session that can automatically determine this."
                 )
+        column_mapping = ensure_column_mapping(column_mapping)  # type: ignore
         self._schema.add_table(table, column_mapping, dialect=self.session.input_dialect)
 
     @normalize(["dbName"])

sqlframe/base/dataframe.py

@@ -417,7 +417,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         from sqlframe.base.session import _BaseSession
 
         value = expression.sql(dialect=_BaseSession().input_dialect).encode("utf-8")
-        hash = f"t{zlib.crc32(value)}"[:6]
+        hash = f"t{zlib.crc32(value)}"[:9]
         return self.session._normalize_string(hash)
 
     def _get_select_expressions(
@@ -606,8 +606,11 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         return df._convert_leaf_to_cte(sequence_id=new_sequence_id)
 
     @operation(Operation.WHERE)
-    def where(self, column: t.Union[Column, bool], **kwargs) -> Self:
-        col = self._ensure_and_normalize_col(column)
+    def where(self, column: t.Union[Column, str, bool], **kwargs) -> Self:
+        if isinstance(column, str):
+            col = sqlglot.parse_one(column, dialect=self.session.input_dialect)
+        else:
+            col = self._ensure_and_normalize_col(column)
         return self.copy(expression=self.expression.where(col.expression))
 
     filter = where
@@ -1094,7 +1097,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         )
         if existing_col_index:
             expression = self.expression.copy()
-            expression.expressions[existing_col_index] = col.expression
+            expression.expressions[existing_col_index] = col.alias(colName).expression
             return self.copy(expression=expression)
         return self.copy().select(col.alias(colName), append=True)
 

sqlframe/base/session.py

@@ -313,24 +313,16 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN]):
         sel_expression = exp.Select(**select_kwargs)
         if empty_df:
             sel_expression = sel_expression.where(exp.false())
-        # if empty_df:
-        #     if not column_mapping:
-        #         # If we don't have rows or columns then we just return a null with a false expression
-        #         sel_expression = (
-        #             exp.Select().select("null").from_("VALUES (NULL)").where(exp.false())
-        #         )
-        #     else:
-        #         # Ensure no results are returned if the dataframe is expected to be empty instead of
-        #         # a row of null values
-        #         sel_expression = sel_expression.where(exp.false())
         return self._create_df(sel_expression)
 
-    def sql(self, sqlQuery: t.Union[str, exp.Expression]) -> DF:
-        expression = self._optimize(
+    def sql(self, sqlQuery: t.Union[str, exp.Expression], optimize: bool = True) -> DF:
+        expression = (
             sqlglot.parse_one(sqlQuery, read=self.input_dialect)
             if isinstance(sqlQuery, str)
             else sqlQuery
         )
+        if optimize:
+            expression = self._optimize(expression)
         if self.temp_views:
             replacement_mapping = {}
             for table in expression.find_all(exp.Table):

sqlframe/base/util.py

@@ -113,6 +113,8 @@ def ensure_column_mapping(schema: t.Union[str, StructType]) -> t.Dict:
         }
     # TODO: Make a protocol with a `simpleString` attribute as what it looks for instead of the actual
     # `StructType` object.
+    elif hasattr(schema, "simpleString"):
+        return {struct_field.name: struct_field.dataType.simpleString() for struct_field in schema}
     return sqlglot_ensure_column_mapping(schema)  # type: ignore
 
 

sqlframe/duckdb/readwriter.py

@@ -5,6 +5,9 @@ from __future__ import annotations
 import logging
 import typing as t
 
+from sqlglot import exp
+from sqlglot.helper import ensure_list
+
 from sqlframe.base.readerwriter import _BaseDataFrameReader, _BaseDataFrameWriter
 from sqlframe.base.util import ensure_column_mapping, to_csv
 
@@ -69,13 +72,22 @@ class DuckDBDataFrameReader(_BaseDataFrameReader["DuckDBSession", "DuckDBDataFra
         |100|NULL|
         +---+----+
         """
+        if schema:
+            column_mapping = ensure_column_mapping(schema)
+            select_columns = [x.expression for x in self._to_casted_columns(column_mapping)]
+            if format == "csv":
+                duckdb_columns = ", ".join(
+                    [f"'{column}': '{dtype}'" for column, dtype in column_mapping.items()]
+                )
+                options["columns"] = "{" + duckdb_columns + "}"
+        else:
+            select_columns = [exp.Star()]
         if format:
-            sql = f"SELECT * FROM read_{format}('{path}', {to_csv(options)})"
+            paths = ",".join([f"'{path}'" for path in ensure_list(path)])
+            from_clause = f"read_{format}([{paths}], {to_csv(options)})"
         else:
-            sql = f"select * from '{path}'"
-        df = self.session.sql(sql)
-        if schema:
-            df = df.select(*self._to_casted_columns(ensure_column_mapping(schema)))
+            from_clause = f"'{path}'"
+        df = self.session.sql(exp.select(*select_columns).from_(from_clause), optimize=False)
         self.session._last_loaded_file = path  # type: ignore
         return df
 

{sqlframe-1.0.0.dist-info → sqlframe-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.0.0
+Version: 1.1.0
 Summary: Taking the Spark out of PySpark by converting to SQL
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -18,7 +18,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: prettytable (<3.11.0)
-Requires-Dist: sqlglot (<23.18,>=23.14.0)
+Requires-Dist: sqlglot (<24.1,>=24.0.0)
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery-storage (<3,>=2) ; extra == 'bigquery'
 Requires-Dist: google-cloud-bigquery[pandas] (<4,>=3) ; extra == 'bigquery'

{sqlframe-1.0.0.dist-info → sqlframe-1.1.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 sqlframe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sqlframe/_version.py,sha256=DGJ4pj32xs3_DRJhSzQwCiRNnAQrMgo09USYpyMZsKc,411
+sqlframe/_version.py,sha256=CqDGE4B1ZqZ-56mxeOFcXRTmlxrdOh4ayrjbcPjziE4,411
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=DuTay8-o9W-pw3RPZCgLunKNJLS9PkaV11G_pxXp9NY,1256
-sqlframe/base/catalog.py,sha256=Y9v7ZdpUVeFjjqcKyrRTBKK1H-IXM5SuIwVm5blSXK4,36984
+sqlframe/base/catalog.py,sha256=P55_mLrk9KUC2LRYyLgSzVAan1Lx6EBNfdDjMEtc9DA,37086
 sqlframe/base/column.py,sha256=K9TtpBjVsFK9NtEX9ZQscU6qZIKiVVh1zj3jG9HifyA,15110
-sqlframe/base/dataframe.py,sha256=MTghHiW5nXDE6p214h93FChUlOdd8c6xf2WIZxrToR0,58817
+sqlframe/base/dataframe.py,sha256=lNBMm79rX1DAt5vj4qKuxDwyhJdnhDROcOPcqVDxNHE,58971
 sqlframe/base/decorators.py,sha256=fnqT1Hqa0J_gUurDcVY1Dcscj6SXFxFJ5PKAw-xe5sU,2097
 sqlframe/base/exceptions.py,sha256=pCB9hXX4jxZWzNg3JN1i38cv3BmpUlee5NoLYx3YXIQ,208
 sqlframe/base/function_alternatives.py,sha256=to0kv3MTJmQFeVTMcitz0AxBIoUJC3cu5LkEY5aJpoo,31318
@@ -13,10 +13,10 @@ sqlframe/base/group.py,sha256=sKoaI2aLMih9nJTQfqzfJ00NbBcGQtArWXYHT40motQ,4060
 sqlframe/base/normalize.py,sha256=Ie6IcrD9dL-xBUKgDoh_c_gfLw68tBK5AmiprCA8MXE,3633
 sqlframe/base/operations.py,sha256=fVlAse6-WdQnEaHghRZVHXOesQ3OnKQwBnVYv5nVRiI,3457
 sqlframe/base/readerwriter.py,sha256=cgg7KuO7Eu8fScKOg1KyNFAcgnsjpU6yusPVs0o52a4,25213
-sqlframe/base/session.py,sha256=0ZyUs5kHcEM2Kk74BH9M1hCvEGBsp1_RD1lRVwPCH9M,22390
+sqlframe/base/session.py,sha256=evVdd-FGKkp-Wg80UG5289iRtBihLFfkqrcXTH64_R8,21926
 sqlframe/base/transforms.py,sha256=EKwUpfp83bncEs_MNmI2OO7gV6vA_Rr89ZWmE4eETSw,468
 sqlframe/base/types.py,sha256=1CwMW9Q1inYzQcPTyjv1QANtVSHha8ZmBigmopQET98,11925
-sqlframe/base/util.py,sha256=mnJKg1c_CpkuB1CqyB1f-WamvV7XL3__Y45tOIqauO4,7455
+sqlframe/base/util.py,sha256=SeUC2pcSBGnsS1W5PL1p-IGC6bJG8_2a7En2hxSTmpA,7597
 sqlframe/base/window.py,sha256=yyKvoNi41vL2t7XK2Ysjp8Q2FNIu3BYv-9EPtp5og6k,4944
 sqlframe/base/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/mixins/catalog_mixins.py,sha256=ZNzNn-cWB0RwT7L1KZCWYke2JlP-cZze0MDNOzSfHew,14093
@@ -37,7 +37,7 @@ sqlframe/duckdb/column.py,sha256=wkEPcp3xVsH5nC3kpacXqNkRv9htPtBgt-0uFRxIRNs,56
 sqlframe/duckdb/dataframe.py,sha256=9T6GV4JScaApFSA4T7fixot78HMUgkjGxU7TgjolOOM,1410
 sqlframe/duckdb/functions.py,sha256=srvzbk_Wg-wQPFGYp624dRDyYJghi47M8E-Tu7pBdY0,1507
 sqlframe/duckdb/group.py,sha256=sYTExtNprfbW74LWc_Lyjc1G6K1FogQsdILU2599Bq8,384
-sqlframe/duckdb/readwriter.py,sha256=ThRTEE_RHsFwJF-SHF_HkPiJ9q0SPSn20McChMZtJeE,3817
+sqlframe/duckdb/readwriter.py,sha256=TC0LigUmCRpcdx4B8Mb5ap5ifFBrjmbqXmhUB5rG87U,4376
 sqlframe/duckdb/session.py,sha256=TCAVsSqBGGj1Otb2iIkSkWqjbzzg1MeDAafGN928-O8,1893
 sqlframe/duckdb/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
 sqlframe/duckdb/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
@@ -91,8 +91,8 @@ sqlframe/standalone/readwriter.py,sha256=n2uoebNdL_t6_eaXNkpu7Zv2UmZ9I3rASuo01gG
 sqlframe/standalone/session.py,sha256=xWxBh-OtH--LmWtpDboOBpwKLcaBK5JV-IF2gCra5k0,1192
 sqlframe/standalone/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
-sqlframe-1.0.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-1.0.0.dist-info/METADATA,sha256=vNKV-_xHHk2p19RFX9cUrSs5cCbzTVcLJqzSntxROLI,6875
-sqlframe-1.0.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-sqlframe-1.0.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-1.0.0.dist-info/RECORD,,
+sqlframe-1.1.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-1.1.0.dist-info/METADATA,sha256=RBSfrpj8FYCqz79aL88JvpDRIkKZvbASKiwT0YqwXm0,6873
+sqlframe-1.1.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+sqlframe-1.1.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-1.1.0.dist-info/RECORD,,