sqlframe 3.29.1__py3-none-any.whl → 3.31.0__py3-none-any.whl

sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.29.1'
-__version_tuple__ = version_tuple = (3, 29, 1)
+__version__ = version = '3.31.0'
+__version_tuple__ = version_tuple = (3, 31, 0)

sqlframe/base/functions.py

@@ -678,7 +678,22 @@ def isnull(col: ColumnOrName) -> Column:
 def last(col: ColumnOrName, ignorenulls: t.Optional[bool] = None) -> Column:
     this = Column.invoke_expression_over_column(col, expression.Last)
     if ignorenulls:
-        return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
+        if _get_session()._is_duckdb:
+            return Column(
+                expression.Filter(
+                    this=this.expression,
+                    expression=expression.Where(
+                        this=expression.Not(
+                            this=expression.Is(
+                                this=Column.ensure_col(col).expression,
+                                expression=expression.Null(),
+                            )
+                        )
+                    ),
+                )
+            )
+        else:
+            return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
     return this
 
 
@@ -3872,7 +3887,7 @@ def json_object_keys(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "json_object_keys")
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
     """Returns the last value of `col` for a group of rows. It will return the last non-null
     value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.
@@ -3913,6 +3928,11 @@ def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]]
     |            b|            2|
     +-------------+-------------+
     """
+    session = _get_session()
+
+    if session._is_duckdb:
+        return last(col, ignoreNulls)  # type: ignore
+
     column = Column.invoke_expression_over_column(col, expression.LastValue)
 
     if ignoreNulls:

sqlframe/databricks/dataframe.py

@@ -14,6 +14,9 @@ from sqlframe.base.util import normalize_string
 from sqlframe.databricks.group import DatabricksGroupedData
 
 if t.TYPE_CHECKING:
+    from databricks.sql.client import Cursor
+    from pyarrow import Table as ArrowTable
+
     from sqlframe.databricks.readwriter import DatabricksDataFrameWriter
     from sqlframe.databricks.session import DatabricksSession
 
@@ -21,6 +24,18 @@ if t.TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class RecordBatchReaderFacade:
+    def __init__(self, cur: Cursor, batch_size: int):
+        self.cur = cur
+        self.batch_size = batch_size
+
+    def read_next_batch(self) -> ArrowTable:
+        result = self.cur.fetchmany_arrow(self.batch_size)
+        if result.num_rows == 0:
+            raise StopIteration
+        return result
+
+
 class DatabricksDataFrameNaFunctions(_BaseDataFrameNaFunctions["DatabricksDataFrame"]):
     pass
 
@@ -68,3 +83,17 @@ class DatabricksDataFrame(
                 )
             )
         return columns
+
+    @t.overload  # type: ignore
+    def toArrow(self) -> ArrowTable: ...
+
+    @t.overload
+    def toArrow(self, batch_size: int) -> RecordBatchReaderFacade: ...
+
+    def toArrow(
+        self, batch_size: t.Optional[int] = None
+    ) -> t.Union[ArrowTable, RecordBatchReaderFacade]:
+        self._collect(skip_rows=True)
+        if not batch_size:
+            return self.session._cur.fetchall_arrow()  # type: ignore
+        return RecordBatchReaderFacade(self.session._cur, batch_size)  # type: ignore

{sqlframe-3.29.1.dist-info → sqlframe-3.31.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.29.1
+Version: 3.31.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -17,13 +17,13 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: prettytable <4
-Requires-Dist: sqlglot <26.15,>=24.0.0
+Requires-Dist: sqlglot <26.17,>=24.0.0
 Requires-Dist: typing-extensions
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery-storage <3,>=2 ; extra == 'bigquery'
 Requires-Dist: google-cloud-bigquery[pandas] <4,>=3 ; extra == 'bigquery'
 Provides-Extra: databricks
-Requires-Dist: databricks-sql-connector <5,>=3.6 ; extra == 'databricks'
+Requires-Dist: databricks-sql-connector[pyarrow] <5,>=3.6 ; extra == 'databricks'
 Provides-Extra: dev
 Requires-Dist: duckdb <1.3,>=1.2 ; extra == 'dev'
 Requires-Dist: findspark <3,>=2 ; extra == 'dev'

{sqlframe-3.29.1.dist-info → sqlframe-3.31.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlframe/__init__.py,sha256=SB80yLTITBXHI2GCDS6n6bN5ObHqgPjfpRPAUwxaots,3403
-sqlframe/_version.py,sha256=GhEeAzqz63f8BRgFh1f-TAAn74ElZQIuJ1Tt4hNpws4,513
+sqlframe/_version.py,sha256=AgRbmN0zJrj6Fie24WbIl_HozqASkUgbWo9IOWaS7vU,513
 sqlframe/py.typed,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
@@ -9,7 +9,7 @@ sqlframe/base/dataframe.py,sha256=D2N2Kvh_tiF60fYODUikq0xRCJYY4WB2aHbEcq5NIUo,84
 sqlframe/base/decorators.py,sha256=IhE5xNQDkwJHacCvulq5WpUKyKmXm7dL2A3o5WuKGP4,2131
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
 sqlframe/base/function_alternatives.py,sha256=Bs1bwl25fN3Yy9rb4GnUWBGunQ1C_yelkb2yV9DSZIY,53918
-sqlframe/base/functions.py,sha256=Rc9GfD3tIDg73FPqBAgqIDlrxH-A1c-ZWO76bcViX3w,224169
+sqlframe/base/functions.py,sha256=iReQ8NW4cwTvgGOXQq6WphC3DQbtKjIHDPJZgWjE614,224862
 sqlframe/base/group.py,sha256=OY4w1WRsCqLgW-Pi7DjF63zbbxSLISCF3qjAbzI2CQ4,4283
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=g-YNcbvNKTOBbYm23GKfB3fmydlR7ZZDAuZUtXIHtzw,4438
@@ -42,7 +42,7 @@ sqlframe/bigquery/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,3
 sqlframe/databricks/__init__.py,sha256=BkB_eO1UYwcf8j6x7bi4BWmDCMkfn0CUMwossWgwaG4,993
 sqlframe/databricks/catalog.py,sha256=T_4NlQ7TD57_UTYFeRezYUu0t9NEx7cgJS60O4DKcBQ,18609
 sqlframe/databricks/column.py,sha256=E1tUa62Y5HajkhgFuebU9zohrGyieudcHzTT8gfalio,40
-sqlframe/databricks/dataframe.py,sha256=uBJ8GxjfiODto2PvhgmsslIwKArW7tcLBA70tvtun3M,2177
+sqlframe/databricks/dataframe.py,sha256=8kwT1kWU2TwGjR9zDrGdmkvabiBCivA_Mcg06r2XVX4,3111
 sqlframe/databricks/functions.py,sha256=La8rjAwO0hD4FBO0QxW5CtZtFAPvOrVc6lG4OtPGgbc,336
 sqlframe/databricks/functions.pyi,sha256=FzVBpzXCJzxIp73sIAo_R8Wx8uOJrix-W12HsgyeTcQ,23799
 sqlframe/databricks/group.py,sha256=dU3g0DVLRlfOSCamKchQFXRd1WTFbdxoXkpEX8tPD6Y,399
@@ -130,8 +130,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.29.1.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.29.1.dist-info/METADATA,sha256=hUTGMe5XJuut9wRLCNRQlHeO6q37d1LjSk5yilKUFMI,8978
-sqlframe-3.29.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.29.1.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.29.1.dist-info/RECORD,,
+sqlframe-3.31.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.31.0.dist-info/METADATA,sha256=9D8GGMLw5XslYPUw8U_cEa_Ab4NgB1FRTXW0Wa6lBb0,8987
+sqlframe-3.31.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.31.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.31.0.dist-info/RECORD,,