sqlframe 3.30.0__tar.gz → 3.31.1__tar.gz

{sqlframe-3.30.0 → sqlframe-3.31.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.30.0
+Version: 3.31.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.30.0 → sqlframe-3.31.1}/docs/duckdb.md

@@ -379,6 +379,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [kurtosis](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.kurtosis.html)
 * [lag](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lag.html)
 * [last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last.html)
+* [last_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_value.html)
 * [last_day](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_day.html)
 * [lcase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lcase.html)
 * [lead](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lead.html)

{sqlframe-3.30.0 → sqlframe-3.31.1}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.16",
+        "sqlglot>=24.0.0,<26.17",
         "typing_extensions",
     ],
     extras_require={

{sqlframe-3.30.0 → sqlframe-3.31.1}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.30.0'
-__version_tuple__ = version_tuple = (3, 30, 0)
+__version__ = version = '3.31.1'
+__version_tuple__ = version_tuple = (3, 31, 1)

{sqlframe-3.30.0 → sqlframe-3.31.1}/sqlframe/base/column.py

@@ -430,8 +430,9 @@ class Column:
         )
 
     def over(self, window: WindowSpec) -> Column:
+        column_expression = self.column_expression.meta.get("window_func", self.column_expression)
         window_expression = window.expression.copy()
-        window_expression.set("this", self.column_expression)
+        window_expression.set("this", column_expression)
         return Column(window_expression)
 
     def getItem(self, key: t.Any) -> Column:

{sqlframe-3.30.0 → sqlframe-3.31.1}/sqlframe/base/functions.py

@@ -678,7 +678,22 @@ def isnull(col: ColumnOrName) -> Column:
 def last(col: ColumnOrName, ignorenulls: t.Optional[bool] = None) -> Column:
     this = Column.invoke_expression_over_column(col, expression.Last)
     if ignorenulls:
-        return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
+        if _get_session()._is_duckdb:
+            return Column(
+                expression.Filter(
+                    this=this.expression,
+                    expression=expression.Where(
+                        this=expression.Not(
+                            this=expression.Is(
+                                this=Column.ensure_col(col).expression,
+                                expression=expression.Null(),
+                            )
+                        )
+                    ),
+                )
+            )
+        else:
+            return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
     return this
 
 
@@ -3872,7 +3887,7 @@ def json_object_keys(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "json_object_keys")
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
     """Returns the last value of `col` for a group of rows. It will return the last non-null
     value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.
@@ -3913,10 +3928,18 @@ def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]]
     |            b|            2|
     +-------------+-------------+
     """
+    session = _get_session()
+
     column = Column.invoke_expression_over_column(col, expression.LastValue)
 
     if ignoreNulls:
-        return Column(expression.IgnoreNulls(this=column.column_expression))
+        column = Column(expression.IgnoreNulls(this=column.column_expression))
+
+    if session._is_duckdb:
+        agg_func = last(col, ignoreNulls)  # type: ignore
+        agg_func.expression._meta = agg_func.expression._meta or {}
+        agg_func.expression._meta["window_func"] = column.expression
+        return agg_func
     return column
 
 

{sqlframe-3.30.0 → sqlframe-3.31.1}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.30.0
+Version: 3.31.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.30.0 → sqlframe-3.31.1}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.16,>=24.0.0
+sqlglot<26.17,>=24.0.0
 typing_extensions
 
 [bigquery]

{sqlframe-3.30.0 → sqlframe-3.31.1}/tests/integration/engines/test_int_functions.py

@@ -6,6 +6,7 @@ import typing as t
 from collections import Counter
 from decimal import Decimal
 
+import pandas as pd
 import pytest
 import pytz
 from pyspark.sql import SparkSession as PySparkSession
@@ -3959,14 +3960,51 @@ def test_json_object_keys(get_session_and_func, get_func):
     ]
 
 
-def test_last_value(get_session_and_func, get_func):
+def test_last_value(get_session_and_func, get_func, get_window):
     session, last_value = get_session_and_func("last_value")
+    col = get_func("col", session)
+    Window = get_window(session)
     assert session.createDataFrame(
         [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
     ).select(last_value("a"), last_value("b")).collect() == [Row(value1=None, value2=2)]
     assert session.createDataFrame(
         [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
     ).select(last_value("a", True), last_value("b", True)).collect() == [Row(value1="b", value2=2)]
+    data = {
+        "b": ["a", None, None, None, "b", "c", None, None, None, "d"],
+        "idx": list(range(10)),
+    }
+    window = (
+        Window().orderBy(col("idx").asc_nulls_first()).rowsBetween(Window.unboundedPreceding, 0)
+    )
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", last_value("b", ignoreNulls=True).over(window)
+    ).collect() == [
+        Row(b="a", idx=0, bf="a"),
+        Row(b=None, idx=1, bf="a"),
+        Row(b=None, idx=2, bf="a"),
+        Row(b=None, idx=3, bf="a"),
+        Row(b="b", idx=4, bf="b"),
+        Row(b="c", idx=5, bf="c"),
+        Row(b=None, idx=6, bf="c"),
+        Row(b=None, idx=7, bf="c"),
+        Row(b=None, idx=8, bf="c"),
+        Row(b="d", idx=9, bf="d"),
+    ]
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", last_value("b").over(window)
+    ).collect() == [
+        Row(b="a", idx=0, bf="a"),
+        Row(b=None, idx=1, bf=None),
+        Row(b=None, idx=2, bf=None),
+        Row(b=None, idx=3, bf=None),
+        Row(b="b", idx=4, bf="b"),
+        Row(b="c", idx=5, bf="c"),
+        Row(b=None, idx=6, bf=None),
+        Row(b=None, idx=7, bf=None),
+        Row(b=None, idx=8, bf=None),
+        Row(b="d", idx=9, bf="d"),
+    ]
 
 
 def test_lcase(get_session_and_func, get_func):

{sqlframe-3.30.0 → sqlframe-3.31.1}/tests/unit/standalone/test_functions.py

@@ -1223,8 +1223,8 @@ def test_date_format(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.year("cola"), "YEAR(TO_DATE(cola))"),
-        (SF.year(SF.col("cola")), "YEAR(TO_DATE(cola))"),
+        (SF.year("cola"), "YEAR(cola)"),
+        (SF.year(SF.col("cola")), "YEAR(cola)"),
     ],
 )
 def test_year(expression, expected):
@@ -1245,8 +1245,8 @@ def test_quarter(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.month("cola"), "MONTH(TO_DATE(cola))"),
-        (SF.month(SF.col("cola")), "MONTH(TO_DATE(cola))"),
+        (SF.month("cola"), "MONTH(cola)"),
+        (SF.month(SF.col("cola")), "MONTH(cola)"),
     ],
 )
 def test_month(expression, expected):