sqlframe 3.31.0__tar.gz → 3.31.2__tar.gz

{sqlframe-3.31.0 → sqlframe-3.31.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.31.0
+Version: 3.31.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.31.0 → sqlframe-3.31.2}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.31.0'
-__version_tuple__ = version_tuple = (3, 31, 0)
+__version__ = version = '3.31.2'
+__version_tuple__ = version_tuple = (3, 31, 2)

{sqlframe-3.31.0 → sqlframe-3.31.2}/sqlframe/base/column.py

@@ -430,8 +430,9 @@ class Column:
         )
 
     def over(self, window: WindowSpec) -> Column:
+        column_expression = self.column_expression.meta.get("window_func", self.column_expression)
         window_expression = window.expression.copy()
-        window_expression.set("this", self.column_expression)
+        window_expression.set("this", column_expression)
         return Column(window_expression)
 
     def getItem(self, key: t.Any) -> Column:

{sqlframe-3.31.0 → sqlframe-3.31.2}/sqlframe/base/functions.py

@@ -625,11 +625,22 @@ def covar_samp(col1: ColumnOrName, col2: ColumnOrName) -> Column:
 def first(col: ColumnOrName, ignorenulls: t.Optional[bool] = None) -> Column:
     session = _get_session()
 
-    if session._is_duckdb:
-        ignorenulls = None
-
     this = Column.invoke_expression_over_column(col, expression.First)
     if ignorenulls:
+        if session._is_duckdb:
+            return Column(
+                expression.Filter(
+                    this=this.expression,
+                    expression=expression.Where(
+                        this=expression.Not(
+                            this=expression.Is(
+                                this=Column.ensure_col(col).expression,
+                                expression=expression.Null(),
+                            )
+                        )
+                    ),
+                )
+            )
         return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
     return this
 
@@ -3266,12 +3277,57 @@ def find_in_set(str: ColumnOrName, str_array: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "find_in_set", str_array)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def first_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
+    """Returns the first value of `col` for a group of rows. It will return the first non-null
+    value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        target column to work on.
+    ignorenulls : :class:`~pyspark.sql.Column` or bool
+        if first value is null then look for first non-null value.
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        some value of `col` for a group of rows.
+
+    Examples
+    --------
+    >>> import pyspark.sql.functions as sf
+    >>> spark.createDataFrame(
+    ...     [(None, 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+    ... ).select(sf.first_value('a'), sf.first_value('b')).show()
+    +--------------+--------------+
+    |first_value(a)|first_value(b)|
+    +--------------+--------------+
+    |          NULL|             1|
+    +--------------+--------------+
+
+    >>> import pyspark.sql.functions as sf
+    >>> spark.createDataFrame(
+    ...     [(None, 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+    ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show()
+    +--------------+--------------+
+    |first_value(a)|first_value(b)|
+    +--------------+--------------+
+    |             a|             1|
+    +--------------+--------------+
+    """
+    session = _get_session()
+
     column = Column.invoke_expression_over_column(col, expression.FirstValue)
 
     if ignoreNulls:
-        return Column(expression.IgnoreNulls(this=column.column_expression))
+        column = Column(expression.IgnoreNulls(this=column.column_expression))
+
+    if session._is_duckdb:
+        agg_func = first(col, ignoreNulls)  # type: ignore
+        agg_func.expression._meta = agg_func.expression._meta or {}
+        agg_func.expression._meta["window_func"] = column.expression
+        return agg_func
     return column
 
 
@@ -3930,13 +3986,16 @@ def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]]
     """
     session = _get_session()
 
-    if session._is_duckdb:
-        return last(col, ignoreNulls)  # type: ignore
-
     column = Column.invoke_expression_over_column(col, expression.LastValue)
 
     if ignoreNulls:
-        return Column(expression.IgnoreNulls(this=column.column_expression))
+        column = Column(expression.IgnoreNulls(this=column.column_expression))
+
+    if session._is_duckdb:
+        agg_func = last(col, ignoreNulls)  # type: ignore
+        agg_func.expression._meta = agg_func.expression._meta or {}
+        agg_func.expression._meta["window_func"] = column.expression
+        return agg_func
     return column
 
 

{sqlframe-3.31.0 → sqlframe-3.31.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.31.0
+Version: 3.31.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.31.0 → sqlframe-3.31.2}/tests/integration/engines/test_int_functions.py

@@ -6,6 +6,7 @@ import typing as t
 from collections import Counter
 from decimal import Decimal
 
+import pandas as pd
 import pytest
 import pytz
 from pyspark.sql import SparkSession as PySparkSession
@@ -3752,8 +3753,10 @@ def test_find_in_set(get_session_and_func, get_func):
     assert df.select(find_in_set(df.a, df.b).alias("r")).first()[0] == 3
 
 
-def test_first_value(get_session_and_func, get_func):
+def test_first_value(get_session_and_func, get_func, get_window):
     session, first_value = get_session_and_func("first_value")
+    col = get_func("col", session)
+    Window = get_window(session)
     assert session.createDataFrame(
         [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
     ).select(first_value("a"), first_value("b")).collect() == [Row(value1=None, value2=1)]
@@ -3762,6 +3765,41 @@ def test_first_value(get_session_and_func, get_func):
     ).select(first_value("a", True), first_value("b", True)).collect() == [
         Row(value1="a", value2=1)
     ]
+    data = {
+        "b": [None, None, "a", None, "b", "c", None, None, None, "d"],
+        "idx": list(range(10)),
+    }
+    window = (
+        Window().orderBy(col("idx").asc_nulls_first()).rowsBetween(Window.unboundedPreceding, 0)
+    )
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", first_value("b", ignoreNulls=True).over(window)
+    ).collect() == [
+        Row(b=None, idx=0, bf=None),
+        Row(b=None, idx=1, bf=None),
+        Row(b="a", idx=2, bf="a"),
+        Row(b=None, idx=3, bf="a"),
+        Row(b="b", idx=4, bf="a"),
+        Row(b="c", idx=5, bf="a"),
+        Row(b=None, idx=6, bf="a"),
+        Row(b=None, idx=7, bf="a"),
+        Row(b=None, idx=8, bf="a"),
+        Row(b="d", idx=9, bf="a"),
+    ]
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", first_value("b").over(window)
+    ).collect() == [
+        Row(b=None, idx=0, bf=None),
+        Row(b=None, idx=1, bf=None),
+        Row(b="a", idx=2, bf=None),
+        Row(b=None, idx=3, bf=None),
+        Row(b="b", idx=4, bf=None),
+        Row(b="c", idx=5, bf=None),
+        Row(b=None, idx=6, bf=None),
+        Row(b=None, idx=7, bf=None),
+        Row(b=None, idx=8, bf=None),
+        Row(b="d", idx=9, bf=None),
+    ]
 
 
 def test_get(get_session_and_func, get_func):
@@ -3959,14 +3997,51 @@ def test_json_object_keys(get_session_and_func, get_func):
     ]
 
 
-def test_last_value(get_session_and_func, get_func):
+def test_last_value(get_session_and_func, get_func, get_window):
     session, last_value = get_session_and_func("last_value")
+    col = get_func("col", session)
+    Window = get_window(session)
     assert session.createDataFrame(
         [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
     ).select(last_value("a"), last_value("b")).collect() == [Row(value1=None, value2=2)]
     assert session.createDataFrame(
         [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
     ).select(last_value("a", True), last_value("b", True)).collect() == [Row(value1="b", value2=2)]
+    data = {
+        "b": ["a", None, None, None, "b", "c", None, None, None, "d"],
+        "idx": list(range(10)),
+    }
+    window = (
+        Window().orderBy(col("idx").asc_nulls_first()).rowsBetween(Window.unboundedPreceding, 0)
+    )
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", last_value("b", ignoreNulls=True).over(window)
+    ).collect() == [
+        Row(b="a", idx=0, bf="a"),
+        Row(b=None, idx=1, bf="a"),
+        Row(b=None, idx=2, bf="a"),
+        Row(b=None, idx=3, bf="a"),
+        Row(b="b", idx=4, bf="b"),
+        Row(b="c", idx=5, bf="c"),
+        Row(b=None, idx=6, bf="c"),
+        Row(b=None, idx=7, bf="c"),
+        Row(b=None, idx=8, bf="c"),
+        Row(b="d", idx=9, bf="d"),
+    ]
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", last_value("b").over(window)
+    ).collect() == [
+        Row(b="a", idx=0, bf="a"),
+        Row(b=None, idx=1, bf=None),
+        Row(b=None, idx=2, bf=None),
+        Row(b=None, idx=3, bf=None),
+        Row(b="b", idx=4, bf="b"),
+        Row(b="c", idx=5, bf="c"),
+        Row(b=None, idx=6, bf=None),
+        Row(b=None, idx=7, bf=None),
+        Row(b=None, idx=8, bf=None),
+        Row(b="d", idx=9, bf="d"),
+    ]
 
 
 def test_lcase(get_session_and_func, get_func):