sqlframe 3.31.1__tar.gz → 3.31.3__tar.gz

{sqlframe-3.31.1 → sqlframe-3.31.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.31.1
+Version: 3.31.3
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.31.1 → sqlframe-3.31.3}/setup.py

@@ -36,7 +36,7 @@ setup(
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
-            "pyarrow>=10,<20",
+            "pyarrow>=10,<21",
             "pyspark>=2,<3.6",
             "pytest>=8.2.0,<8.4",
             "pytest-forked",
@@ -70,7 +70,7 @@ setup(
             "redshift_connector>=2.1.1,<2.2.0",
         ],
         "snowflake": [
-            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.15",
+            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.16",
         ],
         "spark": [
             "pyspark>=2,<3.6",

{sqlframe-3.31.1 → sqlframe-3.31.3}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.31.1'
-__version_tuple__ = version_tuple = (3, 31, 1)
+__version__ = version = '3.31.3'
+__version_tuple__ = version_tuple = (3, 31, 3)

{sqlframe-3.31.1 → sqlframe-3.31.3}/sqlframe/base/functions.py

@@ -625,11 +625,22 @@ def covar_samp(col1: ColumnOrName, col2: ColumnOrName) -> Column:
 def first(col: ColumnOrName, ignorenulls: t.Optional[bool] = None) -> Column:
     session = _get_session()
 
-    if session._is_duckdb:
-        ignorenulls = None
-
     this = Column.invoke_expression_over_column(col, expression.First)
     if ignorenulls:
+        if session._is_duckdb:
+            return Column(
+                expression.Filter(
+                    this=this.expression,
+                    expression=expression.Where(
+                        this=expression.Not(
+                            this=expression.Is(
+                                this=Column.ensure_col(col).expression,
+                                expression=expression.Null(),
+                            )
+                        )
+                    ),
+                )
+            )
         return Column.invoke_expression_over_column(this, expression.IgnoreNulls)
     return this
 
@@ -3266,12 +3277,57 @@ def find_in_set(str: ColumnOrName, str_array: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "find_in_set", str_array)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def first_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
+    """Returns the first value of `col` for a group of rows. It will return the first non-null
+    value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        target column to work on.
+    ignorenulls : :class:`~pyspark.sql.Column` or bool
+        if first value is null then look for first non-null value.
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        some value of `col` for a group of rows.
+
+    Examples
+    --------
+    >>> import pyspark.sql.functions as sf
+    >>> spark.createDataFrame(
+    ...     [(None, 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+    ... ).select(sf.first_value('a'), sf.first_value('b')).show()
+    +--------------+--------------+
+    |first_value(a)|first_value(b)|
+    +--------------+--------------+
+    |          NULL|             1|
+    +--------------+--------------+
+
+    >>> import pyspark.sql.functions as sf
+    >>> spark.createDataFrame(
+    ...     [(None, 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+    ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show()
+    +--------------+--------------+
+    |first_value(a)|first_value(b)|
+    +--------------+--------------+
+    |             a|             1|
+    +--------------+--------------+
+    """
+    session = _get_session()
+
     column = Column.invoke_expression_over_column(col, expression.FirstValue)
 
     if ignoreNulls:
-        return Column(expression.IgnoreNulls(this=column.column_expression))
+        column = Column(expression.IgnoreNulls(this=column.column_expression))
+
+    if session._is_duckdb:
+        agg_func = first(col, ignoreNulls)  # type: ignore
+        agg_func.expression._meta = agg_func.expression._meta or {}
+        agg_func.expression._meta["window_func"] = column.expression
+        return agg_func
     return column
 
 

{sqlframe-3.31.1 → sqlframe-3.31.3}/sqlframe/duckdb/session.py

@@ -41,13 +41,14 @@ class DuckDBSession(
 
     def __init__(self, conn: t.Optional[DuckDBPyConnection] = None, *args, **kwargs):
         import duckdb
+        from duckdb import InvalidInputException
         from duckdb.typing import VARCHAR
 
         if not hasattr(self, "_conn"):
             conn = conn or duckdb.connect()
             try:
                 conn.create_function("SOUNDEX", lambda x: soundex(x), return_type=VARCHAR)
-            except ImportError:
+            except (ImportError, InvalidInputException):
                 pass
 
             super().__init__(conn, *args, **kwargs)

{sqlframe-3.31.1 → sqlframe-3.31.3}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.31.1
+Version: 3.31.3
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.31.1 → sqlframe-3.31.3}/sqlframe.egg-info/requires.txt

@@ -18,7 +18,7 @@ pandas-stubs<3,>=2
 pandas<3,>=2
 pre-commit<5,>=3.7
 psycopg<4,>=3.1
-pyarrow<20,>=10
+pyarrow<21,>=10
 pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<8,>=6
@@ -51,7 +51,7 @@ psycopg2<3,>=2.8
 redshift_connector<2.2.0,>=2.1.1
 
 [snowflake]
-snowflake-connector-python[secure-local-storage]<3.15,>=3.10.0
+snowflake-connector-python[secure-local-storage]<3.16,>=3.10.0
 
 [spark]
 pyspark<3.6,>=2

{sqlframe-3.31.1 → sqlframe-3.31.3}/tests/integration/engines/test_int_functions.py

@@ -3753,8 +3753,10 @@ def test_find_in_set(get_session_and_func, get_func):
     assert df.select(find_in_set(df.a, df.b).alias("r")).first()[0] == 3
 
 
-def test_first_value(get_session_and_func, get_func):
+def test_first_value(get_session_and_func, get_func, get_window):
     session, first_value = get_session_and_func("first_value")
+    col = get_func("col", session)
+    Window = get_window(session)
     assert session.createDataFrame(
         [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
     ).select(first_value("a"), first_value("b")).collect() == [Row(value1=None, value2=1)]
@@ -3763,6 +3765,41 @@ def test_first_value(get_session_and_func, get_func):
     ).select(first_value("a", True), first_value("b", True)).collect() == [
         Row(value1="a", value2=1)
     ]
+    data = {
+        "b": [None, None, "a", None, "b", "c", None, None, None, "d"],
+        "idx": list(range(10)),
+    }
+    window = (
+        Window().orderBy(col("idx").asc_nulls_first()).rowsBetween(Window.unboundedPreceding, 0)
+    )
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", first_value("b", ignoreNulls=True).over(window)
+    ).collect() == [
+        Row(b=None, idx=0, bf=None),
+        Row(b=None, idx=1, bf=None),
+        Row(b="a", idx=2, bf="a"),
+        Row(b=None, idx=3, bf="a"),
+        Row(b="b", idx=4, bf="a"),
+        Row(b="c", idx=5, bf="a"),
+        Row(b=None, idx=6, bf="a"),
+        Row(b=None, idx=7, bf="a"),
+        Row(b=None, idx=8, bf="a"),
+        Row(b="d", idx=9, bf="a"),
+    ]
+    assert session.createDataFrame(pd.DataFrame(data)).withColumn(
+        "bf", first_value("b").over(window)
+    ).collect() == [
+        Row(b=None, idx=0, bf=None),
+        Row(b=None, idx=1, bf=None),
+        Row(b="a", idx=2, bf=None),
+        Row(b=None, idx=3, bf=None),
+        Row(b="b", idx=4, bf=None),
+        Row(b="c", idx=5, bf=None),
+        Row(b=None, idx=6, bf=None),
+        Row(b=None, idx=7, bf=None),
+        Row(b=None, idx=8, bf=None),
+        Row(b="d", idx=9, bf=None),
+    ]
 
 
 def test_get(get_session_and_func, get_func):