sqlframe 3.38.0__tar.gz → 3.38.2__tar.gz

{sqlframe-3.38.0 → sqlframe-3.38.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.38.0
+Version: 3.38.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.38.0 → sqlframe-3.38.2}/docs/bigquery.md

@@ -365,6 +365,10 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [cosh](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cosh.html)
 * [cot](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cot.html)
 * [count](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.count.html)
+* [countDistinct](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.countDistinct.html)
+  * Can only provide a single column 
+* [count_distinct](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.count_distinct.html)
+  * Can only provide a single column 
 * [count_if](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.count_if.html)
 * [covar_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.covar_pop.html)
 * [covar_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.covar_samp.html)

{sqlframe-3.38.0 → sqlframe-3.38.2}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.4",
+        "sqlglot>=24.0.0,<27.7",
         "typing_extensions",
     ],
     extras_require={

{sqlframe-3.38.0 → sqlframe-3.38.2}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.38.0'
-__version_tuple__ = version_tuple = (3, 38, 0)
+__version__ = version = '3.38.2'
+__version_tuple__ = version_tuple = (3, 38, 2)

{sqlframe-3.38.0 → sqlframe-3.38.2}/sqlframe/base/functions.py

@@ -14,6 +14,7 @@ from sqlglot.helper import flatten as _flatten
 
 from sqlframe.base.column import Column
 from sqlframe.base.decorators import func_metadata as meta
+from sqlframe.base.exceptions import UnsupportedOperationError
 from sqlframe.base.util import (
     get_func_from_session,
 )
@@ -81,9 +82,13 @@ def least(*cols: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(cols[0], expression.Least)
 
 
-@meta(unsupported_engines="bigquery")
+@meta()
 def count_distinct(col: ColumnOrName, *cols: ColumnOrName) -> Column:
     columns = [Column.ensure_col(x) for x in [col] + list(cols)]
+    if len(columns) > 1 and _get_session()._is_bigquery:
+        raise UnsupportedOperationError(
+            "BigQuery does not support multiple columns in countDistinct"
+        )
     return Column(
         expression.Count(
             this=expression.Distinct(expressions=[x.column_expression for x in columns])
@@ -2313,6 +2318,14 @@ def array_distinct(col: ColumnOrName) -> Column:
     if session._is_bigquery:
         return array_distinct_bgutil(col)
 
+    if session._is_duckdb:
+        # DuckDB's array_distinct removes nulls, but we need to preserve them
+        # Check if original array contains null and append it back if needed
+        original_col = Column.ensure_col(col)
+        distinct_result = Column.invoke_anonymous_function(col, "ARRAY_DISTINCT")
+        has_null = array_position(original_col, lit(None)) > lit(0)
+        return when(has_null, array_append(distinct_result, lit(None))).otherwise(distinct_result)
+
     return Column.invoke_anonymous_function(col, "ARRAY_DISTINCT")
 
 

{sqlframe-3.38.0 → sqlframe-3.38.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.38.0
+Version: 3.38.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.38.0 → sqlframe-3.38.2}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<27.4,>=24.0.0
+sqlglot<27.7,>=24.0.0
 typing_extensions
 
 [bigquery]

{sqlframe-3.38.0 → sqlframe-3.38.2}/tests/integration/engines/test_int_functions.py

@@ -879,7 +879,11 @@ def test_count_distinct(get_session_and_func):
     df1 = session.createDataFrame([1, 1, 3], "int")
     df2 = session.createDataFrame([1, 2], "int")
     df_joined = df1.join(df2)
-    assert df_joined.select(count_distinct(df1.value, df2.value)).collect() == [Row(value=4)]
+    if isinstance(session, BigQuerySession):
+        # BigQuery does not support count_distinct with multiple columns
+        assert df_joined.select(count_distinct(df1.value)).collect() == [Row(value=2)]
+    else:
+        assert df_joined.select(count_distinct(df1.value, df2.value)).collect() == [Row(value=4)]
 
 
 def test_first(get_session_and_func):
@@ -2286,16 +2290,21 @@ def test_array_remove(get_session_and_func, get_func):
 def test_array_distinct(get_session_and_func, get_func):
     session, array_distinct = get_session_and_func("array_distinct")
     lit = get_func("lit", session)
+    # BigQuery doesn't support null values in arrays
+    value3 = lit([1]) if isinstance(session, BigQuerySession) else lit([1, None])
     results = (
         session.range(1)
         .select(
             array_distinct(lit([1, 2, 3, 2])).alias("value"),
             array_distinct(lit([4, 5, 5, 4])).alias("value2"),
+            array_distinct(lit(value3)).alias("value3"),
         )
         .collect()
     )
     assert results[0][0] in ([1, 2, 3], [3, 2, 1])
     assert results[0][1] in ([4, 5], [5, 4])
+    if not isinstance(session, BigQuerySession):
+        assert results[0][2] in ([1, None], [None, 1])
 
 
 def test_array_intersect(get_session_and_func, get_func):