sqlframe 3.38.1__tar.gz → 3.39.0__tar.gz

{sqlframe-3.38.1 → sqlframe-3.39.0}/.github/workflows/main.workflow.yaml

@@ -27,7 +27,7 @@ jobs:
         python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
     - name: Checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
     - name: Install Python
       uses: actions/setup-python@v5
       with:

{sqlframe-3.38.1 → sqlframe-3.39.0}/.github/workflows/publish.workflow.yaml

@@ -14,7 +14,7 @@ jobs:
       url: https://pypi.org/p/sqlframe
     steps:
     - name: Checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
     - name: Package
       run: make package
     - name: Publish

{sqlframe-3.38.1 → sqlframe-3.39.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.38.1
+Version: 3.39.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.38.1 → sqlframe-3.39.0}/docs/bigquery.md

@@ -382,6 +382,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
+* [date_from_unix_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_from_unix_date.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)

{sqlframe-3.38.1 → sqlframe-3.39.0}/docs/duckdb.md

@@ -341,6 +341,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
+* [date_from_unix_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_from_unix_date.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [day](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.day.html)

{sqlframe-3.38.1 → sqlframe-3.39.0}/docs/postgres.md

@@ -346,6 +346,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
+* [date_from_unix_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_from_unix_date.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
     * Rounded whole number is returned

{sqlframe-3.38.1 → sqlframe-3.39.0}/docs/snowflake.md

@@ -375,6 +375,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
+* [date_from_unix_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_from_unix_date.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)

{sqlframe-3.38.1 → sqlframe-3.39.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.7",
+        "sqlglot>=24.0.0,<27.8",
         "typing_extensions",
     ],
     extras_require={
@@ -71,7 +71,7 @@ setup(
             "redshift_connector>=2.1.1,<2.2.0",
         ],
         "snowflake": [
-            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.17",
+            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.18",
         ],
         "spark": [
             "pyspark>=2,<3.6",

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.38.1'
-__version_tuple__ = version_tuple = (3, 38, 1)
+__version__ = version = '3.39.0'
+__version_tuple__ = version_tuple = (3, 39, 0)

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe/base/functions.py

@@ -2318,6 +2318,14 @@ def array_distinct(col: ColumnOrName) -> Column:
     if session._is_bigquery:
         return array_distinct_bgutil(col)
 
+    if session._is_duckdb:
+        # DuckDB's array_distinct removes nulls, but we need to preserve them
+        # Check if original array contains null and append it back if needed
+        original_col = Column.ensure_col(col)
+        distinct_result = Column.invoke_anonymous_function(col, "ARRAY_DISTINCT")
+        has_null = array_position(original_col, lit(None)) > lit(0)
+        return when(has_null, array_append(distinct_result, lit(None))).otherwise(distinct_result)
+
     return Column.invoke_anonymous_function(col, "ARRAY_DISTINCT")
 
 
@@ -2564,7 +2572,7 @@ def shuffle(col: ColumnOrName) -> Column:
 
 @meta(unsupported_engines="snowflake")
 def reverse(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "REVERSE")
+    return Column.invoke_expression_over_column(col, expression.Reverse)
 
 
 @meta(unsupported_engines=["bigquery", "postgres"])
@@ -3211,9 +3219,9 @@ def current_user() -> Column:
     return Column.invoke_expression_over_column(None, expression.CurrentUser)
 
 
-@meta(unsupported_engines="*")
+@meta()
 def date_from_unix_date(days: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(days, "date_from_unix_date")
+    return Column.invoke_expression_over_column(days, expression.DateFromUnixDate)
 
 
 @meta(unsupported_engines="*")
@@ -6623,7 +6631,7 @@ def unix_micros(col: ColumnOrName) -> Column:
 
     col = to_timestamp(col)
 
-    return Column.invoke_anonymous_function(col, "unix_micros")
+    return Column.invoke_expression_over_column(col, expression.UnixMicros)
 
 
 @meta()
@@ -6643,15 +6651,10 @@ def unix_millis(col: ColumnOrName) -> Column:
     """
     from sqlframe.base.function_alternatives import unix_millis_multiply_epoch
 
-    if (
-        _get_session()._is_bigquery
-        or _get_session()._is_duckdb
-        or _get_session()._is_postgres
-        or _get_session()._is_snowflake
-    ):
+    if _get_session()._is_duckdb or _get_session()._is_postgres or _get_session()._is_snowflake:
         return unix_millis_multiply_epoch(col)
 
-    return Column.invoke_anonymous_function(col, "unix_millis")
+    return Column.invoke_expression_over_column(col, expression.UnixMillis)
 
 
 @meta()

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe/base/session.py

@@ -12,7 +12,6 @@ from collections import defaultdict
 from functools import cached_property
 
 import sqlglot
-from dateutil.relativedelta import relativedelta
 from sqlglot import Dialect, exp
 from sqlglot.dialects.dialect import DialectType, NormalizationStrategy
 from sqlglot.expressions import parse_identifier
@@ -34,6 +33,7 @@ from sqlframe.base.table import _BaseTable
 from sqlframe.base.udf import _BaseUDFRegistration
 from sqlframe.base.util import (
     get_column_mapping_from_schema_input,
+    is_relativedelta_like,
     normalize_string,
     verify_pandas_installed,
 )
@@ -614,7 +614,7 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
             return [cls._to_value(x) for x in value]
         elif isinstance(value, datetime.datetime):
             return value.replace(tzinfo=None)
-        elif isinstance(value, relativedelta):
+        elif is_relativedelta_like(value):
             return datetime.timedelta(
                 days=value.days, hours=value.hours, minutes=value.minutes, seconds=value.seconds
             )

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe/base/util.py

@@ -524,3 +524,12 @@ def split_filepath(filepath: str) -> tuple[str, str]:
     if len(split_) == 2:  # noqa: PLR2004
         return split_[0] + "://", split_[1]
     return "", split_[0]
+
+
+def is_relativedelta_like(value: t.Any) -> bool:
+    return (
+        hasattr(value, "years")
+        and hasattr(value, "months")
+        and hasattr(value, "weeks")
+        and hasattr(value, "leapdays")
+    )

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.38.1
+Version: 3.39.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.38.1 → sqlframe-3.39.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<27.7,>=24.0.0
+sqlglot<27.8,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -52,7 +52,7 @@ psycopg2<3,>=2.8
 redshift_connector<2.2.0,>=2.1.1
 
 [snowflake]
-snowflake-connector-python[secure-local-storage]<3.17,>=3.10.0
+snowflake-connector-python[secure-local-storage]<3.18,>=3.10.0
 
 [spark]
 pyspark<3.6,>=2

{sqlframe-3.38.1 → sqlframe-3.39.0}/tests/integration/engines/test_int_functions.py

@@ -2290,16 +2290,21 @@ def test_array_remove(get_session_and_func, get_func):
 def test_array_distinct(get_session_and_func, get_func):
     session, array_distinct = get_session_and_func("array_distinct")
     lit = get_func("lit", session)
+    # BigQuery doesn't support null values in arrays
+    value3 = lit([1]) if isinstance(session, BigQuerySession) else lit([1, None])
     results = (
         session.range(1)
         .select(
             array_distinct(lit([1, 2, 3, 2])).alias("value"),
             array_distinct(lit([4, 5, 5, 4])).alias("value2"),
+            array_distinct(lit(value3)).alias("value3"),
         )
         .collect()
     )
     assert results[0][0] in ([1, 2, 3], [3, 2, 1])
     assert results[0][1] in ([4, 5], [5, 4])
+    if not isinstance(session, BigQuerySession):
+        assert results[0][2] in ([1, None], [None, 1])
 
 
 def test_array_intersect(get_session_and_func, get_func):
@@ -3670,9 +3675,10 @@ def test_current_timezone(get_session_and_func, get_func):
 def test_date_from_unix_date(get_session_and_func, get_func):
     session, date_from_unix_date = get_session_and_func("date_from_unix_date")
     lit = get_func("lit", session)
-    assert session.range(1).select(date_from_unix_date(lit(1))).first()[0] == datetime.date(
-        1970, 1, 2
-    )
+    assert session.range(1).select(date_from_unix_date(lit(1))).first()[0] in [
+        datetime.date(1970, 1, 2),
+        datetime.datetime(1970, 1, 2, 0, 0),
+    ]
 
 
 def test_date_part(get_session_and_func, get_func):

{sqlframe-3.38.1 → sqlframe-3.39.0}/tests/unit/standalone/test_functions.py

@@ -17,6 +17,7 @@ def test_invoke_anonymous(name, func):
     # make_interval - SQLGlot doesn't support week
     # to_char - convert to a cast that ignores the format provided
     # ltrim/rtrim - don't seem to convert correctly on some engines
+    # unix_micros - it is actually supported just an engine specific override uses an anonymous function
     ignore_funcs = {
         "array_size",
         "exists",
@@ -26,6 +27,7 @@ def test_invoke_anonymous(name, func):
         "rtrim",
         "ascii",
         "current_schema",
+        "unix_micros",
     }
     if "invoke_anonymous_function" in inspect.getsource(func) and name not in ignore_funcs:
         func = parse_one(f"{name}()", read="spark", error_level=ErrorLevel.IGNORE)