sqlframe 3.36.3__tar.gz → 3.37.0__tar.gz

{sqlframe-3.36.3 → sqlframe-3.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.36.3
+Version: 3.37.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.36.3 → sqlframe-3.37.0}/docs/bigquery.md

@@ -332,8 +332,12 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [avg](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.avg.html)
 * [base64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.base64.html)
 * [bin](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bin.html)
+* [bit_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_and.html)
+* [bit_count](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_count.html)
 * [bit_length](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_length.html)
     * Symbols are not supported
+* [bit_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_or.html)
+* [bit_xor](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_xor.html)
 * [bitwiseNOT](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwiseNOT.html)
 * [bitwise_not](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwise_not.html)
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)
@@ -416,6 +420,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [isnull](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.isnull.html)
 * [lag](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lag.html)
 * [last_day](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_day.html)
+* [last_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_value.html)
+  * Must be used with a window function
 * [lcase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lcase.html)
 * [lead](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lead.html)
 * [least](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.least.html)

{sqlframe-3.36.3 → sqlframe-3.37.0}/docs/duckdb.md

@@ -295,7 +295,11 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [avg](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.avg.html)
 * [base64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.base64.html)
 * [bin](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bin.html)
+* [bit_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_and.html)
+* [bit_count](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_count.html)
 * [bit_length](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_length.html)
+* [bit_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_or.html)
+* [bit_xor](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_xor.html)
 * [bitwiseNOT](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwiseNOT.html)
 * [bitwise_not](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwise_not.html)
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)

{sqlframe-3.36.3 → sqlframe-3.37.0}/docs/postgres.md

@@ -304,7 +304,10 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [atanh](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.atanh.html)
 * [avg](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.avg.html)
 * [base64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.base64.html)
+* [bit_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_and.html)
 * [bit_length](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_length.html)
+* [bit_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_or.html)
+* [bit_xor](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bit_xor.html)
 * [bitwiseNOT](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwiseNOT.html)
 * [bitwise_not](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwise_not.html)
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)

{sqlframe-3.36.3 → sqlframe-3.37.0}/docs/snowflake.md

@@ -394,6 +394,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [expr](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.expr.html)
 * [extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.extract.html)
 * [factorial](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.factorial.html)
+* [first_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.first_value.html)
+  * Must be used with a window function and `ignoreNulls` can behave slightly different in certain situations
 * [flatten](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.flatten.html)
 * [floor](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.floor.html)
 * [format_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.format_number.html)
@@ -414,6 +416,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [kurtosis](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.kurtosis.html)
 * [lag](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lag.html)
 * [last_day](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_day.html)
+* [last_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.last_value.html)
+  * Must be used with a window function
 * [lead](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.lead.html)
 * [least](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.least.html)
 * [left](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.left.html)

{sqlframe-3.36.3 → sqlframe-3.37.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.34",
+        "sqlglot>=24.0.0,<27.2",
         "typing_extensions",
     ],
     extras_require={
@@ -31,12 +31,12 @@ setup(
         "dev": [
             "duckdb>=1.2,<1.4",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.17",
+            "mypy>=1.10.0,<1.18",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
-            "pyarrow>=10,<21",
+            "pyarrow>=10,<22",
             "pyspark>=2,<3.6",
             "pytest>=8.2.0,<8.5",
             "pytest-forked",

{sqlframe-3.36.3 → sqlframe-3.37.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.36.3'
-__version_tuple__ = version_tuple = (3, 36, 3)
+__version__ = version = '3.37.0'
+__version_tuple__ = version_tuple = (3, 37, 0)

{sqlframe-3.36.3 → sqlframe-3.37.0}/sqlframe/base/functions.py

@@ -2110,24 +2110,24 @@ def array_size(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.ArraySize)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines="snowflake")
 def bit_and(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "BIT_AND")
+    return Column.invoke_expression_over_column(col, expression.BitwiseAndAgg)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines="snowflake")
 def bit_or(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "BIT_OR")
+    return Column.invoke_expression_over_column(col, expression.BitwiseOrAgg)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines="snowflake")
 def bit_xor(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "BIT_XOR")
+    return Column.invoke_expression_over_column(col, expression.BitwiseXorAgg)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["postgres", "snowflake"])
 def bit_count(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "BIT_COUNT")
+    return Column.invoke_expression_over_column(col, expression.BitwiseCountAgg)
 
 
 @meta(unsupported_engines="*")
@@ -3293,7 +3293,7 @@ def find_in_set(str: ColumnOrName, str_array: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "find_in_set", str_array)
 
 
-@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
+@meta(unsupported_engines=["bigquery", "postgres"])
 def first_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
     """Returns the first value of `col` for a group of rows. It will return the first non-null
     value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.
@@ -3959,7 +3959,7 @@ def json_object_keys(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "json_object_keys")
 
 
-@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
+@meta(unsupported_engines="postgres")
 def last_value(col: ColumnOrName, ignoreNulls: t.Optional[t.Union[bool, Column]] = None) -> Column:
     """Returns the last value of `col` for a group of rows. It will return the last non-null
     value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned.

{sqlframe-3.36.3 → sqlframe-3.37.0}/sqlframe/base/types.py

@@ -42,7 +42,7 @@ class DataTypeWithLength(DataType):
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self.length})"
 
-    def __str__(self) -> str:
+    def simpleString(self) -> str:
         return f"{self.typeName()}({self.length})"
 
 

{sqlframe-3.36.3 → sqlframe-3.37.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.36.3
+Version: 3.37.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.36.3 → sqlframe-3.37.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.34,>=24.0.0
+sqlglot<27.2,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -12,13 +12,13 @@ databricks-sql-connector[pyarrow]<5,>=3.6
 [dev]
 duckdb<1.4,>=1.2
 findspark<3,>=2
-mypy<1.17,>=1.10.0
+mypy<1.18,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
 pre-commit<5,>=3.7
 psycopg<4,>=3.1
-pyarrow<21,>=10
+pyarrow<22,>=10
 pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<8,>=6

{sqlframe-3.36.3 → sqlframe-3.37.0}/tests/integration/engines/test_int_functions.py

@@ -1572,6 +1572,16 @@ def test_timestamp_add(get_session_and_func, get_func):
         actual = row[0].replace(tzinfo=None) if row[0] else None
         assert actual == expected_days[i], f"Day subtraction failed for row {i}"
 
+    # Test adding quarters
+    result = df.select(timestamp_add("quarter", lit(2), "ts")).collect()
+    expected_quarters = [
+        datetime.datetime(2016, 9, 11, 9, 0, 7),
+        datetime.datetime(2024, 10, 2, 9, 0, 7),
+    ]
+    for i, row in enumerate(result):
+        actual = row[0].replace(tzinfo=None) if row[0] else None
+        assert actual == expected_quarters[i], f"Quarter addition failed for row {i}"
+
 
 def test_window(get_session_and_func, get_func):
     session, window = get_session_and_func("window")
@@ -3825,14 +3835,15 @@ def test_first_value(get_session_and_func, get_func, get_window):
     session, first_value = get_session_and_func("first_value")
     col = get_func("col", session)
     Window = get_window(session)
-    assert session.createDataFrame(
-        [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
-    ).select(first_value("a"), first_value("b")).collect() == [Row(value1=None, value2=1)]
-    assert session.createDataFrame(
-        [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
-    ).select(first_value("a", True), first_value("b", True)).collect() == [
-        Row(value1="a", value2=1)
-    ]
+    if not isinstance(session, (BigQuerySession, SnowflakeSession)):
+        assert session.createDataFrame(
+            [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
+        ).select(first_value("a"), first_value("b")).collect() == [Row(value1=None, value2=1)]
+        assert session.createDataFrame(
+            [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"]
+        ).select(first_value("a", True), first_value("b", True)).collect() == [
+            Row(value1="a", value2=1)
+        ]
     data = {
         "b": [None, None, "a", None, "b", "c", None, None, None, "d"],
         "idx": list(range(10)),
@@ -3843,8 +3854,8 @@ def test_first_value(get_session_and_func, get_func, get_window):
     assert session.createDataFrame(pd.DataFrame(data)).withColumn(
         "bf", first_value("b", ignoreNulls=True).over(window)
     ).collect() == [
-        Row(b=None, idx=0, bf=None),
-        Row(b=None, idx=1, bf=None),
+        Row(b=None, idx=0, bf=None if not isinstance(session, SnowflakeSession) else "a"),
+        Row(b=None, idx=1, bf=None if not isinstance(session, SnowflakeSession) else "a"),
         Row(b="a", idx=2, bf="a"),
         Row(b=None, idx=3, bf="a"),
         Row(b="b", idx=4, bf="a"),
@@ -4069,12 +4080,15 @@ def test_last_value(get_session_and_func, get_func, get_window):
     session, last_value = get_session_and_func("last_value")
     col = get_func("col", session)
     Window = get_window(session)
-    assert session.createDataFrame(
-        [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
-    ).select(last_value("a"), last_value("b")).collect() == [Row(value1=None, value2=2)]
-    assert session.createDataFrame(
-        [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
-    ).select(last_value("a", True), last_value("b", True)).collect() == [Row(value1="b", value2=2)]
+    if not isinstance(session, (BigQuerySession, SnowflakeSession)):
+        assert session.createDataFrame(
+            [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+        ).select(last_value("a"), last_value("b")).collect() == [Row(value1=None, value2=2)]
+        assert session.createDataFrame(
+            [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"]
+        ).select(last_value("a", True), last_value("b", True)).collect() == [
+            Row(value1="b", value2=2)
+        ]
     data = {
         "b": ["a", None, None, None, "b", "c", None, None, None, "d"],
         "idx": list(range(10)),

{sqlframe-3.36.3 → sqlframe-3.37.0}/tests/unit/standalone/test_functions.py

@@ -1256,8 +1256,8 @@ def test_month(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.dayofweek("cola"), "DAYOFWEEK(TO_DATE(cola))"),
-        (SF.dayofweek(SF.col("cola")), "DAYOFWEEK(TO_DATE(cola))"),
+        (SF.dayofweek("cola"), "DAYOFWEEK(cola)"),
+        (SF.dayofweek(SF.col("cola")), "DAYOFWEEK(cola)"),
     ],
 )
 def test_dayofweek(expression, expected):
@@ -1267,8 +1267,8 @@ def test_dayofweek(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.dayofmonth("cola"), "DAYOFMONTH(TO_DATE(cola))"),
-        (SF.dayofmonth(SF.col("cola")), "DAYOFMONTH(TO_DATE(cola))"),
+        (SF.dayofmonth("cola"), "DAYOFMONTH(cola)"),
+        (SF.dayofmonth(SF.col("cola")), "DAYOFMONTH(cola)"),
     ],
 )
 def test_dayofmonth(expression, expected):
@@ -1278,8 +1278,8 @@ def test_dayofmonth(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.dayofyear("cola"), "DAYOFYEAR(TO_DATE(cola))"),
-        (SF.dayofyear(SF.col("cola")), "DAYOFYEAR(TO_DATE(cola))"),
+        (SF.dayofyear("cola"), "DAYOFYEAR(cola)"),
+        (SF.dayofyear(SF.col("cola")), "DAYOFYEAR(cola)"),
     ],
 )
 def test_dayofyear(expression, expected):
@@ -1322,8 +1322,8 @@ def test_second(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.weekofyear("cola"), "WEEKOFYEAR(TO_DATE(cola))"),
-        (SF.weekofyear(SF.col("cola")), "WEEKOFYEAR(TO_DATE(cola))"),
+        (SF.weekofyear("cola"), "WEEKOFYEAR(cola)"),
+        (SF.weekofyear(SF.col("cola")), "WEEKOFYEAR(cola)"),
     ],
 )
 def test_weekofyear(expression, expected):

{sqlframe-3.36.3 → sqlframe-3.37.0}/tests/unit/standalone/test_types.py

@@ -7,8 +7,8 @@ from sqlframe.standalone import types
     "dtype, expected",
     [
         (types.StringType(), "string"),
-        (types.CharType(100), "char"),
-        (types.VarcharType(65), "varchar"),
+        (types.CharType(100), "char(100)"),
+        (types.VarcharType(65), "varchar(65)"),
         (types.BinaryType(), "binary"),
         (types.BooleanType(), "boolean"),
         (types.DateType(), "date"),