sqlframe 3.24.1__tar.gz → 3.26.0__tar.gz

{sqlframe-3.24.1 → sqlframe-3.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.24.1
+Version: 3.26.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.24.1 → sqlframe-3.26.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.10",
+        "sqlglot>=24.0.0,<26.12",
         "typing_extensions",
     ],
     extras_require={
@@ -43,7 +43,7 @@ setup(
             "pytest-postgresql>=6,<8",
             "pytest-xdist>=3.6,<3.7",
             "pre-commit>=3.7,<5",
-            "ruff>=0.4.4,<0.10",
+            "ruff>=0.4.4,<0.12",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.24.1'
-__version_tuple__ = version_tuple = (3, 24, 1)
+__version__ = version = '3.26.0'
+__version_tuple__ = version_tuple = (3, 26, 0)

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe/base/function_alternatives.py

@@ -551,6 +551,17 @@ def sha1_force_sha1_and_to_hex(col: ColumnOrName) -> Column:
     )
 
 
+def sha2_sha265(col: ColumnOrName) -> Column:
+    col_func = get_func_from_session("col")
+
+    return Column(
+        expression.Anonymous(
+            this="SHA256",
+            expressions=[col_func(col).column_expression],
+        )
+    )
+
+
 def hash_from_farm_fingerprint(*cols: ColumnOrName) -> Column:
     if len(cols) > 1:
         raise ValueError("This dialect only supports a single column for calculating hash")

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe/base/functions.py

@@ -1492,7 +1492,7 @@ def md5(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.MD5)
 
 
-@meta(unsupported_engines=["duckdb", "postgres"])
+@meta(unsupported_engines=["postgres"])
 def sha1(col: ColumnOrName) -> Column:
     from sqlframe.base.function_alternatives import sha1_force_sha1_and_to_hex
 
@@ -1504,8 +1504,18 @@ def sha1(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.SHA)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres"])
+@meta(unsupported_engines=["bigquery", "postgres"])
 def sha2(col: ColumnOrName, numBits: int) -> Column:
+    from sqlframe.base.function_alternatives import sha2_sha265
+
+    session = _get_session()
+
+    if session._is_duckdb:
+        if numBits in [256, 0]:
+            return sha2_sha265(col)
+        else:
+            raise ValueError("This dialect only supports SHA-265 (numBits=256 or numBits=0)")
+
     return Column.invoke_expression_over_column(col, expression.SHA2, length=lit(numBits))
 
 
@@ -2355,15 +2365,31 @@ def from_json(
         schema = schema.simpleString()
     schema = schema if isinstance(schema, Column) else lit(schema)
     if options is not None:
-        options_col = create_map([lit(x) for x in _flatten(options.items())])
+        options_col = create_map([lit(str(x)) for x in _flatten(options.items())])
         return Column.invoke_anonymous_function(col, "FROM_JSON", schema, options_col)
     return Column.invoke_anonymous_function(col, "FROM_JSON", schema)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def to_json(col: ColumnOrName, options: t.Optional[t.Dict[str, str]] = None) -> Column:
+    session = _get_session()
+    if session._is_duckdb:
+        # check if Spark options match DuckDB's default behavior
+        is_spark_equivalent = (
+            options
+            and list(options.keys()) == ["ignoreNullFields"]  # only one option specified
+            and str(options.get("ignoreNullFields", "true")).lower() == "false"
+        )
+        if not is_spark_equivalent:
+            logger.warning(
+                "Options for `to_json()` ignored, since not supported in this dialect."
+                + " Potential `null` values are included in the returned JSON string."
+                + " This is different from Spark's default behavior."
+            )
+        options = None
+
     if options is not None:
-        options_col = create_map([lit(x) for x in _flatten(options.items())])
+        options_col = create_map([lit(str(x)) for x in _flatten(options.items())])
         return Column.invoke_expression_over_column(col, expression.JSONFormat, options=options_col)
     return Column.invoke_expression_over_column(col, expression.JSONFormat)
 
@@ -2373,7 +2399,7 @@ def schema_of_json(col: ColumnOrName, options: t.Optional[t.Dict[str, str]] = No
     if isinstance(col, str):
         col = lit(col)
     if options is not None:
-        options_col = create_map([lit(x) for x in _flatten(options.items())])
+        options_col = create_map([lit(str(x)) for x in _flatten(options.items())])
         return Column.invoke_anonymous_function(col, "SCHEMA_OF_JSON", options_col)
     return Column.invoke_anonymous_function(col, "SCHEMA_OF_JSON")
 
@@ -2383,7 +2409,7 @@ def schema_of_csv(col: ColumnOrName, options: t.Optional[t.Dict[str, str]] = Non
     if isinstance(col, str):
         col = lit(col)
     if options is not None:
-        options_col = create_map([lit(x) for x in _flatten(options.items())])
+        options_col = create_map([lit(str(x)) for x in _flatten(options.items())])
         return Column.invoke_anonymous_function(col, "SCHEMA_OF_CSV", options_col)
     return Column.invoke_anonymous_function(col, "SCHEMA_OF_CSV")
 
@@ -2391,7 +2417,7 @@ def schema_of_csv(col: ColumnOrName, options: t.Optional[t.Dict[str, str]] = Non
 @meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
 def to_csv(col: ColumnOrName, options: t.Optional[t.Dict[str, str]] = None) -> Column:
     if options is not None:
-        options_col = create_map([lit(x) for x in _flatten(options.items())])
+        options_col = create_map([lit(str(x)) for x in _flatten(options.items())])
         return Column.invoke_anonymous_function(col, "TO_CSV", options_col)
     return Column.invoke_anonymous_function(col, "TO_CSV")
 

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe/duckdb/functions.pyi

@@ -166,6 +166,7 @@ from sqlframe.base.functions import rpad as rpad
 from sqlframe.base.functions import rtrim as rtrim
 from sqlframe.base.functions import second as second
 from sqlframe.base.functions import sequence as sequence
+from sqlframe.base.functions import sha1 as sha1
 from sqlframe.base.functions import shiftLeft as shiftLeft
 from sqlframe.base.functions import shiftleft as shiftleft
 from sqlframe.base.functions import shiftRight as shiftRight
@@ -194,6 +195,7 @@ from sqlframe.base.functions import sumDistinct as sumDistinct
 from sqlframe.base.functions import tan as tan
 from sqlframe.base.functions import timestamp_seconds as timestamp_seconds
 from sqlframe.base.functions import to_date as to_date
+from sqlframe.base.functions import to_json as to_json
 from sqlframe.base.functions import to_timestamp as to_timestamp
 from sqlframe.base.functions import to_unix_timestamp as to_unix_timestamp
 from sqlframe.base.functions import toDegrees as toDegrees

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.24.1
+Version: 3.26.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.24.1 → sqlframe-3.26.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.10,>=24.0.0
+sqlglot<26.12,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -24,7 +24,7 @@ pytest-forked
 pytest-postgresql<8,>=6
 pytest-xdist<3.7,>=3.6
 pytest<8.4,>=8.2.0
-ruff<0.10,>=0.4.4
+ruff<0.12,>=0.4.4
 types-psycopg2<3,>=2.9
 
 [docs]

{sqlframe-3.24.1 → sqlframe-3.26.0}/tests/integration/engines/test_int_functions.py

@@ -2460,15 +2460,19 @@ def test_from_json(get_session_and_func, get_types, get_func):
 
 def test_to_json(get_session_and_func, get_types, get_func):
     session, to_json = get_session_and_func("to_json")
-    data = [(1, Row(age=2, name="Alice"))]
+    data = [(1, Row(id=2, name="Alice", birthday=datetime.date(1995, 1, 9)))]
     df = session.createDataFrame(data, ("key", "value"))
-    assert df.select(to_json(df.value).alias("json")).collect() == [
-        Row(json='{"age":2,"name":"Alice"}')
-    ]
-    data = [(1, [Row(age=2, name="Alice"), Row(age=3, name="Bob")])]
+    assert df.select(
+        to_json(df.value, options={"ignoreNullFields": False, "dateFormat": "yyyy-MM-dd"}).alias(
+            "json"
+        )
+    ).collect() == [Row(json='{"id":2,"name":"Alice","birthday":"1995-01-09"}')]
+    data = [(1, [Row(id=2, name="Alice"), Row(id=3, name="Bob"), Row(id=4, name=None)])]
     df = session.createDataFrame(data, ("key", "value"))
-    assert df.select(to_json(df.value).alias("json")).collect() == [
-        Row(json='[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]')
+    assert df.select(
+        to_json(df.value, options={"ignoreNullFields": "false"}).alias("json")
+    ).collect() == [
+        Row(json='[{"id":2,"name":"Alice"},{"id":3,"name":"Bob"},{"id":4,"name":null}]')
     ]
     data = [(1, {"name": "Alice"})]
     df = session.createDataFrame(data, ("key", "value"))