sqlframe 3.40.2__tar.gz → 3.42.0__tar.gz

{sqlframe-3.40.2 → sqlframe-3.42.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.40.2
+Version: 3.42.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.40.2 → sqlframe-3.42.0}/docs/bigquery.md

@@ -526,6 +526,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.40.2 → sqlframe-3.42.0}/docs/duckdb.md

@@ -268,6 +268,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [abs](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.abs.html)
 * [acos](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.acos.html)
 * [add_months](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.add_months.html)
+* [aggregate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.aggregate.html)
 * [any_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.any_value.html)
     * Always ignores nulls
 * [approxCountDistinct](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.approxCountDistinct.html)
@@ -380,6 +381,7 @@ See something that you would like to see supported? [Open an issue](https://gith
     * Uses a different hash algorithm than Spark
 * [hex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.hex.html)
 * [hour](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.hour.html)
+* [initcap](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.initcap.html)
 * [input_file_name](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.input_file_name.html)
 * [instr](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.instr.html)
 * [isnan](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.isnan.html)
@@ -433,8 +435,10 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [radians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.radians.html)
 * [rand](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rand.html)
 * [rank](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rank.html)
+* [reduce](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.reduce.html)
 * [regexp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp.html)
 * [regexp_extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract.html)
+* [regexp_extract_all](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract_all.html)
 * [regexp_like](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_like.html)
 * [regexp_replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_replace.html)
 * [repeat](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.repeat.html)
@@ -483,6 +487,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
 * [to_unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_unix_timestamp.html)
     * The values must match the format string (null will not be returned if they do not)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.40.2 → sqlframe-3.42.0}/docs/snowflake.md

@@ -472,6 +472,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [rand](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rand.html)
 * [rank](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rank.html)
 * [regexp_extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract.html)
+* [regexp_extract_all](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract_all.html)
 * [regexp_replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_replace.html)
 * [repeat](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.repeat.html)
 * [replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.replace.html)
@@ -521,6 +522,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.40.2 → sqlframe-3.42.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.13",
+        "sqlglot>=24.0.0,<27.15",
         "typing_extensions",
         "more-itertools",
     ],
@@ -32,7 +32,7 @@ setup(
         "dev": [
             "duckdb>=1.2,<1.4",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.18",
+            "mypy>=1.10.0,<1.19",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
@@ -45,7 +45,7 @@ setup(
             "pytest-rerunfailures",
             "pytest-xdist>=3.6,<3.9",
             "pre-commit>=3.7,<5",
-            "ruff>=0.4.4,<0.13",
+            "ruff>=0.4.4,<0.14",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-3.40.2 → sqlframe-3.42.0}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.40.2'
-__version_tuple__ = version_tuple = (3, 40, 2)
+__version__ = version = '3.42.0'
+__version_tuple__ = version_tuple = (3, 42, 0)
 
-__commit_id__ = commit_id = 'gdb7bc99bd'
+__commit_id__ = commit_id = 'gadde0dd66'

{sqlframe-3.40.2 → sqlframe-3.42.0}/sqlframe/base/functions.py

@@ -1974,8 +1974,30 @@ def regexp_replace(
     )
 
 
-@meta(unsupported_engines="duckdb")
+@meta()
 def initcap(col: ColumnOrName) -> Column:
+    session = _get_session()
+
+    if session._is_duckdb:
+        split_func = get_func_from_session("split")
+        transform_func = get_func_from_session("transform")
+        reduce_func = get_func_from_session("reduce")
+        upper_func = get_func_from_session("upper")
+        lower_func = get_func_from_session("lower")
+        length_func = get_func_from_session("length")
+        concat_func = get_func_from_session("concat")
+        concat_ws_func = get_func_from_session("concat_ws")
+        return reduce_func(
+            transform_func(
+                split_func(col, r"\s+"),
+                lambda w: concat_func(
+                    upper_func(w.substr(1, 1)), lower_func(w.substr(2, length_func(w) - 1))
+                ),
+            ),
+            None,
+            merge=lambda x, y: concat_ws_func(" ", x, y),
+        )
+
     return Column.invoke_expression_over_column(col, expression.Initcap)
 
 
@@ -2133,6 +2155,11 @@ def bit_xor(col: ColumnOrName) -> Column:
 
 @meta(unsupported_engines=["postgres", "snowflake"])
 def bit_count(col: ColumnOrName) -> Column:
+    session = _get_session()
+
+    if session._is_duckdb:
+        return Column.invoke_anonymous_function(col, "BIT_COUNT")
+
     return Column.invoke_expression_over_column(col, expression.BitwiseCountAgg)
 
 
@@ -2681,7 +2708,7 @@ def from_csv(
     return Column.invoke_anonymous_function(col, "FROM_CSV", schema)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def aggregate(
     col: ColumnOrName,
     initialValue: ColumnOrName,
@@ -2689,21 +2716,20 @@ def aggregate(
     finish: t.Optional[t.Callable[[Column], Column]] = None,
 ) -> Column:
     merge_exp = _get_lambda_from_func(merge)
+    kwargs = dict(
+        initial=initialValue,
+        merge=merge_exp,
+    )
+    session = _get_session()
     if finish is not None:
         finish_exp = _get_lambda_from_func(finish)
-        return Column.invoke_expression_over_column(
-            col,
-            expression.Reduce,
-            initial=initialValue,
-            merge=Column(merge_exp),
-            finish=Column(finish_exp),
-        )
-    return Column.invoke_expression_over_column(
-        col, expression.Reduce, initial=initialValue, merge=Column(merge_exp)
-    )
+        kwargs["finish"] = Column(finish_exp)
+    if session._is_duckdb:
+        kwargs.pop("initial", None)
+    return Column.invoke_expression_over_column(col, expression.Reduce, **kwargs)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines="postgres")
 def transform(
     col: ColumnOrName,
     f: t.Union[t.Callable[[Column], Column], t.Callable[[Column, Column], Column]],
@@ -5216,7 +5242,7 @@ def regexp_count(str: ColumnOrName, regexp: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "regexp_count", regexp)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres"])
 def regexp_extract_all(
     str: ColumnOrName, regexp: ColumnOrName, idx: t.Optional[t.Union[int, Column]] = None
 ) -> Column:
@@ -5251,6 +5277,9 @@ def regexp_extract_all(
     >>> df.select(regexp_extract_all('str', col("regexp")).alias('d')).collect()
     [Row(d=['100', '300'])]
     """
+    if idx is None:
+        idx = 1
+
     return Column.invoke_expression_over_column(
         str, expression.RegexpExtractAll, expression=regexp, group=idx
     )

{sqlframe-3.40.2 → sqlframe-3.42.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.40.2
+Version: 3.42.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.40.2 → sqlframe-3.42.0}/sqlframe.egg-info/requires.txt

@@ -1,6 +1,6 @@
 more-itertools
 prettytable<4
-sqlglot<27.13,>=24.0.0
+sqlglot<27.15,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -13,7 +13,7 @@ databricks-sql-connector[pyarrow]<5,>=3.6
 [dev]
 duckdb<1.4,>=1.2
 findspark<3,>=2
-mypy<1.18,>=1.10.0
+mypy<1.19,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
@@ -26,7 +26,7 @@ pytest-postgresql<8,>=6
 pytest-rerunfailures
 pytest-xdist<3.9,>=3.6
 pytest<8.5,>=8.2.0
-ruff<0.13,>=0.4.4
+ruff<0.14,>=0.4.4
 types-psycopg2<3,>=2.9
 
 [docs]

{sqlframe-3.40.2 → sqlframe-3.42.0}/tests/unit/standalone/test_dataframe.py

@@ -178,7 +178,7 @@ def test_lineage(standalone_employee: StandaloneDataFrame):
 def test_unquoted_identifiers(standalone_employee: StandaloneDataFrame):
     assert (
         standalone_employee.sql(dialect="snowflake", pretty=False, quote_identifiers=False)
-        == """SELECT A1.EMPLOYEE_ID AS "employee_id", CAST(A1.FNAME AS TEXT) AS "fname", CAST(A1.LNAME AS TEXT) AS "lname", A1.AGE AS "age", A1.STORE_ID AS "store_id" FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS A1(EMPLOYEE_ID, FNAME, LNAME, AGE, STORE_ID)"""
+        == """SELECT A1.EMPLOYEE_ID AS "employee_id", CAST(A1.FNAME AS VARCHAR) AS "fname", CAST(A1.LNAME AS VARCHAR) AS "lname", A1.AGE AS "age", A1.STORE_ID AS "store_id" FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS A1(EMPLOYEE_ID, FNAME, LNAME, AGE, STORE_ID)"""
     )
 
 

{sqlframe-3.40.2 → sqlframe-3.42.0}/tests/unit/standalone/test_functions.py

@@ -19,6 +19,7 @@ def test_invoke_anonymous(name, func):
     # ltrim/rtrim - don't seem to convert correctly on some engines
     # unix_micros - it is actually supported just an engine specific override uses an anonymous function
     # format_string - seemed like a complex match that had overrides so not worth it
+    # bit_count - requires an override for DuckDB
     ignore_funcs = {
         "array_size",
         "exists",
@@ -30,6 +31,7 @@ def test_invoke_anonymous(name, func):
         "current_schema",
         "unix_micros",
         "format_string",
+        "bit_count",
     }
     if "invoke_anonymous_function" in inspect.getsource(func) and name not in ignore_funcs:
         func = parse_one(f"{name}()", read="spark", error_level=ErrorLevel.IGNORE)