sqlframe 3.34.0__tar.gz → 3.35.1__tar.gz

{sqlframe-3.34.0 → sqlframe-3.35.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.34.0
+Version: 3.35.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.34.0 → sqlframe-3.35.1}/docs/bigquery.md

@@ -295,6 +295,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.34.0 → sqlframe-3.35.1}/docs/duckdb.md

@@ -260,6 +260,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.34.0 → sqlframe-3.35.1}/docs/postgres.md

@@ -273,6 +273,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.34.0 → sqlframe-3.35.1}/docs/snowflake.md

@@ -290,6 +290,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.34.0 → sqlframe-3.35.1}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.22",
+        "sqlglot>=24.0.0,<26.26",
         "typing_extensions",
     ],
     extras_require={
@@ -31,14 +31,14 @@ setup(
         "dev": [
             "duckdb>=1.2,<1.4",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.16",
+            "mypy>=1.10.0,<1.17",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
             "pyarrow>=10,<21",
             "pyspark>=2,<3.6",
-            "pytest>=8.2.0,<8.4",
+            "pytest>=8.2.0,<8.5",
             "pytest-forked",
             "pytest-postgresql>=6,<8",
             "pytest-xdist>=3.6,<3.8",

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.34.0'
-__version_tuple__ = version_tuple = (3, 34, 0)
+__version__ = version = '3.35.1'
+__version_tuple__ = version_tuple = (3, 35, 1)

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/base/dataframe.py

@@ -1524,6 +1524,52 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             raise ValueError("Tried to rename a column that doesn't exist")
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
+    @operation(Operation.SELECT)
+    def withColumnsRenamed(self, colsMap: t.Dict[str, str]) -> Self:
+        """
+        Returns a new :class:`DataFrame` by renaming multiple columns. If a non-existing column is
+        provided, it will be silently ignored.
+
+        .. versionadded:: 3.5.0
+
+        Parameters
+        ----------
+        colsMap : dict
+            a dict of column name and new column name.
+
+        Returns
+        -------
+        :class:`DataFrame`
+            DataFrame with renamed columns.
+
+        Examples
+        --------
+        >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"])
+        >>> df.withColumnsRenamed({"age": "years", "name": "firstName"}).show()
+        +-----+---------+
+        |years|firstName|
+        +-----+---------+
+        |    2|    Alice|
+        |    5|      Bob|
+        +-----+---------+
+        """
+        expression = self.expression.copy()
+        columns = self._get_outer_select_columns(expression)
+        results = []
+
+        # Normalize the keys in colsMap
+        normalized_cols_map = {self.session._normalize_string(k): v for k, v in colsMap.items()}
+
+        for column in columns:
+            col_name = column.alias_or_name
+            if col_name in normalized_cols_map:
+                new_name = normalized_cols_map[col_name]
+                column = column.alias(new_name)
+                self._update_display_name_mapping([column], [new_name])
+            results.append(column)
+
+        return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
+
     @operation(Operation.SELECT)
     def withColumns(self, *colsMap: t.Dict[str, Column]) -> Self:
         """

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/base/function_alternatives.py

@@ -999,27 +999,6 @@ def element_at_using_brackets(col: ColumnOrName, value: ColumnOrLiteral) -> Colu
     )
 
 
-def array_remove_using_filter(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
-    lit = get_func_from_session("lit")
-    col_func = get_func_from_session("col")
-
-    value = value if isinstance(value, Column) else lit(value)
-    return Column(
-        expression.Anonymous(
-            this="LIST_FILTER",
-            expressions=[
-                col_func(col).column_expression,
-                expression.Lambda(
-                    this=expression.NEQ(
-                        this=expression.Identifier(this="x"), expression=value.column_expression
-                    ),
-                    expressions=[expression.Identifier(this="x")],
-                ),
-            ],
-        )
-    )
-
-
 def array_union_using_list_concat(col1: ColumnOrName, col2: ColumnOrName) -> Column:
     col_func = get_func_from_session("col")
 
@@ -1664,41 +1643,6 @@ def array_position_bgutil(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
     )
 
 
-def array_remove_bgutil(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
-    lit = get_func_from_session("lit")
-
-    value_col = value if isinstance(value, Column) else lit(value)
-
-    filter_subquery = expression.select(
-        "*",
-    ).from_(
-        expression.Unnest(
-            expressions=[Column.ensure_col(col).column_expression],
-            alias=expression.TableAlias(
-                columns=[expression.to_identifier("x")],
-            ),
-        )
-    )
-
-    agg_subquery = (
-        expression.select(
-            expression.Anonymous(
-                this="ARRAY_AGG",
-                expressions=[expression.column("x")],
-            ),
-        )
-        .from_(filter_subquery.subquery("t"))
-        .where(
-            expression.NEQ(
-                this=expression.column("x", "t"),
-                expression=value_col.column_expression,
-            )
-        )
-    )
-
-    return Column(agg_subquery.subquery())
-
-
 def array_distinct_bgutil(col: ColumnOrName) -> Column:
     return Column(
         expression.Anonymous(

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/base/functions.py

@@ -2268,21 +2268,10 @@ def element_at(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
 
 @meta()
 def array_remove(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
-    from sqlframe.base.function_alternatives import (
-        array_remove_bgutil,
-        array_remove_using_filter,
-    )
-
-    session = _get_session()
-
-    if session._is_bigquery:
-        return array_remove_bgutil(col, value)
-
-    if session._is_duckdb:
-        return array_remove_using_filter(col, value)
-
     value_col = value if isinstance(value, Column) else lit(value)
-    return Column.invoke_anonymous_function(col, "ARRAY_REMOVE", value_col)
+    return Column.invoke_expression_over_column(
+        col, expression.ArrayRemove, expression=value_col.column_expression
+    )
 
 
 @meta(unsupported_engines="postgres")

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/base/types.py

@@ -104,22 +104,22 @@ class FloatType(DataType):
 
 
 class ByteType(DataType):
-    def __str__(self) -> str:
+    def simpleString(self) -> str:
         return "tinyint"
 
 
 class IntegerType(DataType):
-    def __str__(self) -> str:
+    def simpleString(self) -> str:
         return "int"
 
 
 class LongType(DataType):
-    def __str__(self) -> str:
+    def simpleString(self) -> str:
         return "bigint"
 
 
 class ShortType(DataType):
-    def __str__(self) -> str:
+    def simpleString(self) -> str:
         return "smallint"
 
 

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe/bigquery/session.py

@@ -51,7 +51,7 @@ class BigQuerySession(
             super().__init__(conn or connect())
             if self._client.default_query_job_config is None:
                 self._client.default_query_job_config = bigquery.QueryJobConfig()
-            self.default_dataset = default_dataset
+            self.default_dataset = default_dataset  # type: ignore
 
     @property
     def _client(self) -> BigQueryClient:

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.34.0
+Version: 3.35.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.34.0 → sqlframe-3.35.1}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.22,>=24.0.0
+sqlglot<26.26,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -12,7 +12,7 @@ databricks-sql-connector[pyarrow]<5,>=3.6
 [dev]
 duckdb<1.4,>=1.2
 findspark<3,>=2
-mypy<1.16,>=1.10.0
+mypy<1.17,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
@@ -23,7 +23,7 @@ pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<8,>=6
 pytest-xdist<3.8,>=3.6
-pytest<8.4,>=8.2.0
+pytest<8.5,>=8.2.0
 ruff<0.12,>=0.4.4
 types-psycopg2<3,>=2.9
 

{sqlframe-3.34.0 → sqlframe-3.35.1}/tests/integration/test_int_dataframe.py

@@ -1835,6 +1835,23 @@ def test_with_columns_reference_another(
     compare_frames(df, dfs)
 
 
+def test_with_columns_renamed(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    employee = get_df("employee")
+    df = pyspark_employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+
+    dfs = employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+
+    compare_frames(df, dfs)
+
+
 def test_drop_column_single(
     pyspark_employee: PySparkDataFrame,
     get_df: t.Callable[[str], BaseDataFrame],

{sqlframe-3.34.0 → sqlframe-3.35.1}/tests/unit/standalone/test_dataframe.py

@@ -76,6 +76,30 @@ def test_with_columns(standalone_employee: StandaloneDataFrame):
     )
 
 
+def test_with_columns_renamed(standalone_employee: StandaloneDataFrame):
+    df = standalone_employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+    assert df.columns == [
+        "employee_id",
+        "first_name",
+        "last_name",
+        "age",
+        "store_id",
+    ]
+    assert (
+        df.sql(pretty=False)
+        == "SELECT `a1`.`employee_id` AS `employee_id`, CAST(`a1`.`fname` AS STRING) AS `first_name`, CAST(`a1`.`lname` AS STRING) AS `last_name`, `a1`.`age` AS `age`, `a1`.`store_id` AS `store_id` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`)"
+    )
+
+
+def test_with_columns_renamed_nonexistent(standalone_employee: StandaloneDataFrame):
+    # Test that non-existent columns are silently ignored
+    df = standalone_employee.withColumnsRenamed({"nonexistent_col": "new_name"})
+    # Verify that the DataFrame columns remain unchanged
+    assert df.columns == ["employee_id", "fname", "lname", "age", "store_id"]
+
+
 def test_transform(standalone_employee: StandaloneDataFrame):
     def cast_all_to_int(input_df):
         return input_df.select([F.col(col_name).cast("int") for col_name in input_df.columns])

{sqlframe-3.34.0 → sqlframe-3.35.1}/tests/unit/standalone/test_types.py

@@ -17,12 +17,12 @@ from sqlframe.standalone import types
         (types.DecimalType(10, 3), "decimal(10, 3)"),
         (types.DoubleType(), "double"),
         (types.FloatType(), "float"),
-        (types.ByteType(), "byte"),
-        (types.IntegerType(), "integer"),
-        (types.LongType(), "long"),
-        (types.ShortType(), "short"),
-        (types.ArrayType(types.IntegerType()), "array<integer>"),
-        (types.MapType(types.IntegerType(), types.StringType()), "map<integer, string>"),
+        (types.ByteType(), "tinyint"),
+        (types.IntegerType(), "int"),
+        (types.LongType(), "bigint"),
+        (types.ShortType(), "smallint"),
+        (types.ArrayType(types.IntegerType()), "array<int>"),
+        (types.MapType(types.IntegerType(), types.StringType()), "map<int, string>"),
         (
             types.StructType(
                 [
@@ -30,7 +30,7 @@ from sqlframe.standalone import types
                     types.StructField("colb", types.StringType()),
                 ]
             ),
-            "struct<cola:integer, colb:string>",
+            "struct<cola:int, colb:string>",
         ),
     ],
 )