sqlframe 3.33.1__tar.gz → 3.35.0__tar.gz

{sqlframe-3.33.1 → sqlframe-3.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.33.1
+Version: 3.35.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.33.1 → sqlframe-3.35.0}/docs/bigquery.md

@@ -295,6 +295,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.33.1 → sqlframe-3.35.0}/docs/duckdb.md

@@ -260,6 +260,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.33.1 → sqlframe-3.35.0}/docs/postgres.md

@@ -273,6 +273,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.33.1 → sqlframe-3.35.0}/docs/snowflake.md

@@ -290,6 +290,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [where](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.where.html)
 * [withColumn](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html)
 * [withColumnRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnRenamed.html)
+* [withColumnsRenamed](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumnsRenamed.html)
 * [write](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.write.html)
 
 ### Functions

{sqlframe-3.33.1 → sqlframe-3.35.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.20",
+        "sqlglot>=24.0.0,<26.25",
         "typing_extensions",
     ],
     extras_require={
@@ -31,7 +31,7 @@ setup(
         "dev": [
             "duckdb>=1.2,<1.4",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.16",
+            "mypy>=1.10.0,<1.17",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
@@ -41,7 +41,7 @@ setup(
             "pytest>=8.2.0,<8.4",
             "pytest-forked",
             "pytest-postgresql>=6,<8",
-            "pytest-xdist>=3.6,<3.7",
+            "pytest-xdist>=3.6,<3.8",
             "pre-commit>=3.7,<5",
             "ruff>=0.4.4,<0.12",
             "types-psycopg2>=2.9,<3",

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.33.1'
-__version_tuple__ = version_tuple = (3, 33, 1)
+__version__ = version = '3.35.0'
+__version_tuple__ = version_tuple = (3, 35, 0)

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe/base/column.py

@@ -128,6 +128,39 @@ class Column:
             "Tried to call a column which is unexpected. Did you mean to call a method on a DataFrame? If so, make sure the method is typed correctly and is supported. If not, please open an issue requesting support: https://github.com/eakmanrq/sqlframe/issues"
         )
 
+    def __getitem__(self, key: t.Any) -> Column:
+        """
+        An expression that gets an item at position ``ordinal`` out of a list,
+        or gets an item by key out of a dict.
+
+        .. versionadded:: 1.3.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        k
+            a literal value, or a slice object without step.
+
+        Returns
+        -------
+        :class:`Column`
+            Column representing the item got by key out of a dict, or substrings sliced by
+            the given slice object.
+
+        Examples
+        --------
+        >>> df = spark.createDataFrame([('abcedfg', {"key": "value"})], ["l", "d"])
+        >>> df.select(df.l[slice(1, 3)], df.d['key']).show()
+        +---------------+------+
+        |substr(l, 1, 3)|d[key]|
+        +---------------+------+
+        |            abc| value|
+        +---------------+------+
+        """
+        return self.getItem(key)
+
     def __getattr__(self, name: str) -> Column:
         """
         Enables accessing nested fields using dot notation for struct types.

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe/base/dataframe.py

@@ -1524,6 +1524,52 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             raise ValueError("Tried to rename a column that doesn't exist")
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
+    @operation(Operation.SELECT)
+    def withColumnsRenamed(self, colsMap: t.Dict[str, str]) -> Self:
+        """
+        Returns a new :class:`DataFrame` by renaming multiple columns. If a non-existing column is
+        provided, it will be silently ignored.
+
+        .. versionadded:: 3.5.0
+
+        Parameters
+        ----------
+        colsMap : dict
+            a dict of column name and new column name.
+
+        Returns
+        -------
+        :class:`DataFrame`
+            DataFrame with renamed columns.
+
+        Examples
+        --------
+        >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"])
+        >>> df.withColumnsRenamed({"age": "years", "name": "firstName"}).show()
+        +-----+---------+
+        |years|firstName|
+        +-----+---------+
+        |    2|    Alice|
+        |    5|      Bob|
+        +-----+---------+
+        """
+        expression = self.expression.copy()
+        columns = self._get_outer_select_columns(expression)
+        results = []
+
+        # Normalize the keys in colsMap
+        normalized_cols_map = {self.session._normalize_string(k): v for k, v in colsMap.items()}
+
+        for column in columns:
+            col_name = column.alias_or_name
+            if col_name in normalized_cols_map:
+                new_name = normalized_cols_map[col_name]
+                column = column.alias(new_name)
+                self._update_display_name_mapping([column], [new_name])
+            results.append(column)
+
+        return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
+
     @operation(Operation.SELECT)
     def withColumns(self, *colsMap: t.Dict[str, Column]) -> Self:
         """

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe/bigquery/session.py

@@ -51,7 +51,7 @@ class BigQuerySession(
             super().__init__(conn or connect())
             if self._client.default_query_job_config is None:
                 self._client.default_query_job_config = bigquery.QueryJobConfig()
-            self.default_dataset = default_dataset
+            self.default_dataset = default_dataset  # type: ignore
 
     @property
     def _client(self) -> BigQueryClient:

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.33.1
+Version: 3.35.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.33.1 → sqlframe-3.35.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.20,>=24.0.0
+sqlglot<26.25,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -12,7 +12,7 @@ databricks-sql-connector[pyarrow]<5,>=3.6
 [dev]
 duckdb<1.4,>=1.2
 findspark<3,>=2
-mypy<1.16,>=1.10.0
+mypy<1.17,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
@@ -22,7 +22,7 @@ pyarrow<21,>=10
 pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<8,>=6
-pytest-xdist<3.7,>=3.6
+pytest-xdist<3.8,>=3.6
 pytest<8.4,>=8.2.0
 ruff<0.12,>=0.4.4
 types-psycopg2<3,>=2.9

{sqlframe-3.33.1 → sqlframe-3.35.0}/tests/integration/engines/test_engine_column.py

@@ -37,6 +37,7 @@ def test_column_get_item_array(get_session: t.Callable[[], _BaseSession], get_fu
     session = get_session()
     lit = get_func("lit", session)
     assert session.range(1).select(lit(["a", "b", "c"]).getItem(0).alias("value")).first()[0] == "a"
+    assert session.range(1).select(lit(["a", "b", "c"])[0].alias("value")).first()[0] == "a"
 
 
 def test_column_get_item_map(get_session: t.Callable[[], _BaseSession], get_func):
@@ -44,6 +45,7 @@ def test_column_get_item_map(get_session: t.Callable[[], _BaseSession], get_func
     lit = get_func("lit", session)
     if not isinstance(session, (PostgresSession, BigQuerySession)):
         assert session.range(1).select(lit({"key": "value"}).getItem("key")).first()[0] == "value"
+        assert session.range(1).select(lit({"key": "value"})["key"]).first()[0] == "value"
 
 
 def test_column_get_field_struct(get_session: t.Callable[[], _BaseSession]):

{sqlframe-3.33.1 → sqlframe-3.35.0}/tests/integration/test_int_dataframe.py

@@ -1835,6 +1835,23 @@ def test_with_columns_reference_another(
     compare_frames(df, dfs)
 
 
+def test_with_columns_renamed(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    employee = get_df("employee")
+    df = pyspark_employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+
+    dfs = employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+
+    compare_frames(df, dfs)
+
+
 def test_drop_column_single(
     pyspark_employee: PySparkDataFrame,
     get_df: t.Callable[[str], BaseDataFrame],

{sqlframe-3.33.1 → sqlframe-3.35.0}/tests/unit/standalone/test_column.py

@@ -230,5 +230,16 @@ def test_get_item():
     assert F.col("cola").getItem(1).sql() == "ELEMENT_AT(cola, (1 + 1)) AS element_at__cola__"
 
 
+def test_get_item_with_array_syntax():
+    assert F.col("cola")[1].sql() == "ELEMENT_AT(cola, (1 + 1)) AS element_at__cola__"
+
+
+def test_get_item_by_key_with_array_syntax():
+    assert (
+        F.lit({"key": "value"})["key"].sql()
+        == "ELEMENT_AT(MAP('key', 'value'), 'key') AS element_at__key__"
+    )
+
+
 def test_contains():
     assert F.col("cola").contains("foo").sql() == "CONTAINS(cola, 'foo')"

{sqlframe-3.33.1 → sqlframe-3.35.0}/tests/unit/standalone/test_dataframe.py

@@ -76,6 +76,30 @@ def test_with_columns(standalone_employee: StandaloneDataFrame):
     )
 
 
+def test_with_columns_renamed(standalone_employee: StandaloneDataFrame):
+    df = standalone_employee.withColumnsRenamed(
+        {"fname": "first_name", "lname": "last_name", "nonexistent_col": "new_name"}
+    )
+    assert df.columns == [
+        "employee_id",
+        "first_name",
+        "last_name",
+        "age",
+        "store_id",
+    ]
+    assert (
+        df.sql(pretty=False)
+        == "SELECT `a1`.`employee_id` AS `employee_id`, CAST(`a1`.`fname` AS STRING) AS `first_name`, CAST(`a1`.`lname` AS STRING) AS `last_name`, `a1`.`age` AS `age`, `a1`.`store_id` AS `store_id` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`)"
+    )
+
+
+def test_with_columns_renamed_nonexistent(standalone_employee: StandaloneDataFrame):
+    # Test that non-existent columns are silently ignored
+    df = standalone_employee.withColumnsRenamed({"nonexistent_col": "new_name"})
+    # Verify that the DataFrame columns remain unchanged
+    assert df.columns == ["employee_id", "fname", "lname", "age", "store_id"]
+
+
 def test_transform(standalone_employee: StandaloneDataFrame):
     def cast_all_to_int(input_df):
         return input_df.select([F.col(col_name).cast("int") for col_name in input_df.columns])