sqlframe 3.43.0__tar.gz → 3.43.2__tar.gz

{sqlframe-3.43.0 → sqlframe-3.43.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.43.0
+Version: 3.43.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.43.0 → sqlframe-3.43.2}/docs/bigquery.md

@@ -526,7 +526,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
-* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.43.0 → sqlframe-3.43.2}/docs/snowflake.md

@@ -522,7 +522,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
-* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.43.0 → sqlframe-3.43.2}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.16",
+        "sqlglot>=24.0.0,<27.18",
         "typing_extensions",
         "more-itertools",
     ],

{sqlframe-3.43.0 → sqlframe-3.43.2}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.43.0'
-__version_tuple__ = version_tuple = (3, 43, 0)
+__version__ = version = '3.43.2'
+__version_tuple__ = version_tuple = (3, 43, 2)
 
-__commit_id__ = commit_id = 'g38d15d998'
+__commit_id__ = commit_id = 'gf6f09ead9'

{sqlframe-3.43.0 → sqlframe-3.43.2}/sqlframe/base/dataframe.py

@@ -921,8 +921,30 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def groupBy(self, *cols, **kwargs) -> GROUP_DATA:
         if cols and isinstance(cols[0], list):
             cols = cols[0]  # type: ignore
+
+        # Special handling for groupBy operations with column aliases
+        # `_ensure_and_normalize_cols` sets CTE aliases that may not exist in the final context.
         columns = self._ensure_and_normalize_cols(cols)
-        return self._group_data(self, columns, self.last_op)
+
+        # Post-process the columns to fix any issues with CTE aliases
+        from sqlframe.base.normalize import (
+            _extract_column_name,
+            is_column_unambiguously_available,
+        )
+
+        processed_columns: t.List[Column] = []
+
+        for col in columns:
+            # Check if this column has a table qualifier that might be problematic
+            if col.column_expression.args.get("table"):
+                # Check if the column is unambiguously available without the qualifier
+                column_name = _extract_column_name(this) if (this := col.expression.this) else None
+                if column_name and is_column_unambiguously_available(self.expression, column_name):
+                    # Remove the table qualifier if the column is unambiguous
+                    col.column_expression.set("table", None)
+            processed_columns.append(col)
+
+        return self._group_data(self, processed_columns, self.last_op)
 
     groupby = groupBy
 

{sqlframe-3.43.0 → sqlframe-3.43.2}/sqlframe/base/functions.py

@@ -2067,7 +2067,7 @@ def bit_length(col: ColumnOrName) -> Column:
     if session._is_bigquery:
         return bit_length_from_length(col)
 
-    return Column.invoke_anonymous_function(col, "BIT_LENGTH")
+    return Column.invoke_expression_over_column(col, expression.BitLength)
 
 
 @meta()
@@ -2729,7 +2729,7 @@ def aggregate(
     return Column.invoke_expression_over_column(col, expression.Reduce, **kwargs)
 
 
-@meta(unsupported_engines="postgres")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def transform(
     col: ColumnOrName,
     f: t.Union[t.Callable[[Column], Column], t.Callable[[Column, Column], Column]],
@@ -7169,12 +7169,32 @@ def _lambda_quoted(value: str) -> t.Optional[bool]:
 
 @meta()
 def _get_lambda_from_func(lambda_expression: t.Callable):
-    variables = [
-        expression.to_identifier(x, quoted=_lambda_quoted(x))
-        for x in lambda_expression.__code__.co_varnames
-    ]
+    import inspect
+
+    # Get the function signature
+    sig = inspect.signature(lambda_expression)
+    param_names = list(sig.parameters.keys())
+
+    # Check if this looks like a column function (single 'col' parameter)
+    if len(param_names) == 1 and param_names[0] in ["col", "column"]:
+        # Wrap column functions to work with transform
+        variables = [expression.to_identifier("x", quoted=_lambda_quoted("x"))]
+        result = lambda_expression(Column("x"))
+        return expression.Lambda(
+            this=result.column_expression,
+            expressions=variables,
+        )
+
+    # Handle regular functions and lambdas
+    var_names = lambda_expression.__code__.co_varnames[: lambda_expression.__code__.co_argcount]
+
+    variables = [expression.to_identifier(x, quoted=_lambda_quoted(x)) for x in var_names]
+
+    # Call with Column objects for each parameter
+    result = lambda_expression(*[Column(x) for x in var_names])
+
     return expression.Lambda(
-        this=lambda_expression(*[Column(x) for x in variables]).column_expression,
+        this=result.column_expression,
         expressions=variables,
     )
 

{sqlframe-3.43.0 → sqlframe-3.43.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.43.0
+Version: 3.43.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.43.0 → sqlframe-3.43.2}/sqlframe.egg-info/requires.txt

@@ -1,6 +1,6 @@
 more-itertools
 prettytable<4
-sqlglot<27.16,>=24.0.0
+sqlglot<27.18,>=24.0.0
 typing_extensions
 
 [bigquery]

{sqlframe-3.43.0 → sqlframe-3.43.2}/tests/integration/engines/test_int_functions.py

@@ -2941,19 +2941,22 @@ def test_aggregate(get_session_and_func, get_func):
 def test_transform(get_session_and_func, get_func):
     session, transform = get_session_and_func("transform")
     when = get_func("when", session)
+    initcap = get_func("initcap", session)
+    df = session.createDataFrame([(["ab cd"],)], ["a"])
+    assert df.select(transform("a", f=lambda w: initcap(w)).alias("v")).first()[0] == ["Ab Cd"]
+    assert df.select(transform("a", f=initcap).alias("v")).first()[0] == ["Ab Cd"]
     df = session.createDataFrame([(1, [1, 2, 3, 4])], ("key", "some_values"))
     assert df.select(transform("some_values", lambda x: x * 2).alias("doubled")).collect() == [
         Row(doubled=[2, 4, 6, 8])
     ]
 
-    if isinstance(session, PySparkSession):
-
-        def alternate(x, i):
-            return when(i % 2 == 0, x).otherwise(-x)
+    def alternate(x, i):
+        return when(i % 2 == 0, x).otherwise(-x)
 
-        assert df.select(transform("some_values", alternate).alias("alternated")).collect() == [
-            Row(alternated=[1, -2, 3, -4])
-        ]
+    assert df.select(transform("some_values", alternate).alias("alternated")).collect() in [
+        [Row(alternated=[1, -2, 3, -4])],
+        [Row(alternated=[-1, 2, -3, 4])],
+    ]
 
 
 def test_exists(get_session_and_func):

{sqlframe-3.43.0 → sqlframe-3.43.2}/tests/integration/test_int_dataframe.py

@@ -2758,6 +2758,30 @@ def test_alias_group_by_column(
     compare_frames(df, dfs, compare_schema=False, sort=True)
 
 
+# https://github.com/eakmanrq/sqlframe/issues/502
+def test_group_by_alias(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df = pyspark_employee.sparkSession.createDataFrame(
+        [(2, "Alice"), (2, "Bob"), (2, "Bob"), (5, "Bob")], schema=["age", "name"]
+    )
+    df = df.select(F.col("name"), F.col("age")).groupBy(F.col("name")).count()
+
+    dfs = get_df("employee").sparkSession.createDataFrame(
+        [(2, "Alice"), (2, "Bob"), (2, "Bob"), (5, "Bob")], schema=["age", "name"]
+    )
+    dfs = (
+        dfs.alias("df")
+        .select(SF.col("df.name"), SF.col("df.age"))
+        .groupBy(SF.col("df.name"))
+        .count()
+    )
+
+    compare_frames(df, dfs, compare_schema=False, sort=True)
+
+
 # https://github.com/eakmanrq/sqlframe/issues/493
 def test_alias_multiple_joins(
     pyspark_employee: PySparkDataFrame,

{sqlframe-3.43.0 → sqlframe-3.43.2}/tests/unit/standalone/test_dataframe.py

@@ -188,3 +188,14 @@ def test_aliased_dictionary_agg(standalone_employee: StandaloneDataFrame):
         standalone_employee.groupBy("fname").agg({"age": "avg", "lname": "count"}).sql(pretty=False)
         == "SELECT CAST(`a1`.`fname` AS STRING) AS `fname`, AVG(`a1`.`age`) AS `avg(age)`, COUNT(CAST(`a1`.`lname` AS STRING)) AS `count(lname)` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`) GROUP BY CAST(`a1`.`fname` AS STRING)"
     )
+
+
+def test_aliased_group_by(standalone_employee: StandaloneDataFrame):
+    assert (
+        standalone_employee.alias("employee")
+        .select(F.col("employee.fname"))
+        .groupBy(F.col("employee.fname"))
+        .count()
+        .sql(pretty=False)
+        == "SELECT CAST(`a1`.`fname` AS STRING) AS `fname`, COUNT(*) AS `count` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`) GROUP BY CAST(`a1`.`fname` AS STRING)"
+    )