sqlframe 3.39.3__tar.gz → 3.39.4__tar.gz

{sqlframe-3.39.3 → sqlframe-3.39.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.39.3
+Version: 3.39.4
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.39.3 → sqlframe-3.39.4}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.9",
+        "sqlglot>=24.0.0,<27.10",
         "typing_extensions",
         "more-itertools",
     ],

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.39.3'
-__version_tuple__ = version_tuple = (3, 39, 3)
+__version__ = version = '3.39.4'
+__version_tuple__ = version_tuple = (3, 39, 4)
 
-__commit_id__ = commit_id = 'g9d915cb1e'
+__commit_id__ = commit_id = 'g7103a1e73'

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe/base/dataframe.py

@@ -16,7 +16,6 @@ from dataclasses import dataclass
 from uuid import uuid4
 
 import sqlglot
-from more_itertools import partition
 from prettytable import PrettyTable
 from sqlglot import Dialect, maybe_parse
 from sqlglot import expressions as exp
@@ -542,23 +541,16 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         expression.set("with", exp.With(expressions=existing_ctes))
         return expression
 
-    @classmethod
-    def _get_outer_select_expressions(
-        cls, item: exp.Expression
-    ) -> t.List[t.Union[exp.Column, exp.Alias]]:
-        outer_select = item.find(exp.Select)
-        if outer_select:
-            return outer_select.expressions
-        return []
-
     @classmethod
     def _get_outer_select_columns(cls, item: exp.Expression) -> t.List[Column]:
         from sqlframe.base.session import _BaseSession
 
         col = get_func_from_session("col", _BaseSession())
 
-        outer_expressions = cls._get_outer_select_expressions(item)
-        return [col(quote_preserving_alias_or_name(x)) for x in outer_expressions]
+        outer_select = item.find(exp.Select)
+        if outer_select:
+            return [col(quote_preserving_alias_or_name(x)) for x in outer_select.expressions]
+        return []
 
     def _create_hash_from_expression(self, expression: exp.Expression) -> str:
         from sqlframe.base.session import _BaseSession
@@ -1512,23 +1504,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         """
         return func(self, *args, **kwargs)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumn(self, colName: str, col: Column) -> Self:
         return self.withColumns.__wrapped__(self, {colName: col})  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumnRenamed(self, existing: str, new: str) -> Self:
-        col_func = get_func_from_session("col", self.session)
         expression = self.expression.copy()
         existing = self.session._normalize_string(existing)
-        outer_expressions = self._get_outer_select_expressions(expression)
+        columns = self._get_outer_select_columns(expression)
         results = []
         found_match = False
-        for expr in outer_expressions:
-            column = col_func(expr.copy())
-            if existing == quote_preserving_alias_or_name(expr):
-                if isinstance(column.expression, exp.Alias):
-                    column.expression.set("alias", exp.to_identifier(new))
+        for column in columns:
+            if column.alias_or_name == existing:
+                column = column.alias(new)
                 self._update_display_name_mapping([column], [new])
                 found_match = True
             results.append(column)
@@ -1536,7 +1525,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             raise ValueError("Tried to rename a column that doesn't exist")
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumnsRenamed(self, colsMap: t.Dict[str, str]) -> Self:
         """
         Returns a new :class:`DataFrame` by renaming multiple columns. If a non-existing column is
@@ -1582,7 +1571,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumns(self, *colsMap: t.Dict[str, Column]) -> Self:
         """
         Returns a new :class:`DataFrame` by adding multiple columns or replacing the
@@ -1620,14 +1609,13 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         """
         if len(colsMap) != 1:
             raise ValueError("Only a single map is supported")
-        col_func = get_func_from_session("col")
         col_map = {
             self._ensure_and_normalize_col(k): (self._ensure_and_normalize_col(v), k)
             for k, v in colsMap[0].items()
         }
-        existing_expr = self._get_outer_select_expressions(self.expression)
-        existing_col_names = [x.alias_or_name for x in existing_expr]
-        select_columns = [col_func(x) for x in existing_expr]
+        existing_cols = self._get_outer_select_columns(self.expression)
+        existing_col_names = [x.alias_or_name for x in existing_cols]
+        select_columns = existing_cols
         for col, (col_value, display_name) in col_map.items():
             column_name = col.alias_or_name
             existing_col_index = (
@@ -1644,7 +1632,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         )
         return self.select.__wrapped__(self, *select_columns, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def drop(self, *cols: t.Union[str, Column]) -> Self:
         # Separate string column names from Column objects for different handling
         column_objs, column_names = partition_to(lambda x: isinstance(x, str), cols, list, set)

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe/base/functions.py

@@ -37,9 +37,7 @@ def _get_session() -> _BaseSession:
 
 @meta()
 def col(column_name: t.Union[ColumnOrName, t.Any]) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    dialect = _BaseSession().input_dialect
+    dialect = _get_session().input_dialect
     if isinstance(column_name, str):
         col_expression = expression.to_column(column_name, dialect=dialect).transform(
             dialect.normalize_identifier
@@ -662,9 +660,7 @@ def grouping_id(*cols: ColumnOrName) -> Column:
 
 @meta()
 def input_file_name() -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    return Column(expression.Literal.string(_BaseSession()._last_loaded_file or ""))
+    return Column(expression.Literal.string(_get_session()._last_loaded_file or ""))
 
 
 @meta()
@@ -959,12 +955,10 @@ def current_timestamp() -> Column:
 
 @meta()
 def date_format(col: ColumnOrName, format: str) -> Column:
-    from sqlframe.base.session import _BaseSession
-
     return Column.invoke_expression_over_column(
         Column(expression.TimeStrToTime(this=Column.ensure_col(col).column_expression)),
         expression.TimeToStr,
-        format=_BaseSession().format_time(format),
+        format=_get_session().format_time(format),
     )
 
 
@@ -3378,10 +3372,9 @@ def get(col: ColumnOrName, index: t.Union[ColumnOrName, int]) -> Column:
 def get_active_spark_context() -> SparkContext:
     """Raise RuntimeError if SparkContext is not initialized,
     otherwise, returns the active SparkContext."""
-    from sqlframe.base.session import _BaseSession
     from sqlframe.spark.session import SparkSession
 
-    session: _BaseSession = _BaseSession()
+    session = _get_session()
     if not isinstance(session, SparkSession):
         raise RuntimeError("This function is only available in SparkSession.")
     return session.spark_session.sparkContext
@@ -6344,7 +6337,7 @@ def to_unix_timestamp(
     session = _get_session()
 
     if session._is_duckdb:
-        format = format or _BaseSession().default_time_format
+        format = format or session.default_time_format
         timestamp = Column.ensure_col(timestamp).cast("string")
 
     if format is not None:

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe/base/operations.py

@@ -27,10 +27,9 @@ class Operation(IntEnum):
     WHERE = 2
     GROUP_BY = 3
     HAVING = 4
-    SELECT_CONSTRAINED = 5
-    SELECT = 6
-    ORDER_BY = 7
-    LIMIT = 8
+    SELECT = 5
+    ORDER_BY = 6
+    LIMIT = 7
 
 
 # We want to decorate a function (self: DF, *args, **kwargs) -> T

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.39.3
+Version: 3.39.4
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.39.3 → sqlframe-3.39.4}/sqlframe.egg-info/requires.txt

@@ -1,6 +1,6 @@
 more-itertools
 prettytable<4
-sqlglot<27.9,>=24.0.0
+sqlglot<27.10,>=24.0.0
 typing_extensions
 
 [bigquery]

{sqlframe-3.39.3 → sqlframe-3.39.4}/tests/integration/test_int_dataframe.py

@@ -1892,97 +1892,6 @@ def test_drop_column_reference_join(
     compare_frames(df, dfs, sort=True)
 
 
-# TODO: This test exposes a bug in Spark where the column order is not preserved after a join
-# for some reason spark flips foo/bar to bar/foo
-# def test_left_join_column_order(
-#     pyspark_employee: PySparkDataFrame,
-#     get_df: t.Callable[[str], BaseDataFrame],
-#     compare_frames: t.Callable,
-# ):
-#     df1 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]).alias("df1")
-#     df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")
-#     df_joined = df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left")
-#
-#     employee = get_df("employee")
-#     dfs1 = employee.sparkSession.createDataFrame([{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]).alias("dfs1")
-#     dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
-#     dfs_joined = dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left")
-#
-#     compare_frames(df_joined, dfs_joined, sort=True)
-
-
-# def test_drop_column_join_column(
-#     pyspark_employee: PySparkDataFrame,
-#     get_df: t.Callable[[str], BaseDataFrame],
-#     compare_frames: t.Callable,
-# ):
-#     df1 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]).alias("df1")
-#     df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")
-#     df_joined = df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left").drop(F.col("df1.foo"))
-#
-#     employee = get_df("employee")
-#     dfs1 = employee.sparkSession.createDataFrame([{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]).alias("dfs1")
-#     dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
-#     dfs_joined = dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left").drop(SF.col("dfs1.foo"))
-#
-#     compare_frames(df_joined, dfs_joined, sort=True)
-
-
-def test_drop_column_join_column_df_reference(
-    pyspark_employee: PySparkDataFrame,
-    get_df: t.Callable[[str], BaseDataFrame],
-    compare_frames: t.Callable,
-):
-    df1 = pyspark_employee.sparkSession.createDataFrame(  # type: ignore
-        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
-    ).alias("df1")
-    df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")  # type: ignore
-    df_joined = (
-        df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left")
-        .drop(df1.foo)
-        .select(df1.bar, df2.baz, df2.foo)
-    )
-
-    employee = get_df("employee")
-    dfs1 = employee.sparkSession.createDataFrame(
-        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
-    ).alias("dfs1")
-    dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
-    dfs_joined = (
-        dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left")
-        .drop(dfs1.foo)
-        .select(dfs1.bar, dfs2.baz, dfs2.foo)
-    )
-
-    compare_frames(df_joined, dfs_joined, sort=True, compare_schema=False)
-
-
-def test_drop_join_column_unqualified(
-    pyspark_employee: PySparkDataFrame,
-    get_df: t.Callable[[str], BaseDataFrame],
-    compare_frames: t.Callable,
-):
-    df1 = pyspark_employee.sparkSession.createDataFrame(  # type: ignore
-        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
-    ).alias("df1")
-    df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")  # type: ignore
-    df_joined = (
-        df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left").drop("foo")
-        # select the columns to work around column order bug
-    )
-
-    employee = get_df("employee")
-    dfs1 = employee.sparkSession.createDataFrame(
-        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
-    ).alias("dfs1")
-    dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
-    dfs_joined = dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left").drop(
-        "foo"
-    )
-
-    compare_frames(df_joined, dfs_joined, sort=True, compare_schema=False)
-
-
 def test_limit(
     pyspark_employee: PySparkDataFrame,
     get_df: t.Callable[[str], BaseDataFrame],
@@ -2258,10 +2167,10 @@ def test_transform(
     get_df: t.Callable[[str], BaseDataFrame],
     compare_frames: t.Callable,
 ):
-    def cast_all_to_string_pyspark(input_df):
+    def cast_all_to_int_pyspark(input_df):
         return input_df.select([F.col(col_name).cast("string") for col_name in input_df.columns])
 
-    def cast_all_to_string_sqlframe(input_df):
+    def cast_all_to_int_sqlframe(input_df):
         return input_df.select([SF.col(col_name).cast("string") for col_name in input_df.columns])
 
     def sort_columns_asc(input_df):
@@ -2269,8 +2178,8 @@ def test_transform(
 
     employee = get_df("employee")
 
-    df = pyspark_employee.transform(cast_all_to_string_pyspark).transform(sort_columns_asc)
-    dfs = employee.transform(cast_all_to_string_sqlframe).transform(sort_columns_asc)
+    df = pyspark_employee.transform(cast_all_to_int_pyspark).transform(sort_columns_asc)
+    dfs = employee.transform(cast_all_to_int_sqlframe).transform(sort_columns_asc)
     compare_frames(df, dfs)