sqlframe 3.41.0__tar.gz → 3.43.0__tar.gz

{sqlframe-3.41.0 → sqlframe-3.43.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.41.0
+Version: 3.43.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.41.0 → sqlframe-3.43.0}/docs/bigquery.md

@@ -526,6 +526,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.41.0 → sqlframe-3.43.0}/docs/duckdb.md

@@ -268,6 +268,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [abs](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.abs.html)
 * [acos](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.acos.html)
 * [add_months](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.add_months.html)
+* [aggregate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.aggregate.html)
 * [any_value](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.any_value.html)
     * Always ignores nulls
 * [approxCountDistinct](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.approxCountDistinct.html)
@@ -380,6 +381,7 @@ See something that you would like to see supported? [Open an issue](https://gith
     * Uses a different hash algorithm than Spark
 * [hex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.hex.html)
 * [hour](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.hour.html)
+* [initcap](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.initcap.html)
 * [input_file_name](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.input_file_name.html)
 * [instr](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.instr.html)
 * [isnan](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.isnan.html)
@@ -433,6 +435,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [radians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.radians.html)
 * [rand](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rand.html)
 * [rank](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rank.html)
+* [reduce](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.reduce.html)
 * [regexp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp.html)
 * [regexp_extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract.html)
 * [regexp_extract_all](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract_all.html)
@@ -484,6 +487,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
 * [to_unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_unix_timestamp.html)
     * The values must match the format string (null will not be returned if they do not)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-3.41.0 → sqlframe-3.43.0}/docs/snowflake.md

@@ -522,6 +522,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
 * [to_timestamp_ntz](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp_ntz.html)
+* [transform](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.transform.html)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

sqlframe-3.43.0/mypy.ini

@@ -0,0 +1,13 @@
+[mypy]
+
+[mypy-duckdb.*]
+ignore_missing_imports = True
+
+[mypy-pyarrow.*]
+ignore_missing_imports = True
+
+[mypy-findspark.*]
+ignore_missing_imports = True
+
+[mypy-redshift_connector.*]
+ignore_missing_imports = True

{sqlframe-3.41.0 → sqlframe-3.43.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.15",
+        "sqlglot>=24.0.0,<27.16",
         "typing_extensions",
         "more-itertools",
     ],
@@ -30,7 +30,7 @@ setup(
             "google-cloud-bigquery-storage>=2,<3",
         ],
         "dev": [
-            "duckdb>=1.2,<1.4",
+            "duckdb>=1.2,<1.5",
             "findspark>=2,<3",
             "mypy>=1.10.0,<1.19",
             "openai>=1.30,<2",
@@ -56,7 +56,7 @@ setup(
             "pymdown-extensions",
         ],
         "duckdb": [
-            "duckdb>=1.2,<1.4",
+            "duckdb>=1.2,<1.5",
             "pandas>=2,<3",
         ],
         "openai": [

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.41.0'
-__version_tuple__ = version_tuple = (3, 41, 0)
+__version__ = version = '3.43.0'
+__version_tuple__ = version_tuple = (3, 43, 0)
 
-__commit_id__ = commit_id = 'g961d3fda2'
+__commit_id__ = commit_id = 'g38d15d998'

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe/base/dataframe.py

@@ -1665,10 +1665,34 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         def should_drop_expression(expr: exp.Expression) -> bool:
             # Check against fully qualified Column objects and
             # Check against unqualified string column names (drop ALL columns with this name)
-            if expr.sql() in drop_sql or (
-                isinstance(expr, exp.Column) and expr.alias_or_name in column_names
-            ):
+            if expr.sql() in drop_sql:
                 return True
+
+            if isinstance(expr, exp.Column) and (alias_or_name := expr.alias_or_name):
+                # Check direct match first
+                if alias_or_name in column_names:
+                    return True
+
+                # Handle string column references that contain aliases
+                for col_name in column_names:
+                    if ("." in col_name) and alias_or_name == (col_name.split(".", maxsplit=1)[-1]):
+                        # Extract the column name part after the last dot
+                        return True
+
+                # Handle case where normalized columns have table qualifiers but actual expressions
+                # are unqualified. This happens when using aliased column references like
+                # f.col('df.foo')
+
+                # Check if any drop column matches by column name AND table qualifier
+                for drop_col in drop_cols:
+                    if ((drop_expression := drop_col.expression).alias_or_name) == alias_or_name:
+                        if expr_table := expr.table:
+                            drop_table = drop_expression.args.get("table")
+                            if (not drop_table) or (expr_table == drop_table):
+                                return True
+                        else:
+                            return True
+
             return False
 
         new_expressions = [expr for expr in current_expressions if not should_drop_expression(expr)]

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe/base/functions.py

@@ -1974,8 +1974,30 @@ def regexp_replace(
     )
 
 
-@meta(unsupported_engines="duckdb")
+@meta()
 def initcap(col: ColumnOrName) -> Column:
+    session = _get_session()
+
+    if session._is_duckdb:
+        split_func = get_func_from_session("split")
+        transform_func = get_func_from_session("transform")
+        reduce_func = get_func_from_session("reduce")
+        upper_func = get_func_from_session("upper")
+        lower_func = get_func_from_session("lower")
+        length_func = get_func_from_session("length")
+        concat_func = get_func_from_session("concat")
+        concat_ws_func = get_func_from_session("concat_ws")
+        return reduce_func(
+            transform_func(
+                split_func(col, r"\s+"),
+                lambda w: concat_func(
+                    upper_func(w.substr(1, 1)), lower_func(w.substr(2, length_func(w) - 1))
+                ),
+            ),
+            None,
+            merge=lambda x, y: concat_ws_func(" ", x, y),
+        )
+
     return Column.invoke_expression_over_column(col, expression.Initcap)
 
 
@@ -2686,7 +2708,7 @@ def from_csv(
     return Column.invoke_anonymous_function(col, "FROM_CSV", schema)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def aggregate(
     col: ColumnOrName,
     initialValue: ColumnOrName,
@@ -2694,21 +2716,20 @@ def aggregate(
     finish: t.Optional[t.Callable[[Column], Column]] = None,
 ) -> Column:
     merge_exp = _get_lambda_from_func(merge)
+    kwargs = dict(
+        initial=initialValue,
+        merge=merge_exp,
+    )
+    session = _get_session()
     if finish is not None:
         finish_exp = _get_lambda_from_func(finish)
-        return Column.invoke_expression_over_column(
-            col,
-            expression.Reduce,
-            initial=initialValue,
-            merge=Column(merge_exp),
-            finish=Column(finish_exp),
-        )
-    return Column.invoke_expression_over_column(
-        col, expression.Reduce, initial=initialValue, merge=Column(merge_exp)
-    )
+        kwargs["finish"] = Column(finish_exp)
+    if session._is_duckdb:
+        kwargs.pop("initial", None)
+    return Column.invoke_expression_over_column(col, expression.Reduce, **kwargs)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines="postgres")
 def transform(
     col: ColumnOrName,
     f: t.Union[t.Callable[[Column], Column], t.Callable[[Column, Column], Column]],

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe/base/session.py

@@ -437,7 +437,7 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
                 for cte in df.expression.ctes:
                     if cte.alias_or_name not in expression_ctes:
                         ctes_to_add.append(cte)
-                expression.set("with", exp.With(expressions=expression.ctes + ctes_to_add))  # type: ignore
+                expression.set("with", exp.With(expressions=ctes_to_add + expression.ctes))  # type: ignore
 
             def replace_temp_view_name_with_cte(node: exp.Expression) -> exp.Expression:
                 if isinstance(node, exp.Table):

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe/duckdb/dataframe.py

@@ -61,5 +61,5 @@ class DuckDBDataFrame(
     def toArrow(self, batch_size: t.Optional[int] = None) -> t.Union[ArrowTable, RecordBatchReader]:
         self._collect(skip_rows=True)
         if not batch_size:
-            return self.session._last_result.arrow()
+            return self.session._last_result.fetch_arrow_table()
         return self.session._last_result.fetch_record_batch(batch_size)

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.41.0
+Version: 3.43.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ LICENSE
 Makefile
 README.md
 mkdocs.yml
+mypy.ini
 pytest.ini
 renovate.json
 setup.cfg

{sqlframe-3.41.0 → sqlframe-3.43.0}/sqlframe.egg-info/requires.txt

@@ -1,6 +1,6 @@
 more-itertools
 prettytable<4
-sqlglot<27.15,>=24.0.0
+sqlglot<27.16,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -11,7 +11,7 @@ google-cloud-bigquery[pandas]<4,>=3
 databricks-sql-connector[pyarrow]<5,>=3.6
 
 [dev]
-duckdb<1.4,>=1.2
+duckdb<1.5,>=1.2
 findspark<3,>=2
 mypy<1.19,>=1.10.0
 openai<2,>=1.30
@@ -37,7 +37,7 @@ mkdocs==1.4.2
 pymdown-extensions
 
 [duckdb]
-duckdb<1.4,>=1.2
+duckdb<1.5,>=1.2
 pandas<3,>=2
 
 [openai]

{sqlframe-3.41.0 → sqlframe-3.43.0}/tests/integration/engines/duck/test_duckdb_dataframe.py

@@ -262,7 +262,7 @@ def test_explain(duckdb_employee: DuckDBDataFrame, capsys):
 ┌───────────────────────────┐
 │      COLUMN_DATA_SCAN     │
 │    ────────────────────   │
-│          ~5 Rows          │
+│          ~5 rows          │
 └───────────────────────────┘
 """.strip()
     )

{sqlframe-3.41.0 → sqlframe-3.43.0}/tests/integration/engines/duck/test_duckdb_session.py

@@ -1,5 +1,5 @@
 import pytest
-from duckdb.duckdb import ConnectionException
+from duckdb import ConnectionException
 from sqlglot import exp
 
 from sqlframe.base.types import Row

{sqlframe-3.41.0 → sqlframe-3.43.0}/tests/integration/test_int_dataframe.py

@@ -1865,6 +1865,18 @@ def test_drop_column_single(
     compare_frames(df, dfs)
 
 
+def test_drop_alias_column(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    employee = get_df("employee")
+    df = pyspark_employee.select(F.col("fname"), F.col("age")).drop("age")
+
+    dfs = employee.alias("df").select(SF.col("fname"), SF.col("age")).drop("df.age")
+    compare_frames(df, dfs)
+
+
 def test_drop_column_reference_join(
     pyspark_employee: PySparkDataFrame,
     pyspark_store: PySparkDataFrame,
@@ -1892,6 +1904,61 @@ def test_drop_column_reference_join(
     compare_frames(df, dfs, sort=True)
 
 
+def test_drop_column_join_column_df_reference(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df1 = pyspark_employee.sparkSession.createDataFrame(  # type: ignore
+        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
+    ).alias("df1")
+    df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")  # type: ignore
+    df_joined = (
+        df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left")
+        .drop(df1.foo)
+        .select(df1.bar, df2.baz, df2.foo)
+    )
+
+    employee = get_df("employee")
+    dfs1 = employee.sparkSession.createDataFrame(
+        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
+    ).alias("dfs1")
+    dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
+    dfs_joined = (
+        dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left")
+        .drop(dfs1.foo)
+        .select(dfs1.bar, dfs2.baz, dfs2.foo)
+    )
+
+    compare_frames(df_joined, dfs_joined, sort=True, compare_schema=False)
+
+
+def test_drop_join_column_unqualified(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df1 = pyspark_employee.sparkSession.createDataFrame(  # type: ignore
+        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
+    ).alias("df1")
+    df2 = pyspark_employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("df2")  # type: ignore
+    df_joined = (
+        df1.join(df2, on=F.col("df1.foo") == F.col("df2.foo"), how="left").drop("foo")
+        # select the columns to work around column order bug
+    )
+
+    employee = get_df("employee")
+    dfs1 = employee.sparkSession.createDataFrame(
+        [{"foo": 0, "bar": "a"}, {"foo": 1, "bar": "b"}]
+    ).alias("dfs1")
+    dfs2 = employee.sparkSession.createDataFrame([{"foo": 0, "baz": 1.5}]).alias("dfs2")
+    dfs_joined = dfs1.join(dfs2, on=SF.col("dfs1.foo") == SF.col("dfs2.foo"), how="left").drop(
+        "foo"
+    )
+
+    compare_frames(df_joined, dfs_joined, sort=True, compare_schema=False)
+
+
 def test_limit(
     pyspark_employee: PySparkDataFrame,
     get_df: t.Callable[[str], BaseDataFrame],