sqlframe 3.41.0__py3-none-any.whl → 3.43.0__py3-none-any.whl

sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.41.0'
-__version_tuple__ = version_tuple = (3, 41, 0)
+__version__ = version = '3.43.0'
+__version_tuple__ = version_tuple = (3, 43, 0)
 
-__commit_id__ = commit_id = 'g961d3fda2'
+__commit_id__ = commit_id = 'g38d15d998'

sqlframe/base/dataframe.py

@@ -1665,10 +1665,34 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         def should_drop_expression(expr: exp.Expression) -> bool:
             # Check against fully qualified Column objects and
             # Check against unqualified string column names (drop ALL columns with this name)
-            if expr.sql() in drop_sql or (
-                isinstance(expr, exp.Column) and expr.alias_or_name in column_names
-            ):
+            if expr.sql() in drop_sql:
                 return True
+
+            if isinstance(expr, exp.Column) and (alias_or_name := expr.alias_or_name):
+                # Check direct match first
+                if alias_or_name in column_names:
+                    return True
+
+                # Handle string column references that contain aliases
+                for col_name in column_names:
+                    if ("." in col_name) and alias_or_name == (col_name.split(".", maxsplit=1)[-1]):
+                        # Extract the column name part after the last dot
+                        return True
+
+                # Handle case where normalized columns have table qualifiers but actual expressions
+                # are unqualified. This happens when using aliased column references like
+                # f.col('df.foo')
+
+                # Check if any drop column matches by column name AND table qualifier
+                for drop_col in drop_cols:
+                    if ((drop_expression := drop_col.expression).alias_or_name) == alias_or_name:
+                        if expr_table := expr.table:
+                            drop_table = drop_expression.args.get("table")
+                            if (not drop_table) or (expr_table == drop_table):
+                                return True
+                        else:
+                            return True
+
             return False
 
         new_expressions = [expr for expr in current_expressions if not should_drop_expression(expr)]

sqlframe/base/functions.py

@@ -1974,8 +1974,30 @@ def regexp_replace(
     )
 
 
-@meta(unsupported_engines="duckdb")
+@meta()
 def initcap(col: ColumnOrName) -> Column:
+    session = _get_session()
+
+    if session._is_duckdb:
+        split_func = get_func_from_session("split")
+        transform_func = get_func_from_session("transform")
+        reduce_func = get_func_from_session("reduce")
+        upper_func = get_func_from_session("upper")
+        lower_func = get_func_from_session("lower")
+        length_func = get_func_from_session("length")
+        concat_func = get_func_from_session("concat")
+        concat_ws_func = get_func_from_session("concat_ws")
+        return reduce_func(
+            transform_func(
+                split_func(col, r"\s+"),
+                lambda w: concat_func(
+                    upper_func(w.substr(1, 1)), lower_func(w.substr(2, length_func(w) - 1))
+                ),
+            ),
+            None,
+            merge=lambda x, y: concat_ws_func(" ", x, y),
+        )
+
     return Column.invoke_expression_over_column(col, expression.Initcap)
 
 
@@ -2686,7 +2708,7 @@ def from_csv(
     return Column.invoke_anonymous_function(col, "FROM_CSV", schema)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def aggregate(
     col: ColumnOrName,
     initialValue: ColumnOrName,
@@ -2694,21 +2716,20 @@ def aggregate(
     finish: t.Optional[t.Callable[[Column], Column]] = None,
 ) -> Column:
     merge_exp = _get_lambda_from_func(merge)
+    kwargs = dict(
+        initial=initialValue,
+        merge=merge_exp,
+    )
+    session = _get_session()
     if finish is not None:
         finish_exp = _get_lambda_from_func(finish)
-        return Column.invoke_expression_over_column(
-            col,
-            expression.Reduce,
-            initial=initialValue,
-            merge=Column(merge_exp),
-            finish=Column(finish_exp),
-        )
-    return Column.invoke_expression_over_column(
-        col, expression.Reduce, initial=initialValue, merge=Column(merge_exp)
-    )
+        kwargs["finish"] = Column(finish_exp)
+    if session._is_duckdb:
+        kwargs.pop("initial", None)
+    return Column.invoke_expression_over_column(col, expression.Reduce, **kwargs)
 
 
-@meta(unsupported_engines=["bigquery", "duckdb", "postgres", "snowflake"])
+@meta(unsupported_engines="postgres")
 def transform(
     col: ColumnOrName,
     f: t.Union[t.Callable[[Column], Column], t.Callable[[Column, Column], Column]],

sqlframe/base/session.py

@@ -437,7 +437,7 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
                 for cte in df.expression.ctes:
                     if cte.alias_or_name not in expression_ctes:
                         ctes_to_add.append(cte)
-                expression.set("with", exp.With(expressions=expression.ctes + ctes_to_add))  # type: ignore
+                expression.set("with", exp.With(expressions=ctes_to_add + expression.ctes))  # type: ignore
 
             def replace_temp_view_name_with_cte(node: exp.Expression) -> exp.Expression:
                 if isinstance(node, exp.Table):

sqlframe/duckdb/dataframe.py

@@ -61,5 +61,5 @@ class DuckDBDataFrame(
     def toArrow(self, batch_size: t.Optional[int] = None) -> t.Union[ArrowTable, RecordBatchReader]:
         self._collect(skip_rows=True)
         if not batch_size:
-            return self.session._last_result.arrow()
+            return self.session._last_result.fetch_arrow_table()
         return self.session._last_result.fetch_record_batch(batch_size)

{sqlframe-3.41.0.dist-info → sqlframe-3.43.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.41.0
+Version: 3.43.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: more-itertools
 Requires-Dist: prettytable <4
-Requires-Dist: sqlglot <27.15,>=24.0.0
+Requires-Dist: sqlglot <27.16,>=24.0.0
 Requires-Dist: typing-extensions
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery-storage <3,>=2 ; extra == 'bigquery'
@@ -26,7 +26,7 @@ Requires-Dist: google-cloud-bigquery[pandas] <4,>=3 ; extra == 'bigquery'
 Provides-Extra: databricks
 Requires-Dist: databricks-sql-connector[pyarrow] <5,>=3.6 ; extra == 'databricks'
 Provides-Extra: dev
-Requires-Dist: duckdb <1.4,>=1.2 ; extra == 'dev'
+Requires-Dist: duckdb <1.5,>=1.2 ; extra == 'dev'
 Requires-Dist: findspark <3,>=2 ; extra == 'dev'
 Requires-Dist: mypy <1.19,>=1.10.0 ; extra == 'dev'
 Requires-Dist: openai <2,>=1.30 ; extra == 'dev'
@@ -50,7 +50,7 @@ Requires-Dist: mkdocs-material ==9.0.5 ; extra == 'docs'
 Requires-Dist: mkdocs ==1.4.2 ; extra == 'docs'
 Requires-Dist: pymdown-extensions ; extra == 'docs'
 Provides-Extra: duckdb
-Requires-Dist: duckdb <1.4,>=1.2 ; extra == 'duckdb'
+Requires-Dist: duckdb <1.5,>=1.2 ; extra == 'duckdb'
 Requires-Dist: pandas <3,>=2 ; extra == 'duckdb'
 Provides-Extra: openai
 Requires-Dist: openai <2,>=1.30 ; extra == 'openai'

{sqlframe-3.41.0.dist-info → sqlframe-3.43.0.dist-info}/RECORD

@@ -1,20 +1,20 @@
 sqlframe/__init__.py,sha256=SB80yLTITBXHI2GCDS6n6bN5ObHqgPjfpRPAUwxaots,3403
-sqlframe/_version.py,sha256=dgLbChf8wsQ5H9o1FXadKfn_qB0pcaOHICUJO8Rhj6U,714
+sqlframe/_version.py,sha256=GkrwqhnbavsDQW5LQsruCWe67_xixzoHIDhkzjlbf38,714
 sqlframe/py.typed,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
 sqlframe/base/catalog.py,sha256=-YulM2BMK8MoWbXi05AsJIPxd4AuiZDBCZuk4HoeMlE,38900
 sqlframe/base/column.py,sha256=f6rK6-hTiNx9WwJP7t6tqL3xEC2gwERPDlhWCS5iCBw,21417
-sqlframe/base/dataframe.py,sha256=Kl3WycARIWBBIze0enmZDGkfOt65mZDQ2hx_6pxRsxI,87329
+sqlframe/base/dataframe.py,sha256=wT3R2Qmq4edOefxXCRepiMDNJfVOIvSf6eHd6GOn4i0,88543
 sqlframe/base/decorators.py,sha256=IhE5xNQDkwJHacCvulq5WpUKyKmXm7dL2A3o5WuKGP4,2131
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
 sqlframe/base/function_alternatives.py,sha256=aTu3nQhIAkZoxrI1IpjpaHEAMxBNms0AnhS0EMR-TwY,51727
-sqlframe/base/functions.py,sha256=vlPGxKlgU1oFmXD8WDClpZlDvvC4L4q5qJs4qrG9xjw,228101
+sqlframe/base/functions.py,sha256=QaCeMMBLz69LE-73x4ksXN6NbZlsshYADg-F8yRXTPA,228816
 sqlframe/base/group.py,sha256=fBm8EUve7W7xz11nybTXr09ih-yZxL_vvEiZVE1eb_0,12025
 sqlframe/base/normalize.py,sha256=YPeopWr8ZRjevArYfrM-DZBkQp4t4UfAEwynoj4VvcU,11773
 sqlframe/base/operations.py,sha256=g-YNcbvNKTOBbYm23GKfB3fmydlR7ZZDAuZUtXIHtzw,4438
 sqlframe/base/readerwriter.py,sha256=b1CZgOZv-8h0sC3PWqPVAwAwlDMjpmRys6FGhugKspU,31391
-sqlframe/base/session.py,sha256=99X-ShK9ohHCX6WdIJs0HhjfK23snaE3Gv6RYc5wqUI,27687
+sqlframe/base/session.py,sha256=jMm18v5MYW7Z61pXda-yd-WNYYwnYU9i2jIaT8gKSvA,27687
 sqlframe/base/table.py,sha256=rCeh1W5SWbtEVfkLAUiexzrZwNgmZeptLEmLcM1ABkE,6961
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
 sqlframe/base/types.py,sha256=OktuJ5f7tEogOW0oupI0RBlHfzZMmKh7zGLke9cwllo,12305
@@ -55,7 +55,7 @@ sqlframe/databricks/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0
 sqlframe/duckdb/__init__.py,sha256=KAw_uZEhFMwi3D9Wj6AgHAKqLNk-EAx2uDIYu56oL44,872
 sqlframe/duckdb/catalog.py,sha256=89FCSJglMbOxonk3IXmlkMcdXCfMdePpGfqlbkkB_d0,5107
 sqlframe/duckdb/column.py,sha256=E1tUa62Y5HajkhgFuebU9zohrGyieudcHzTT8gfalio,40
-sqlframe/duckdb/dataframe.py,sha256=Z8_K69UQGZVeBfVGXVwIJP8OMuIvNBB3DPKTP3Lfu4w,1908
+sqlframe/duckdb/dataframe.py,sha256=G99muPe0kUV6pBO4nx5zzcjb4H1s9JA0WzUUX8_vjE0,1920
 sqlframe/duckdb/functions.py,sha256=ix2efGGD4HLaY1rtCtEd3IrsicGEVGiBAeKOo5OD8rA,424
 sqlframe/duckdb/functions.pyi,sha256=hDjpT-tGDO8LyElcno5YYRUnJg1dXXbGcRjJ69Zqk_U,12542
 sqlframe/duckdb/group.py,sha256=IkhbW42Ng1U5YT3FkIdiB4zBqRkW4QyTb-1detY1e_4,383
@@ -130,8 +130,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.41.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.41.0.dist-info/METADATA,sha256=O8Y62mZw3zncgCc6RWb8i4_zHyeNySrjl6AeQV_tflc,9070
-sqlframe-3.41.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.41.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.41.0.dist-info/RECORD,,
+sqlframe-3.43.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.43.0.dist-info/METADATA,sha256=-M-YLCPxdylEzhy7aPCDjVQc4lkJN47zxzJJdWmvVUo,9070
+sqlframe-3.43.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.43.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.43.0.dist-info/RECORD,,