fugue 0.9.2.dev1__py3-none-any.whl → 0.9.3__py3-none-any.whl

fugue/_utils/io.py

@@ -20,6 +20,10 @@ class FileParser(object):
         self._has_glob = "*" in path or "?" in path
         self._raw_path = path
         self._fs, self._fs_path = url_to_fs(path)
+        if not self._has_glob and self._fs.isdir(self._fs_path):
+            self._is_dir = True
+        else:
+            self._is_dir = False
         if not self.is_local:
             self._path = self._fs.unstrip_protocol(self._fs_path)
         else:
@@ -43,11 +47,15 @@ class FileParser(object):
         return self
 
     @property
-    def has_glob(self):
+    def is_dir(self) -> bool:
+        return self._is_dir
+
+    @property
+    def has_glob(self) -> bool:
         return self._has_glob
 
     @property
-    def is_local(self):
+    def is_local(self) -> bool:
         return isinstance(self._fs, LocalFileSystem)
 
     def join(self, path: str, format_hint: Optional[str] = None) -> "FileParser":
@@ -65,6 +73,10 @@ class FileParser(object):
     def path(self) -> str:
         return self._path
 
+    def as_dir_path(self) -> str:
+        assert_or_throw(self.is_dir, f"{self.raw_path} is not a directory")
+        return self.path + self._fs.sep
+
     @property
     def raw_path(self) -> str:
         return self._raw_path

fugue/dataframe/function_wrapper.py

@@ -91,7 +91,9 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    if v.is_per_row:
+                    if v.is_per_row:  # pragma: no cover
+                        # TODO: this branch is used only if row annotations
+                        # are allowed as input
                         assert_or_throw(
                             row_param_info is None,
                             lambda: ValueError("only one row parameter is allowed"),
@@ -110,7 +112,9 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             raise ValueError(f"{p} are not acceptable parameters")
         if row_param_info is None:
             return self._run_func(rargs, output, output_schema, ctx, raw=False)
-        else:  # input contains row parameter
+        else:  # pragma: no cover
+            # input contains row parameter
+            # TODO: this branch is used only if row annotations are allowed as input
 
             def _dfs() -> Iterable[Any]:
                 k, v, df = row_param_info
@@ -154,6 +158,7 @@ fugue_annotated_param = DataFrameFunctionWrapper.annotated_param
         annotation == Callable
         or annotation == callable  # pylint: disable=comparison-with-callable
         or str(annotation).startswith("typing.Callable")
+        or str(annotation).startswith("collections.abc.Callable")
     ),
 )
 class _CallableParam(AnnotatedParam):
@@ -168,6 +173,9 @@ class _CallableParam(AnnotatedParam):
         or annotation == Optional[callable]
         or str(annotation).startswith("typing.Union[typing.Callable")  # 3.8-
         or str(annotation).startswith("typing.Optional[typing.Callable")  # 3.9+
+        or str(annotation).startswith(
+            "typing.Optional[collections.abc.Callable]"
+        )  # 3.9+
     ),
 )
 class _OptionalCallableParam(AnnotatedParam):
@@ -233,7 +241,8 @@ class DataFrameParam(_DataFrameParamBase):
 
 
 @fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
-class RowParam(_DataFrameParamBase):
+class RowParam(_DataFrameParamBase):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
     @property
     def is_per_row(self) -> bool:
         return True
@@ -243,7 +252,8 @@ class RowParam(_DataFrameParamBase):
 
 
 @fugue_annotated_param(Dict[str, Any])
-class DictParam(RowParam):
+class DictParam(RowParam):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
     def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
         yield from df.as_dict_iterable()
 

fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspqr][fF]?x*z?$", "^[lspqr]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspqr][fF]?x*z?$", "^[lspnqr]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore

fugue/rpc/flask.py

@@ -1,6 +1,5 @@
-import base64
 import logging
-import cloudpickle
+import json
 from threading import Thread
 from typing import Any, Optional, Tuple, Dict, List
 
@@ -60,6 +59,7 @@ class FlaskRPCServer(RPCServer):
             -1.0 if timeout is None else to_timedelta(timeout).total_seconds()
         )
         self._server: Optional[FlaskRPCServer._Thread] = None
+        self._log = logging.getLogger()
 
     def make_client(self, handler: Any) -> RPCClient:
         """Add ``handler`` and correspondent :class:`~.FlaskRPCClient`
@@ -77,6 +77,14 @@ class FlaskRPCServer(RPCServer):
 
     def start_server(self) -> None:
         """Start Flask RPC server"""
+        msg = (
+            "Starting RPC server on %s:%s. "
+            "This server has no authentication and relies on network isolation. "
+            "Ensure proper VPC/firewall configuration in production. "
+            "See https://fugue-tutorials.readthedocs.io/tutorials/resources/"
+            "security.html"
+        )
+        self._log.warning(msg, self._host, self._port)
         app = Flask("FlaskRPCServer")
         app.route("/invoke", methods=["POST"])(self._invoke)
         self._server = FlaskRPCServer._Thread(app, self._host, self._port)
@@ -122,10 +130,10 @@ class FlaskRPCClient(RPCClient):
 
 
 def _encode(*args: Any, **kwargs: Any) -> str:
-    data = base64.b64encode(cloudpickle.dumps(dict(args=args, kwargs=kwargs)))
-    return data.decode("ascii")
+    data = json.dumps(dict(args=args, kwargs=kwargs))
+    return data
 
 
 def _decode(data: str) -> Tuple[List[Any], Dict[str, Any]]:
-    data = cloudpickle.loads(base64.b64decode(data.encode("ascii")))
+    data = json.loads(data)
     return data["args"], data["kwargs"]  # type: ignore

{fugue-0.9.2.dev1.dist-info → fugue-0.9.3.dist-info}/METADATA

@@ -1,13 +1,12 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.2.dev1
+Version: 0.9.3
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
 Author-email: hello@fugue.ai
 License: Apache-2.0
 Keywords: distributed spark dask ray sql dsl domain specific language
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -17,67 +16,81 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad >=0.9.7
-Requires-Dist: adagio >=0.2.4
-Provides-Extra: all
-Requires-Dist: qpd >=0.4.4 ; extra == 'all'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'all'
-Requires-Dist: sqlglot ; extra == 'all'
-Requires-Dist: jinja2 ; extra == 'all'
-Requires-Dist: pyspark >=3.1.1 ; extra == 'all'
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'all'
-Requires-Dist: dask-sql ; extra == 'all'
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'all'
-Requires-Dist: notebook ; extra == 'all'
-Requires-Dist: jupyterlab ; extra == 'all'
-Requires-Dist: ipython >=7.10.0 ; extra == 'all'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'all'
-Requires-Dist: pyarrow >=6.0.1 ; extra == 'all'
-Requires-Dist: pandas <2.2,>=2.0.2 ; extra == 'all'
-Requires-Dist: ibis-framework ; extra == 'all'
-Requires-Dist: polars ; extra == 'all'
-Provides-Extra: cpp_sql_parser
-Requires-Dist: fugue-sql-antlr[cpp] >=0.2.0 ; extra == 'cpp_sql_parser'
+License-File: LICENSE
+Requires-Dist: triad>=1.0.0
+Requires-Dist: adagio>=0.2.6
+Provides-Extra: sql
+Requires-Dist: qpd>=0.4.4; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: sqlglot<28; extra == "sql"
+Requires-Dist: jinja2; extra == "sql"
+Provides-Extra: cpp-sql-parser
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Provides-Extra: dask
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'dask'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'dask'
-Requires-Dist: pandas >=2.0.2 ; extra == 'dask'
-Requires-Dist: dask[dataframe,distributed] >=2024.4.0 ; (python_version >= "3.11.9") and extra == 'dask'
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "dask"
+Requires-Dist: pyarrow>=7.0.0; extra == "dask"
+Requires-Dist: pandas>=2.0.2; extra == "dask"
+Provides-Extra: ray
+Requires-Dist: ray[data]>=2.30.0; extra == "ray"
+Requires-Dist: duckdb>=0.5.0; extra == "ray"
+Requires-Dist: pyarrow>=7.0.0; extra == "ray"
+Requires-Dist: pandas<2.2; extra == "ray"
 Provides-Extra: duckdb
-Requires-Dist: qpd >=0.4.4 ; extra == 'duckdb'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'duckdb'
-Requires-Dist: sqlglot ; extra == 'duckdb'
-Requires-Dist: jinja2 ; extra == 'duckdb'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'duckdb'
-Requires-Dist: numpy ; extra == 'duckdb'
+Requires-Dist: qpd>=0.4.4; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: sqlglot<28; extra == "duckdb"
+Requires-Dist: jinja2; extra == "duckdb"
+Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
+Requires-Dist: numpy; extra == "duckdb"
+Provides-Extra: polars
+Requires-Dist: polars; extra == "polars"
 Provides-Extra: ibis
-Requires-Dist: qpd >=0.4.4 ; extra == 'ibis'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'ibis'
-Requires-Dist: sqlglot ; extra == 'ibis'
-Requires-Dist: jinja2 ; extra == 'ibis'
-Requires-Dist: ibis-framework ; extra == 'ibis'
-Requires-Dist: pandas <2.2 ; extra == 'ibis'
+Requires-Dist: qpd>=0.4.4; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: sqlglot<28; extra == "ibis"
+Requires-Dist: jinja2; extra == "ibis"
+Requires-Dist: ibis-framework[pandas]; extra == "ibis"
+Requires-Dist: pandas<2.2; extra == "ibis"
 Provides-Extra: notebook
-Requires-Dist: notebook ; extra == 'notebook'
-Requires-Dist: jupyterlab ; extra == 'notebook'
-Requires-Dist: ipython >=7.10.0 ; extra == 'notebook'
-Provides-Extra: polars
-Requires-Dist: polars ; extra == 'polars'
-Provides-Extra: ray
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'ray'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'ray'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'ray'
-Requires-Dist: pandas <2.2 ; extra == 'ray'
-Provides-Extra: spark
-Requires-Dist: pyspark >=3.1.1 ; extra == 'spark'
-Provides-Extra: sql
-Requires-Dist: qpd >=0.4.4 ; extra == 'sql'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'sql'
-Requires-Dist: sqlglot ; extra == 'sql'
-Requires-Dist: jinja2 ; extra == 'sql'
+Requires-Dist: notebook; extra == "notebook"
+Requires-Dist: jupyterlab; extra == "notebook"
+Requires-Dist: ipython>=7.10.0; extra == "notebook"
+Provides-Extra: all
+Requires-Dist: qpd>=0.4.4; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: sqlglot<28; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: pyspark>=3.1.1; extra == "all"
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "all"
+Requires-Dist: dask-sql; extra == "all"
+Requires-Dist: ray[data]>=2.30.0; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Requires-Dist: ipython>=7.10.0; extra == "all"
+Requires-Dist: duckdb>=0.5.0; extra == "all"
+Requires-Dist: pyarrow>=6.0.1; extra == "all"
+Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
+Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
+Requires-Dist: polars; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Fugue
 
@@ -355,4 +368,3 @@ View some of our latest conferences presentations and content. For a more comple
 *   [Large Scale Data Validation with Spark and Dask (PyCon US)](https://www.youtube.com/watch?v=2AdvBgjO_3Q)
 *   [FugueSQL - The Enhanced SQL Interface for Pandas, Spark, and Dask DataFrames (PyData Global)](https://www.youtube.com/watch?v=OBpnGYjNBBI)
 *   [Distributed Hybrid Parameter Tuning](https://www.youtube.com/watch?v=_GBjqskD8Qk)
-

{fugue-0.9.2.dev1.dist-info → fugue-0.9.3.dist-info}/RECORD

@@ -10,7 +10,7 @@ fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/_utils/display.py,sha256=JV8oDA7efHm1wceZulCBOY5dMvjbWHvIm6ASisKfoWY,3164
 fugue/_utils/exception.py,sha256=SFIjwjV4CIEovp3P9k7ePNOFB12A5D8hDdhtfFUeM5Y,2247
 fugue/_utils/interfaceless.py,sha256=wI0H6L4W_1uQjh9tpjgT9HzN-fbrrtXXHC1x6Q_rrPg,2203
-fugue/_utils/io.py,sha256=adrtj6Dq0ti426DNlkliApbTkp8b3bfBysAiE5MVQVc,9265
+fugue/_utils/io.py,sha256=5twd99LBzHtIMT67il1qwnEUa5n13WZmVKNd1shO4No,9649
 fugue/_utils/misc.py,sha256=_huy0eylmRTEFoReGR2M4rbAI8m79hFcfY5bDceVEXU,887
 fugue/_utils/registry.py,sha256=lrbzTdUEVnW6paBGDj-Yb-aTIbP5mjCqrXuRU9_N6os,316
 fugue/bag/__init__.py,sha256=0Q0_rnrEThrTx2U-1xGNyAg95idp_xcnywymIcW4Xck,46
@@ -31,7 +31,7 @@ fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvr
 fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=cG-0ICf-WxgxmEJIdU2jx4GzYhVfP69AyzjTjNDpXGE,17710
+fugue/dataframe/function_wrapper.py,sha256=7Sb6XrWTD_swtHJbHDWZRxHvFNWkERynnCDzLM0wSbo,18340
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
 fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
@@ -61,11 +61,11 @@ fugue/extensions/processor/convert.py,sha256=zG0lMtHGwY5TsqK4eplbMdlTg7J_PD3HbI0
 fugue/extensions/processor/processor.py,sha256=czhQlQgMpAXXoLVAX9Q0TFUMYEEhsgufTammxcKSmOY,1665
 fugue/extensions/transformer/__init__.py,sha256=VD6d-8xW1Yl8fUPj43cBWNR9pCOlYD9xWyGIHAlHwvI,456
 fugue/extensions/transformer/constants.py,sha256=76DfpoTOGQ8gp5XtCs_xznfbr_H015-prXpHWSqMNDU,59
-fugue/extensions/transformer/convert.py,sha256=SU_KvzZp_nV8oCxZGx7qDsdCE0CJ--8UAp5m8z4d4HY,23386
+fugue/extensions/transformer/convert.py,sha256=zDDIpZawMnHFarjZNZAyiw1jfyXGuPjnvgQk9jpYLak,23384
 fugue/extensions/transformer/transformer.py,sha256=zhOUgyv5-DPxYd1CP_98WeEw-zUgwknRnPW_6di-q3g,9098
 fugue/rpc/__init__.py,sha256=3GzUl4QZQuCChjD7eaTJW8tnTwfke6ZY9r9g5nCeBZ8,167
 fugue/rpc/base.py,sha256=3Fq5SvwLZqw9NXru3r32WuJKBGFr9bl7nFgy6e9boGo,8470
-fugue/rpc/flask.py,sha256=1oD0dE4Q-0PKeZ7RG3c0pfwyx21dUavfXWORu_gV7mg,4485
+fugue/rpc/flask.py,sha256=VzJEND8Pqatf6pYYT9LDXeO1JDMmYAOY0lm8XUncKbA,4807
 fugue/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/sql/_utils.py,sha256=khpjGeFCVlaqf2JIYvS4TVTJO3fe5-8bEsvy6AIP_5Q,2083
 fugue/sql/_visitors.py,sha256=2pc0J-AHJAiIexsKgNjcgrCGOyhC3_7rzonSgtjy--k,33844
@@ -82,6 +82,7 @@ fugue/workflow/api.py,sha256=uQoxPSCZ91-ST4vwuPWG7qioRGW4eo-Sgi3DdwtSL4k,12495
 fugue/workflow/input.py,sha256=V_zLDNzndmQuYJAPXtdK4n-vOp7LrimGIf_wQtwf2mc,321
 fugue/workflow/module.py,sha256=ajyqgMwX6hFMZY9xp4Bp1Q-Zdta0p5f_W_n_SNrc4LE,5547
 fugue/workflow/workflow.py,sha256=-SFCXkyxgXbS6DpQGSBox4d3Ws3psIlB6PnraJLSu9Y,88219
+fugue-0.9.3.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
 fugue_contrib/__init__.py,sha256=QJioX-r2AiU7Pvt24M-k2c4vNq29qpK-3WNUde7ucck,222
 fugue_contrib/contrib.py,sha256=3B--6oIVBMZ-GwjIOXwZqYqkloH7Cxfq1I8vkwl2yPk,267
 fugue_contrib/seaborn/__init__.py,sha256=NuVv8EI4Om4gHcHwYO8ddextLQqw24vDj8qJio3E1MU,1405
@@ -89,25 +90,26 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
+fugue_dask/_dask_sql_wrapper.py,sha256=lj38gJIOdoMV9W44gpwzLjUEtPVsQNKjRWuEkfI7-PM,2618
 fugue_dask/_io.py,sha256=pl4F7mbVgP7Rwh1FFG7xfOz2TBZRUj1l3lLvDY4jOf4,6020
-fugue_dask/_utils.py,sha256=1uplEqvpCDZDp2YdwJxa6cuGScpgG9VvN3057J02bys,8956
-fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=60IiwYRBVhN-pX3v6i9BZ8Pa4bcSh5UoklvCScM_XAM,21361
+fugue_dask/_utils.py,sha256=0R0pCh4B47kQsAS_o0QGaosIqVcZnSakm6pfMB7fSXs,9059
+fugue_dask/dataframe.py,sha256=4Dvckpc4mlld2WsEFTTemxoA1zYK8Cn6jMKxUxYQCEE,13491
+fugue_dask/execution_engine.py,sha256=mFN_IurhdBEu8C5OreqpGSRdTbTBqSpzJO2dMQzEF-o,21264
 fugue_dask/registry.py,sha256=jepWKH55VWNIWV3pOF5vpCl2OpO0rI1IULx5GM2Gk6w,2274
 fugue_dask/tester.py,sha256=E7BZjgFpJgrHsLMKzvSO5im5OwocYcratjzulJSQZl0,718
 fugue_duckdb/__init__.py,sha256=ZzhmAWbROR1YL9Kmlt7OlwkgPZzFhsSdwLV2pFmAqGI,268
 fugue_duckdb/_io.py,sha256=vnd8m8C6XeMCBJBbAdA5h695NMfsduQrvONyS0HcEFA,8475
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
-fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
-fugue_duckdb/execution_engine.py,sha256=IZDmSAtOMJGvulTStxjTmsqJyI5QRNyxBgSMlFMSrBI,20389
+fugue_duckdb/dataframe.py,sha256=LAPoPOad9hgGhjyhlMGMfrnhkyBKe06Xzn6eP1hkl-w,8504
+fugue_duckdb/execution_engine.py,sha256=3f5hbWcX1y9mAtfFixrri-snYxVIQAf4HOgo9fHbDwQ,20385
 fugue_duckdb/registry.py,sha256=9_41KO42kXqcjF4yParQ5JGyg5TckcbhH-Q2IlGpSho,3987
 fugue_duckdb/tester.py,sha256=MzTkv3sdOwOjI59LRrSGGl4w59Njv3OArTU5kSRL-P0,1526
 fugue_ibis/__init__.py,sha256=z7TkK7M2_0p9XO6jQATNDgT0aHXn5k69Ttz2ga-eQG8,190
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
 fugue_ibis/dataframe.py,sha256=k4Q6qBLBIADF5YhbvaDplXO7OkMZSHuf_Wg5o-AusEI,7796
-fugue_ibis/execution_engine.py,sha256=5I-ou5xPdomVu-srdvidvP8f7wDYbGrCV_lGffZa_ac,18679
+fugue_ibis/execution_engine.py,sha256=jRnp1m1wuTicS29A-WA043f8QwdoK8b9rwPXvTkm8r8,18751
 fugue_notebook/__init__.py,sha256=9r_-2uxu1lBeZ8GgpYCKom_OZy2soIOYZajg7JDO-HY,4326
 fugue_notebook/env.py,sha256=TYiTxYPFi-BVJJY49jDsvw9mddhK8WrifeRxBke30I8,4773
 fugue_notebook/nbextension/README.md,sha256=QLnr957YeGfwzy2r4c4qbZPaXyCbyGrKPvcqSBQYSnU,123
@@ -127,7 +129,7 @@ fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
-fugue_ray/_utils/io.py,sha256=Dz0WuQrh_8Ix7jU5viFIA6caJcfxV4ew0ruBZLQbD1s,9930
+fugue_ray/_utils/io.py,sha256=JZdL7pdpk1DUIj77NJSzU_EZOW4cN7oNjwGy2w-LRTw,10142
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
@@ -135,7 +137,7 @@ fugue_spark/execution_engine.py,sha256=YBMtNxCvpy77xICFSg9PHMa6feNoYhWEZe8MmxznX
 fugue_spark/registry.py,sha256=_NmiV2cOooYK0YmqATEnNkPEMT9suUMtuecw2NNbIIk,4530
 fugue_spark/tester.py,sha256=VX003yGNlBukaZTQSN-w7XvgSk4rqxrWQIzno0dWrXg,2481
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
+fugue_spark/_utils/convert.py,sha256=J3HtbuzomTYTN6A11iuvsC1h2C7o3fQBW5U360xGDhE,10234
 fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
 fugue_spark/_utils/misc.py,sha256=9LsbBp6nOEhqXFLr8oWTc3VKzKk-vuVyixlRoquGnEs,858
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
@@ -143,14 +145,13 @@ fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=BpGwa66cAUuuc7ULOsPP3ax8IKQtNIPoSmlUFgqUKQk,79252
+fugue_test/builtin_suite.py,sha256=jP3xiq2vRZNNGzoSRjcUfrUk8NVg31SU0kpJaEvP25E,79400
 fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
-fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
+fugue_test/execution_suite.py,sha256=wUiGdb8wLRd13JXo7Lo19vPOLh7t1C-L2NPLeU0k-uE,48736
 fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
-fugue_version/__init__.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
-fugue-0.9.2.dev1.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.9.2.dev1.dist-info/METADATA,sha256=0ML_xHPma0CaGuGtZn45doFbLu2UtzB-VlPX4koFLYg,18385
-fugue-0.9.2.dev1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fugue-0.9.2.dev1.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
-fugue-0.9.2.dev1.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.9.2.dev1.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=xKd3pzbczuMsdB08eLAOqZDUd_q1IRxwZ_ccAFL4c4A,22
+fugue-0.9.3.dist-info/METADATA,sha256=AXA7npC7pohZQCKiAqe6M5Zoq2K--K4SisIvBi_l1Tc,18570
+fugue-0.9.3.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fugue-0.9.3.dist-info/entry_points.txt,sha256=2Vxp1qew_tswacA8m0RzIliLlFOQMlzezvSXPugM_KA,295
+fugue-0.9.3.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.3.dist-info/RECORD,,

{fugue-0.9.2.dev1.dist-info → fugue-0.9.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fugue-0.9.3.dist-info/entry_points.txt

@@ -0,0 +1,11 @@
+[fugue.plugins]
+dask = fugue_dask.registry[dask]
+duckdb = fugue_duckdb.registry[duckdb]
+ibis = fugue_ibis[ibis]
+polars = fugue_polars.registry[polars]
+ray = fugue_ray.registry[ray]
+spark = fugue_spark.registry[spark]
+
+[pytest11]
+fugue_test = fugue_test
+fugue_test_fixtures = fugue_test.fixtures

fugue_dask/_dask_sql_wrapper.py

@@ -0,0 +1,76 @@
+from typing import Any, Optional
+
+import dask.dataframe as dd
+
+try:
+    from dask.dataframe.dask_expr.io.parquet import ReadParquet
+
+    HAS_DASK_EXPR = True  # newer dask
+except ImportError:  # pragma: no cover
+    HAS_DASK_EXPR = False  # older dask
+
+if not HAS_DASK_EXPR:  # pragma: no cover
+    try:
+        from dask_sql import Context as ContextWrapper  # pylint: disable-all
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+else:
+    from triad.utils.assertion import assert_or_throw
+
+    try:
+        from dask_sql import Context
+        from dask_sql.datacontainer import Statistics
+        from dask_sql.input_utils import InputUtil
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+
+    class ContextWrapper(Context):  # type: ignore
+        def create_table(
+            self,
+            table_name: str,
+            input_table: dd.DataFrame,
+            format: Optional[str] = None,  # noqa
+            persist: bool = False,
+            schema_name: Optional[str] = None,
+            statistics: Optional[Statistics] = None,
+            gpu: bool = False,
+            **kwargs: Any,
+        ) -> None:  # pragma: no cover
+            assert_or_throw(
+                isinstance(input_table, dd.DataFrame),
+                lambda: ValueError(
+                    f"input_table must be a dask dataframe, but got {type(input_table)}"
+                ),
+            )
+            assert_or_throw(
+                dd._dask_expr_enabled(), lambda: ValueError("Dask expr must be enabled")
+            )
+            schema_name = schema_name or self.schema_name
+
+            dc = InputUtil.to_dc(
+                input_table,
+                table_name=table_name,
+                format=format,
+                persist=persist,
+                gpu=gpu,
+                **kwargs,
+            )
+
+            dask_filepath = None
+            operations = input_table.find_operations(ReadParquet)
+            for op in operations:
+                dask_filepath = op._args[0]
+
+            dc.filepath = dask_filepath
+            self.schema[schema_name].filepaths[table_name.lower()] = dask_filepath
+
+            if not statistics:
+                statistics = Statistics(float("nan"))
+            dc.statistics = statistics
+
+            self.schema[schema_name].tables[table_name.lower()] = dc
+            self.schema[schema_name].statistics[table_name.lower()] = statistics

fugue_dask/_utils.py

@@ -5,7 +5,7 @@ import dask.dataframe as dd
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from dask.dataframe.core import DataFrame
+from dask.dataframe import DataFrame
 from dask.delayed import delayed
 from dask.distributed import Client, get_client
 from triad.utils.pandas_like import PD_UTILS, PandasLikeUtils
@@ -149,7 +149,7 @@ def _add_hash_index(
     if len(cols) == 0:
         cols = list(df.columns)
 
-    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:
+    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:  # pragma: no cover
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
         return df.assign(
@@ -171,7 +171,7 @@ def _add_hash_index(
 
 def _add_random_index(
     df: dd.DataFrame, num: int, seed: Any = None
-) -> Tuple[dd.DataFrame, int]:
+) -> Tuple[dd.DataFrame, int]:  # pragma: no cover
     def _add_rand(df: pd.DataFrame) -> pd.DataFrame:
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
@@ -189,7 +189,9 @@ def _add_random_index(
 
 
 def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
-    def _get_info(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _get_info(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return pd.DataFrame(dict(no=[partition_info["number"]], ct=[len(df)]))
 
     pinfo = (
@@ -200,7 +202,9 @@ def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
     counts = pinfo.sort_values("no").ct.cumsum().tolist()
     starts = [0] + counts[0:-1]
 
-    def _add_index(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _add_index(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return df.assign(
             **{
                 _FUGUE_DASK_TEMP_IDX_COLUMN: np.arange(len(df))

fugue_dask/dataframe.py

@@ -379,7 +379,7 @@ def _to_array_chunks(
     assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
     _df = df if columns is None or len(columns) == 0 else df[columns]
 
-    def _to_list(pdf: pd.DataFrame) -> List[Any]:
+    def _to_list(pdf: pd.DataFrame) -> List[Any]:  # pragma: no cover
         return list(
             PD_UTILS.as_array_iterable(
                 pdf,

fugue_dask/execution_engine.py

@@ -9,9 +9,10 @@ from triad.collections import Schema
 from triad.collections.dict import IndexedOrderedDict, ParamDict
 from triad.utils.assertion import assert_or_throw
 from triad.utils.hash import to_uuid
+from triad.utils.io import makedirs
 from triad.utils.pandas_like import PandasUtils
 from triad.utils.threading import RunOnce
-from triad.utils.io import makedirs
+
 from fugue import StructuredRawSQL
 from fugue.collections.partition import (
     PartitionCursor,
@@ -61,14 +62,9 @@ class DaskSQLEngine(SQLEngine):
         return True
 
     def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        try:
-            from dask_sql import Context
-        except ImportError:  # pragma: no cover
-            raise ImportError(
-                "dask-sql is not installed. "
-                "Please install it with `pip install dask-sql`"
-            )
-        ctx = Context()
+        from ._dask_sql_wrapper import ContextWrapper
+
+        ctx = ContextWrapper()
         _dfs: Dict[str, dd.DataFrame] = {k: self._to_safe_df(v) for k, v in dfs.items()}
         sql = statement.construct(dialect=self.dialect, log=self.log)
         res = ctx.sql(
@@ -102,7 +98,8 @@ class DaskMapEngine(MapEngine):
         partition_spec: PartitionSpec,
         on_init: Optional[Callable[[int, DataFrame], Any]] = None,
         map_func_format_hint: Optional[str] = None,
-    ) -> DataFrame:
+    ) -> DataFrame:  # pragma: no cover
+        # It is well tested but not captured by coverage
         presort = partition_spec.get_sorts(
             df.schema, with_partition_keys=partition_spec.algo == "coarse"
         )
@@ -475,7 +472,7 @@ class DaskExecutionEngine(ExecutionEngine):
         # Use presort over partition_spec.presort if possible
         _presort: IndexedOrderedDict = presort or partition_spec.presort
 
-        def _partition_take(partition, n, presort):
+        def _partition_take(partition, n, presort):  # pragma: no cover
             assert_or_throw(
                 partition.shape[1] == len(meta),
                 FugueBug("hitting the dask bug where partition keys are lost"),

fugue_duckdb/dataframe.py

@@ -165,7 +165,7 @@ def _duck_as_local(df: DuckDBPyRelation) -> DuckDBPyRelation:
 
 @as_arrow.candidate(lambda df: isinstance(df, DuckDBPyRelation))
 def _duck_as_arrow(df: DuckDBPyRelation) -> pa.Table:
-    _df = df.arrow()
+    _df = df.fetch_arrow_table()
     _df = replace_types_in_table(_df, LARGE_TYPES_REPLACEMENT, recursive=True)
     return _df
 
@@ -216,7 +216,7 @@ def _drop_duckdb_columns(df: DuckDBPyRelation, columns: List[str]) -> DuckDBPyRe
 def _duck_as_array(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> List[Any]:
-    return pa_table_as_array(df.arrow(), columns=columns)
+    return pa_table_as_array(df.fetch_arrow_table(), columns=columns)
 
 
 @as_array_iterable.candidate(
@@ -225,14 +225,14 @@ def _duck_as_array(
 def _duck_as_array_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> Iterable[Any]:
-    yield from pa_table_as_array_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_array_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
 def _duck_as_dicts(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> List[Dict[str, Any]]:
-    return pa_table_as_dicts(df.arrow(), columns=columns)
+    return pa_table_as_dicts(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dict_iterable.candidate(
@@ -241,7 +241,7 @@ def _duck_as_dicts(
 def _duck_as_dict_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> Iterable[Dict[str, Any]]:
-    yield from pa_table_as_dict_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_dict_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 def _assert_no_missing(df: DuckDBPyRelation, columns: Iterable[Any]) -> None:

fugue_duckdb/execution_engine.py

@@ -108,7 +108,7 @@ class DuckDBEngine(SQLEngine):
         try:
             for k, v in dfs.items():
                 duckdb.from_arrow(v.as_arrow(), connection=conn).create_view(k)
-            return ArrowDataFrame(_duck_as_arrow(conn.execute(statement)))
+            return ArrowDataFrame(_duck_as_arrow(conn.sql(statement)))
         finally:
             conn.close()
 

fugue_ibis/execution_engine.py

@@ -92,7 +92,8 @@ class IbisSQLEngine(SQLEngine):
         _df2 = self.to_df(df2)
         key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
         on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
-        if ibis.__version__ < "6":  # pragma: no cover
+        version = int(ibis.__version__.split(".")[0])
+        if version < 6:  # pragma: no cover
             suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
         else:
             # breaking change in ibis 6.0
@@ -113,7 +114,7 @@ class IbisSQLEngine(SQLEngine):
                     cols.append(
                         ibis.coalesce(tb[k], tb[k + _JOIN_RIGHT_SUFFIX]).name(k)
                     )
-            tb = tb[cols]
+            tb = tb.select(*cols)
         elif how.lower() in ["semi", "left_semi"]:
             tb = _df1.native.semi_join(_df2.native, on_fields, **suffixes)
         elif how.lower() in ["anti", "left_anti"]:
@@ -153,7 +154,7 @@ class IbisSQLEngine(SQLEngine):
         self,
         df: DataFrame,
         how: str = "any",
-        thresh: int = None,
+        thresh: Optional[int] = None,
         subset: Optional[List[str]] = None,
     ) -> DataFrame:
         schema = df.schema
@@ -161,7 +162,7 @@ class IbisSQLEngine(SQLEngine):
             schema = schema.extract(subset)
         _df = self.to_df(df)
         if thresh is None:
-            tb = _df.native.dropna(subset=subset, how=how)
+            tb = _df.native.drop_null(subset, how=how)
             return self.to_df(tb, df.schema)
         assert_or_throw(
             how == "any", ValueError("when thresh is set, how must be 'any'")
@@ -204,7 +205,7 @@ class IbisSQLEngine(SQLEngine):
             ibis.coalesce(tb[f], ibis.literal(vd[f])).name(f) if f in names else tb[f]
             for f in df.columns
         ]
-        return self.to_df(tb[cols], schema=df.schema)
+        return self.to_df(tb.select(cols), schema=df.schema)
 
     def take(
         self,
@@ -241,7 +242,7 @@ class IbisSQLEngine(SQLEngine):
                 f") WHERE __fugue_take_param<={n}"
             )
             tb = self.query_to_table(sql, {tbn: idf})
-            return self.to_df(tb[df.columns], schema=df.schema)
+            return self.to_df(tb.select(*df.columns), schema=df.schema)
 
         sorts: List[str] = []
         for k, v in _presort.items():

fugue_ray/_utils/io.py

@@ -74,7 +74,7 @@ class RayIO(object):
             len(fmts) == 1, NotImplementedError("can't support multiple formats")
         )
         fmt = fmts[0]
-        files = [f.path for f in fp]
+        files = [f.as_dir_path() if f.is_dir else f.path for f in fp]
         return self._loads[fmt](files, columns, **kwargs)
 
     def save_df(
@@ -153,6 +153,10 @@ class RayIO(object):
     def _load_parquet(
         self, p: List[str], columns: Any = None, **kwargs: Any
     ) -> DataFrame:
+        # in 2.52.0 the default changes to ["parquet"]
+        if "file_extensions" not in kwargs:
+            kwargs = kwargs.copy()
+            kwargs["file_extensions"] = None
         sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
         if columns is None:
             return RayDataFrame(sdf)

fugue_spark/_utils/convert.py

@@ -174,20 +174,26 @@ def pd_to_spark_df(
 
 
 def to_pandas(df: ps.DataFrame) -> pd.DataFrame:
-    if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
-        isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
-        for x in df.schema.fields
-    ):
-        return df.toPandas()
-    else:  # pragma: no cover
+    def _to_df() -> pd.DataFrame:
+        if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
+            isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
+            for x in df.schema.fields
+        ):
+            return df.toPandas()
+        else:  # pragma: no cover
+
+            def serialize(dfs):
+                for df in dfs:
+                    data = pickle.dumps(df)
+                    yield pd.DataFrame([[data]], columns=["data"])
 
-        def serialize(dfs):
-            for df in dfs:
-                data = pickle.dumps(df)
-                yield pd.DataFrame([[data]], columns=["data"])
+            sdf = df.mapInPandas(serialize, schema="data binary")
+            return pd.concat(pickle.loads(x.data) for x in sdf.collect())
 
-        sdf = df.mapInPandas(serialize, schema="data binary")
-        return pd.concat(pickle.loads(x.data) for x in sdf.collect())
+    pdf = _to_df()
+    if hasattr(pdf, "attrs") and "metrics" in pdf.attrs:  # pragma: no cover
+        del pdf.attrs["metrics"]
+    return pdf
 
 
 def to_arrow(df: ps.DataFrame) -> pa.Table:

fugue_test/builtin_suite.py

@@ -486,7 +486,9 @@ class BuiltInTests(object):
                 dag.df([], "a:int,b:int").assert_eq(b)
             dag.run(self.engine)
 
-        def test_transform_row_wise(self):
+        def _test_transform_row_wise(self):  # pragma: no cover
+            # TODO: currently disabled because we don't support Dict[str, Any]
+            # as dataframe input
             def t1(row: Dict[str, Any]) -> Dict[str, Any]:
                 row["b"] = 1
                 return row
@@ -695,11 +697,11 @@ class BuiltInTests(object):
                 incr()
                 yield pa.Table.from_pandas(df)
 
-            def t11(row: Dict[str, Any]) -> Dict[str, Any]:
+            def t11(row: list[dict[str, Any]]) -> dict[str, Any]:
                 incr()
-                return row
+                return row[0]
 
-            def t12(row: Dict[str, Any]) -> None:
+            def t12(row: list[dict[str, Any]]) -> None:
                 incr()
 
             with FugueWorkflow() as dag:

fugue_test/execution_suite.py

@@ -9,6 +9,7 @@ except ImportError:  # pragma: no cover
 import copy
 import os
 import pickle
+import sys
 from datetime import datetime
 
 import pandas as pd
@@ -1194,6 +1195,7 @@ class ExecutionEngineTests(object):
             )
             self.df_eq(c, [[1.1, 6.1], [7.1, 2.1]], "a:double,c:double", throw=True)
 
+        @pytest.mark.skipif(sys.platform == "win32", reason="skip on Windows")
         def test_load_csv_folder(self):
             native = NativeExecutionEngine()
             a = ArrayDataFrame([[6.1, 1.1]], "c:double,a:double")

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.2"
+__version__ = "0.9.3"

fugue-0.9.2.dev1.dist-info/entry_points.txt

@@ -1,12 +0,0 @@
-[fugue.plugins]
-dask = fugue_dask.registry [dask]
-duckdb = fugue_duckdb.registry [duckdb]
-ibis = fugue_ibis [ibis]
-polars = fugue_polars.registry [polars]
-ray = fugue_ray.registry [ray]
-spark = fugue_spark.registry [spark]
-
-[pytest11]
-fugue_test = fugue_test
-fugue_test_fixtures = fugue_test.fixtures
-