fugue 0.9.2.dev2__py3-none-any.whl → 0.9.3__py3-none-any.whl

fugue/_utils/io.py

@@ -20,6 +20,10 @@ class FileParser(object):
         self._has_glob = "*" in path or "?" in path
         self._raw_path = path
         self._fs, self._fs_path = url_to_fs(path)
+        if not self._has_glob and self._fs.isdir(self._fs_path):
+            self._is_dir = True
+        else:
+            self._is_dir = False
         if not self.is_local:
             self._path = self._fs.unstrip_protocol(self._fs_path)
         else:
@@ -43,11 +47,15 @@ class FileParser(object):
         return self
 
     @property
-    def has_glob(self):
+    def is_dir(self) -> bool:
+        return self._is_dir
+
+    @property
+    def has_glob(self) -> bool:
         return self._has_glob
 
     @property
-    def is_local(self):
+    def is_local(self) -> bool:
         return isinstance(self._fs, LocalFileSystem)
 
     def join(self, path: str, format_hint: Optional[str] = None) -> "FileParser":
@@ -65,6 +73,10 @@ class FileParser(object):
     def path(self) -> str:
         return self._path
 
+    def as_dir_path(self) -> str:
+        assert_or_throw(self.is_dir, f"{self.raw_path} is not a directory")
+        return self.path + self._fs.sep
+
     @property
     def raw_path(self) -> str:
         return self._raw_path

fugue/dataframe/function_wrapper.py

@@ -91,7 +91,9 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    if v.is_per_row:
+                    if v.is_per_row:  # pragma: no cover
+                        # TODO: this branch is used only if row annotations
+                        # are allowed as input
                         assert_or_throw(
                             row_param_info is None,
                             lambda: ValueError("only one row parameter is allowed"),
@@ -110,7 +112,9 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             raise ValueError(f"{p} are not acceptable parameters")
         if row_param_info is None:
             return self._run_func(rargs, output, output_schema, ctx, raw=False)
-        else:  # input contains row parameter
+        else:  # pragma: no cover
+            # input contains row parameter
+            # TODO: this branch is used only if row annotations are allowed as input
 
             def _dfs() -> Iterable[Any]:
                 k, v, df = row_param_info
@@ -237,7 +241,8 @@ class DataFrameParam(_DataFrameParamBase):
 
 
 @fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
-class RowParam(_DataFrameParamBase):
+class RowParam(_DataFrameParamBase):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
     @property
     def is_per_row(self) -> bool:
         return True
@@ -247,7 +252,8 @@ class RowParam(_DataFrameParamBase):
 
 
 @fugue_annotated_param(Dict[str, Any])
-class DictParam(RowParam):
+class DictParam(RowParam):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
     def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
         yield from df.as_dict_iterable()
 

fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspqr][fF]?x*z?$", "^[lspqr]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspqr][fF]?x*z?$", "^[lspnqr]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore

fugue/rpc/flask.py

@@ -1,6 +1,5 @@
-import base64
 import logging
-import cloudpickle
+import json
 from threading import Thread
 from typing import Any, Optional, Tuple, Dict, List
 
@@ -60,6 +59,7 @@ class FlaskRPCServer(RPCServer):
             -1.0 if timeout is None else to_timedelta(timeout).total_seconds()
         )
         self._server: Optional[FlaskRPCServer._Thread] = None
+        self._log = logging.getLogger()
 
     def make_client(self, handler: Any) -> RPCClient:
         """Add ``handler`` and correspondent :class:`~.FlaskRPCClient`
@@ -77,6 +77,14 @@ class FlaskRPCServer(RPCServer):
 
     def start_server(self) -> None:
         """Start Flask RPC server"""
+        msg = (
+            "Starting RPC server on %s:%s. "
+            "This server has no authentication and relies on network isolation. "
+            "Ensure proper VPC/firewall configuration in production. "
+            "See https://fugue-tutorials.readthedocs.io/tutorials/resources/"
+            "security.html"
+        )
+        self._log.warning(msg, self._host, self._port)
         app = Flask("FlaskRPCServer")
         app.route("/invoke", methods=["POST"])(self._invoke)
         self._server = FlaskRPCServer._Thread(app, self._host, self._port)
@@ -122,10 +130,10 @@ class FlaskRPCClient(RPCClient):
 
 
 def _encode(*args: Any, **kwargs: Any) -> str:
-    data = base64.b64encode(cloudpickle.dumps(dict(args=args, kwargs=kwargs)))
-    return data.decode("ascii")
+    data = json.dumps(dict(args=args, kwargs=kwargs))
+    return data
 
 
 def _decode(data: str) -> Tuple[List[Any], Dict[str, Any]]:
-    data = cloudpickle.loads(base64.b64decode(data.encode("ascii")))
+    data = json.loads(data)
     return data["args"], data["kwargs"]  # type: ignore

{fugue-0.9.2.dev2.dist-info → fugue-0.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.2.dev2
+Version: 0.9.3
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -21,42 +21,39 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: triad>=0.9.7
+Requires-Dist: triad>=1.0.0
 Requires-Dist: adagio>=0.2.6
-Provides-Extra: all
-Requires-Dist: qpd>=0.4.4; extra == "all"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
-Requires-Dist: sqlglot; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: pyspark>=3.1.1; extra == "all"
-Requires-Dist: dask[dataframe,distributed]>=2023.5.0; extra == "all"
-Requires-Dist: dask-sql; extra == "all"
-Requires-Dist: ray[data]>=2.5.0; extra == "all"
-Requires-Dist: notebook; extra == "all"
-Requires-Dist: jupyterlab; extra == "all"
-Requires-Dist: ipython>=7.10.0; extra == "all"
-Requires-Dist: duckdb>=0.5.0; extra == "all"
-Requires-Dist: pyarrow>=6.0.1; extra == "all"
-Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
-Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
-Requires-Dist: polars; extra == "all"
-Provides-Extra: cpp_sql_parser
+Provides-Extra: sql
+Requires-Dist: qpd>=0.4.4; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: sqlglot<28; extra == "sql"
+Requires-Dist: jinja2; extra == "sql"
+Provides-Extra: cpp-sql-parser
 Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Provides-Extra: dask
 Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "dask"
 Requires-Dist: pyarrow>=7.0.0; extra == "dask"
 Requires-Dist: pandas>=2.0.2; extra == "dask"
+Provides-Extra: ray
+Requires-Dist: ray[data]>=2.30.0; extra == "ray"
+Requires-Dist: duckdb>=0.5.0; extra == "ray"
+Requires-Dist: pyarrow>=7.0.0; extra == "ray"
+Requires-Dist: pandas<2.2; extra == "ray"
 Provides-Extra: duckdb
 Requires-Dist: qpd>=0.4.4; extra == "duckdb"
 Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
-Requires-Dist: sqlglot; extra == "duckdb"
+Requires-Dist: sqlglot<28; extra == "duckdb"
 Requires-Dist: jinja2; extra == "duckdb"
 Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
 Requires-Dist: numpy; extra == "duckdb"
+Provides-Extra: polars
+Requires-Dist: polars; extra == "polars"
 Provides-Extra: ibis
 Requires-Dist: qpd>=0.4.4; extra == "ibis"
 Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
-Requires-Dist: sqlglot; extra == "ibis"
+Requires-Dist: sqlglot<28; extra == "ibis"
 Requires-Dist: jinja2; extra == "ibis"
 Requires-Dist: ibis-framework[pandas]; extra == "ibis"
 Requires-Dist: pandas<2.2; extra == "ibis"
@@ -64,20 +61,36 @@ Provides-Extra: notebook
 Requires-Dist: notebook; extra == "notebook"
 Requires-Dist: jupyterlab; extra == "notebook"
 Requires-Dist: ipython>=7.10.0; extra == "notebook"
-Provides-Extra: polars
-Requires-Dist: polars; extra == "polars"
-Provides-Extra: ray
-Requires-Dist: ray[data]>=2.5.0; extra == "ray"
-Requires-Dist: duckdb>=0.5.0; extra == "ray"
-Requires-Dist: pyarrow>=7.0.0; extra == "ray"
-Requires-Dist: pandas<2.2; extra == "ray"
-Provides-Extra: spark
-Requires-Dist: pyspark>=3.1.1; extra == "spark"
-Provides-Extra: sql
-Requires-Dist: qpd>=0.4.4; extra == "sql"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
-Requires-Dist: sqlglot; extra == "sql"
-Requires-Dist: jinja2; extra == "sql"
+Provides-Extra: all
+Requires-Dist: qpd>=0.4.4; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: sqlglot<28; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: pyspark>=3.1.1; extra == "all"
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "all"
+Requires-Dist: dask-sql; extra == "all"
+Requires-Dist: ray[data]>=2.30.0; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Requires-Dist: ipython>=7.10.0; extra == "all"
+Requires-Dist: duckdb>=0.5.0; extra == "all"
+Requires-Dist: pyarrow>=6.0.1; extra == "all"
+Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
+Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
+Requires-Dist: polars; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Fugue
 

{fugue-0.9.2.dev2.dist-info → fugue-0.9.3.dist-info}/RECORD

@@ -10,7 +10,7 @@ fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/_utils/display.py,sha256=JV8oDA7efHm1wceZulCBOY5dMvjbWHvIm6ASisKfoWY,3164
 fugue/_utils/exception.py,sha256=SFIjwjV4CIEovp3P9k7ePNOFB12A5D8hDdhtfFUeM5Y,2247
 fugue/_utils/interfaceless.py,sha256=wI0H6L4W_1uQjh9tpjgT9HzN-fbrrtXXHC1x6Q_rrPg,2203
-fugue/_utils/io.py,sha256=adrtj6Dq0ti426DNlkliApbTkp8b3bfBysAiE5MVQVc,9265
+fugue/_utils/io.py,sha256=5twd99LBzHtIMT67il1qwnEUa5n13WZmVKNd1shO4No,9649
 fugue/_utils/misc.py,sha256=_huy0eylmRTEFoReGR2M4rbAI8m79hFcfY5bDceVEXU,887
 fugue/_utils/registry.py,sha256=lrbzTdUEVnW6paBGDj-Yb-aTIbP5mjCqrXuRU9_N6os,316
 fugue/bag/__init__.py,sha256=0Q0_rnrEThrTx2U-1xGNyAg95idp_xcnywymIcW4Xck,46
@@ -31,7 +31,7 @@ fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvr
 fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=1CjI4UXHffomylK0_u0CGL1dPv_sSXTN22S5grD10_w,17889
+fugue/dataframe/function_wrapper.py,sha256=7Sb6XrWTD_swtHJbHDWZRxHvFNWkERynnCDzLM0wSbo,18340
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
 fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
@@ -61,11 +61,11 @@ fugue/extensions/processor/convert.py,sha256=zG0lMtHGwY5TsqK4eplbMdlTg7J_PD3HbI0
 fugue/extensions/processor/processor.py,sha256=czhQlQgMpAXXoLVAX9Q0TFUMYEEhsgufTammxcKSmOY,1665
 fugue/extensions/transformer/__init__.py,sha256=VD6d-8xW1Yl8fUPj43cBWNR9pCOlYD9xWyGIHAlHwvI,456
 fugue/extensions/transformer/constants.py,sha256=76DfpoTOGQ8gp5XtCs_xznfbr_H015-prXpHWSqMNDU,59
-fugue/extensions/transformer/convert.py,sha256=SU_KvzZp_nV8oCxZGx7qDsdCE0CJ--8UAp5m8z4d4HY,23386
+fugue/extensions/transformer/convert.py,sha256=zDDIpZawMnHFarjZNZAyiw1jfyXGuPjnvgQk9jpYLak,23384
 fugue/extensions/transformer/transformer.py,sha256=zhOUgyv5-DPxYd1CP_98WeEw-zUgwknRnPW_6di-q3g,9098
 fugue/rpc/__init__.py,sha256=3GzUl4QZQuCChjD7eaTJW8tnTwfke6ZY9r9g5nCeBZ8,167
 fugue/rpc/base.py,sha256=3Fq5SvwLZqw9NXru3r32WuJKBGFr9bl7nFgy6e9boGo,8470
-fugue/rpc/flask.py,sha256=1oD0dE4Q-0PKeZ7RG3c0pfwyx21dUavfXWORu_gV7mg,4485
+fugue/rpc/flask.py,sha256=VzJEND8Pqatf6pYYT9LDXeO1JDMmYAOY0lm8XUncKbA,4807
 fugue/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/sql/_utils.py,sha256=khpjGeFCVlaqf2JIYvS4TVTJO3fe5-8bEsvy6AIP_5Q,2083
 fugue/sql/_visitors.py,sha256=2pc0J-AHJAiIexsKgNjcgrCGOyhC3_7rzonSgtjy--k,33844
@@ -82,6 +82,7 @@ fugue/workflow/api.py,sha256=uQoxPSCZ91-ST4vwuPWG7qioRGW4eo-Sgi3DdwtSL4k,12495
 fugue/workflow/input.py,sha256=V_zLDNzndmQuYJAPXtdK4n-vOp7LrimGIf_wQtwf2mc,321
 fugue/workflow/module.py,sha256=ajyqgMwX6hFMZY9xp4Bp1Q-Zdta0p5f_W_n_SNrc4LE,5547
 fugue/workflow/workflow.py,sha256=-SFCXkyxgXbS6DpQGSBox4d3Ws3psIlB6PnraJLSu9Y,88219
+fugue-0.9.3.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
 fugue_contrib/__init__.py,sha256=QJioX-r2AiU7Pvt24M-k2c4vNq29qpK-3WNUde7ucck,222
 fugue_contrib/contrib.py,sha256=3B--6oIVBMZ-GwjIOXwZqYqkloH7Cxfq1I8vkwl2yPk,267
 fugue_contrib/seaborn/__init__.py,sha256=NuVv8EI4Om4gHcHwYO8ddextLQqw24vDj8qJio3E1MU,1405
@@ -91,17 +92,17 @@ fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
 fugue_dask/_dask_sql_wrapper.py,sha256=lj38gJIOdoMV9W44gpwzLjUEtPVsQNKjRWuEkfI7-PM,2618
 fugue_dask/_io.py,sha256=pl4F7mbVgP7Rwh1FFG7xfOz2TBZRUj1l3lLvDY4jOf4,6020
-fugue_dask/_utils.py,sha256=dGUkhOoXQqgGQH_BY6aeYFo9UIWUAyo8YjwtdB7QD4s,8951
-fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=Em9pN6cw5w5DGLcjV6oKQKQeLLblc9DZ0DkvxKVFxQQ,21167
+fugue_dask/_utils.py,sha256=0R0pCh4B47kQsAS_o0QGaosIqVcZnSakm6pfMB7fSXs,9059
+fugue_dask/dataframe.py,sha256=4Dvckpc4mlld2WsEFTTemxoA1zYK8Cn6jMKxUxYQCEE,13491
+fugue_dask/execution_engine.py,sha256=mFN_IurhdBEu8C5OreqpGSRdTbTBqSpzJO2dMQzEF-o,21264
 fugue_dask/registry.py,sha256=jepWKH55VWNIWV3pOF5vpCl2OpO0rI1IULx5GM2Gk6w,2274
 fugue_dask/tester.py,sha256=E7BZjgFpJgrHsLMKzvSO5im5OwocYcratjzulJSQZl0,718
 fugue_duckdb/__init__.py,sha256=ZzhmAWbROR1YL9Kmlt7OlwkgPZzFhsSdwLV2pFmAqGI,268
 fugue_duckdb/_io.py,sha256=vnd8m8C6XeMCBJBbAdA5h695NMfsduQrvONyS0HcEFA,8475
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
-fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
-fugue_duckdb/execution_engine.py,sha256=IZDmSAtOMJGvulTStxjTmsqJyI5QRNyxBgSMlFMSrBI,20389
+fugue_duckdb/dataframe.py,sha256=LAPoPOad9hgGhjyhlMGMfrnhkyBKe06Xzn6eP1hkl-w,8504
+fugue_duckdb/execution_engine.py,sha256=3f5hbWcX1y9mAtfFixrri-snYxVIQAf4HOgo9fHbDwQ,20385
 fugue_duckdb/registry.py,sha256=9_41KO42kXqcjF4yParQ5JGyg5TckcbhH-Q2IlGpSho,3987
 fugue_duckdb/tester.py,sha256=MzTkv3sdOwOjI59LRrSGGl4w59Njv3OArTU5kSRL-P0,1526
 fugue_ibis/__init__.py,sha256=z7TkK7M2_0p9XO6jQATNDgT0aHXn5k69Ttz2ga-eQG8,190
@@ -128,7 +129,7 @@ fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
-fugue_ray/_utils/io.py,sha256=Dz0WuQrh_8Ix7jU5viFIA6caJcfxV4ew0ruBZLQbD1s,9930
+fugue_ray/_utils/io.py,sha256=JZdL7pdpk1DUIj77NJSzU_EZOW4cN7oNjwGy2w-LRTw,10142
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
@@ -136,7 +137,7 @@ fugue_spark/execution_engine.py,sha256=YBMtNxCvpy77xICFSg9PHMa6feNoYhWEZe8MmxznX
 fugue_spark/registry.py,sha256=_NmiV2cOooYK0YmqATEnNkPEMT9suUMtuecw2NNbIIk,4530
 fugue_spark/tester.py,sha256=VX003yGNlBukaZTQSN-w7XvgSk4rqxrWQIzno0dWrXg,2481
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
+fugue_spark/_utils/convert.py,sha256=J3HtbuzomTYTN6A11iuvsC1h2C7o3fQBW5U360xGDhE,10234
 fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
 fugue_spark/_utils/misc.py,sha256=9LsbBp6nOEhqXFLr8oWTc3VKzKk-vuVyixlRoquGnEs,858
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
@@ -144,14 +145,13 @@ fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=BpGwa66cAUuuc7ULOsPP3ax8IKQtNIPoSmlUFgqUKQk,79252
+fugue_test/builtin_suite.py,sha256=jP3xiq2vRZNNGzoSRjcUfrUk8NVg31SU0kpJaEvP25E,79400
 fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
-fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
+fugue_test/execution_suite.py,sha256=wUiGdb8wLRd13JXo7Lo19vPOLh7t1C-L2NPLeU0k-uE,48736
 fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
-fugue_version/__init__.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
-fugue-0.9.2.dev2.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.9.2.dev2.dist-info/METADATA,sha256=eR5mL6Tf1RGa_-Do5Dmzy4ZkbcbKf-FzW4qA0cAW1Ec,18283
-fugue-0.9.2.dev2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-fugue-0.9.2.dev2.dist-info/entry_points.txt,sha256=2Vxp1qew_tswacA8m0RzIliLlFOQMlzezvSXPugM_KA,295
-fugue-0.9.2.dev2.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.9.2.dev2.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=xKd3pzbczuMsdB08eLAOqZDUd_q1IRxwZ_ccAFL4c4A,22
+fugue-0.9.3.dist-info/METADATA,sha256=AXA7npC7pohZQCKiAqe6M5Zoq2K--K4SisIvBi_l1Tc,18570
+fugue-0.9.3.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fugue-0.9.3.dist-info/entry_points.txt,sha256=2Vxp1qew_tswacA8m0RzIliLlFOQMlzezvSXPugM_KA,295
+fugue-0.9.3.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.3.dist-info/RECORD,,

{fugue-0.9.2.dev2.dist-info → fugue-0.9.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fugue_dask/_utils.py

@@ -149,7 +149,7 @@ def _add_hash_index(
     if len(cols) == 0:
         cols = list(df.columns)
 
-    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:
+    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:  # pragma: no cover
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
         return df.assign(
@@ -171,7 +171,7 @@ def _add_hash_index(
 
 def _add_random_index(
     df: dd.DataFrame, num: int, seed: Any = None
-) -> Tuple[dd.DataFrame, int]:
+) -> Tuple[dd.DataFrame, int]:  # pragma: no cover
     def _add_rand(df: pd.DataFrame) -> pd.DataFrame:
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
@@ -189,7 +189,9 @@ def _add_random_index(
 
 
 def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
-    def _get_info(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _get_info(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return pd.DataFrame(dict(no=[partition_info["number"]], ct=[len(df)]))
 
     pinfo = (
@@ -200,7 +202,9 @@ def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
     counts = pinfo.sort_values("no").ct.cumsum().tolist()
     starts = [0] + counts[0:-1]
 
-    def _add_index(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _add_index(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return df.assign(
             **{
                 _FUGUE_DASK_TEMP_IDX_COLUMN: np.arange(len(df))

fugue_dask/dataframe.py

@@ -379,7 +379,7 @@ def _to_array_chunks(
     assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
     _df = df if columns is None or len(columns) == 0 else df[columns]
 
-    def _to_list(pdf: pd.DataFrame) -> List[Any]:
+    def _to_list(pdf: pd.DataFrame) -> List[Any]:  # pragma: no cover
         return list(
             PD_UTILS.as_array_iterable(
                 pdf,

fugue_dask/execution_engine.py

@@ -98,7 +98,8 @@ class DaskMapEngine(MapEngine):
         partition_spec: PartitionSpec,
         on_init: Optional[Callable[[int, DataFrame], Any]] = None,
         map_func_format_hint: Optional[str] = None,
-    ) -> DataFrame:
+    ) -> DataFrame:  # pragma: no cover
+        # It is well tested but not captured by coverage
         presort = partition_spec.get_sorts(
             df.schema, with_partition_keys=partition_spec.algo == "coarse"
         )
@@ -471,7 +472,7 @@ class DaskExecutionEngine(ExecutionEngine):
         # Use presort over partition_spec.presort if possible
         _presort: IndexedOrderedDict = presort or partition_spec.presort
 
-        def _partition_take(partition, n, presort):
+        def _partition_take(partition, n, presort):  # pragma: no cover
             assert_or_throw(
                 partition.shape[1] == len(meta),
                 FugueBug("hitting the dask bug where partition keys are lost"),

fugue_duckdb/dataframe.py

@@ -165,7 +165,7 @@ def _duck_as_local(df: DuckDBPyRelation) -> DuckDBPyRelation:
 
 @as_arrow.candidate(lambda df: isinstance(df, DuckDBPyRelation))
 def _duck_as_arrow(df: DuckDBPyRelation) -> pa.Table:
-    _df = df.arrow()
+    _df = df.fetch_arrow_table()
     _df = replace_types_in_table(_df, LARGE_TYPES_REPLACEMENT, recursive=True)
     return _df
 
@@ -216,7 +216,7 @@ def _drop_duckdb_columns(df: DuckDBPyRelation, columns: List[str]) -> DuckDBPyRe
 def _duck_as_array(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> List[Any]:
-    return pa_table_as_array(df.arrow(), columns=columns)
+    return pa_table_as_array(df.fetch_arrow_table(), columns=columns)
 
 
 @as_array_iterable.candidate(
@@ -225,14 +225,14 @@ def _duck_as_array(
 def _duck_as_array_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> Iterable[Any]:
-    yield from pa_table_as_array_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_array_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
 def _duck_as_dicts(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> List[Dict[str, Any]]:
-    return pa_table_as_dicts(df.arrow(), columns=columns)
+    return pa_table_as_dicts(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dict_iterable.candidate(
@@ -241,7 +241,7 @@ def _duck_as_dicts(
 def _duck_as_dict_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> Iterable[Dict[str, Any]]:
-    yield from pa_table_as_dict_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_dict_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 def _assert_no_missing(df: DuckDBPyRelation, columns: Iterable[Any]) -> None:

fugue_duckdb/execution_engine.py

@@ -108,7 +108,7 @@ class DuckDBEngine(SQLEngine):
         try:
             for k, v in dfs.items():
                 duckdb.from_arrow(v.as_arrow(), connection=conn).create_view(k)
-            return ArrowDataFrame(_duck_as_arrow(conn.execute(statement)))
+            return ArrowDataFrame(_duck_as_arrow(conn.sql(statement)))
         finally:
             conn.close()
 

fugue_ray/_utils/io.py

@@ -74,7 +74,7 @@ class RayIO(object):
             len(fmts) == 1, NotImplementedError("can't support multiple formats")
         )
         fmt = fmts[0]
-        files = [f.path for f in fp]
+        files = [f.as_dir_path() if f.is_dir else f.path for f in fp]
         return self._loads[fmt](files, columns, **kwargs)
 
     def save_df(
@@ -153,6 +153,10 @@ class RayIO(object):
     def _load_parquet(
         self, p: List[str], columns: Any = None, **kwargs: Any
     ) -> DataFrame:
+        # in 2.52.0 the default changes to ["parquet"]
+        if "file_extensions" not in kwargs:
+            kwargs = kwargs.copy()
+            kwargs["file_extensions"] = None
         sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
         if columns is None:
             return RayDataFrame(sdf)

fugue_spark/_utils/convert.py

@@ -174,20 +174,26 @@ def pd_to_spark_df(
 
 
 def to_pandas(df: ps.DataFrame) -> pd.DataFrame:
-    if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
-        isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
-        for x in df.schema.fields
-    ):
-        return df.toPandas()
-    else:  # pragma: no cover
+    def _to_df() -> pd.DataFrame:
+        if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
+            isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
+            for x in df.schema.fields
+        ):
+            return df.toPandas()
+        else:  # pragma: no cover
+
+            def serialize(dfs):
+                for df in dfs:
+                    data = pickle.dumps(df)
+                    yield pd.DataFrame([[data]], columns=["data"])
 
-        def serialize(dfs):
-            for df in dfs:
-                data = pickle.dumps(df)
-                yield pd.DataFrame([[data]], columns=["data"])
+            sdf = df.mapInPandas(serialize, schema="data binary")
+            return pd.concat(pickle.loads(x.data) for x in sdf.collect())
 
-        sdf = df.mapInPandas(serialize, schema="data binary")
-        return pd.concat(pickle.loads(x.data) for x in sdf.collect())
+    pdf = _to_df()
+    if hasattr(pdf, "attrs") and "metrics" in pdf.attrs:  # pragma: no cover
+        del pdf.attrs["metrics"]
+    return pdf
 
 
 def to_arrow(df: ps.DataFrame) -> pa.Table:

fugue_test/builtin_suite.py

@@ -486,7 +486,9 @@ class BuiltInTests(object):
                 dag.df([], "a:int,b:int").assert_eq(b)
             dag.run(self.engine)
 
-        def test_transform_row_wise(self):
+        def _test_transform_row_wise(self):  # pragma: no cover
+            # TODO: currently disabled because we don't support Dict[str, Any]
+            # as dataframe input
             def t1(row: Dict[str, Any]) -> Dict[str, Any]:
                 row["b"] = 1
                 return row
@@ -695,11 +697,11 @@ class BuiltInTests(object):
                 incr()
                 yield pa.Table.from_pandas(df)
 
-            def t11(row: Dict[str, Any]) -> Dict[str, Any]:
+            def t11(row: list[dict[str, Any]]) -> dict[str, Any]:
                 incr()
-                return row
+                return row[0]
 
-            def t12(row: Dict[str, Any]) -> None:
+            def t12(row: list[dict[str, Any]]) -> None:
                 incr()
 
             with FugueWorkflow() as dag:

fugue_test/execution_suite.py

@@ -9,6 +9,7 @@ except ImportError:  # pragma: no cover
 import copy
 import os
 import pickle
+import sys
 from datetime import datetime
 
 import pandas as pd
@@ -1194,6 +1195,7 @@ class ExecutionEngineTests(object):
             )
             self.df_eq(c, [[1.1, 6.1], [7.1, 2.1]], "a:double,c:double", throw=True)
 
+        @pytest.mark.skipif(sys.platform == "win32", reason="skip on Windows")
         def test_load_csv_folder(self):
             native = NativeExecutionEngine()
             a = ArrayDataFrame([[6.1, 1.1]], "c:double,a:double")

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.2"
+__version__ = "0.9.3"