fugue 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

fugue/_utils/io.py

@@ -20,6 +20,10 @@ class FileParser(object):
         self._has_glob = "*" in path or "?" in path
         self._raw_path = path
         self._fs, self._fs_path = url_to_fs(path)
+        if not self._has_glob and self._fs.isdir(self._fs_path):
+            self._is_dir = True
+        else:
+            self._is_dir = False
         if not self.is_local:
             self._path = self._fs.unstrip_protocol(self._fs_path)
         else:
@@ -43,11 +47,15 @@ class FileParser(object):
         return self
 
     @property
-    def has_glob(self):
+    def is_dir(self) -> bool:
+        return self._is_dir
+
+    @property
+    def has_glob(self) -> bool:
         return self._has_glob
 
     @property
-    def is_local(self):
+    def is_local(self) -> bool:
         return isinstance(self._fs, LocalFileSystem)
 
     def join(self, path: str, format_hint: Optional[str] = None) -> "FileParser":
@@ -65,6 +73,10 @@ class FileParser(object):
     def path(self) -> str:
         return self._path
 
+    def as_dir_path(self) -> str:
+        assert_or_throw(self.is_dir, f"{self.raw_path} is not a directory")
+        return self.path + self._fs.sep
+
     @property
     def raw_path(self) -> str:
         return self._raw_path

fugue/dataframe/function_wrapper.py

@@ -80,6 +80,7 @@ class DataFrameFunctionWrapper(FunctionWrapper):
         p.update(kwargs)
         has_kw = False
         rargs: Dict[str, Any] = {}
+        row_param_info: Any = None
         for k, v in self._params.items():
             if isinstance(v, (PositionalParam, KeywordParam)):
                 if isinstance(v, KeywordParam):
@@ -90,7 +91,16 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    rargs[k] = v.to_input_data(p[k], ctx=ctx)
+                    if v.is_per_row:  # pragma: no cover
+                        # TODO: this branch is used only if row annotations
+                        # are allowed as input
+                        assert_or_throw(
+                            row_param_info is None,
+                            lambda: ValueError("only one row parameter is allowed"),
+                        )
+                        row_param_info = (k, v, p[k])
+                    else:
+                        rargs[k] = v.to_input_data(p[k], ctx=ctx)
                 else:
                     rargs[k] = p[k]  # TODO: should we do auto type conversion?
                 del p[k]
@@ -100,12 +110,40 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             rargs.update(p)
         elif not ignore_unknown and len(p) > 0:
             raise ValueError(f"{p} are not acceptable parameters")
+        if row_param_info is None:
+            return self._run_func(rargs, output, output_schema, ctx, raw=False)
+        else:  # pragma: no cover
+            # input contains row parameter
+            # TODO: this branch is used only if row annotations are allowed as input
+
+            def _dfs() -> Iterable[Any]:
+                k, v, df = row_param_info
+                for row in v.to_input_rows(df, ctx):
+                    rargs[k] = None
+                    _rargs = rargs.copy()
+                    _rargs[k] = row
+                    yield self._run_func(_rargs, output, output_schema, ctx, raw=True)
+
+            if not output:
+                sum(1 for _ in _dfs())
+                return
+            else:
+                return self._rt.iterable_to_output_df(_dfs(), output_schema, ctx)
+
+    def _run_func(
+        self,
+        rargs: Dict[str, Any],
+        output: bool,
+        output_schema: Any,
+        ctx: Any,
+        raw: bool,
+    ) -> Any:
         rt = self._func(**rargs)
         if not output:
             if isinstance(self._rt, _DataFrameParamBase):
                 self._rt.count(rt)
             return
-        if isinstance(self._rt, _DataFrameParamBase):
+        if not raw and isinstance(self._rt, _DataFrameParamBase):
             return self._rt.to_output_df(rt, output_schema, ctx=ctx)
         return rt
 
@@ -120,6 +158,7 @@ fugue_annotated_param = DataFrameFunctionWrapper.annotated_param
         annotation == Callable
         or annotation == callable  # pylint: disable=comparison-with-callable
         or str(annotation).startswith("typing.Callable")
+        or str(annotation).startswith("collections.abc.Callable")
     ),
 )
 class _CallableParam(AnnotatedParam):
@@ -134,6 +173,9 @@ class _CallableParam(AnnotatedParam):
         or annotation == Optional[callable]
         or str(annotation).startswith("typing.Union[typing.Callable")  # 3.8-
         or str(annotation).startswith("typing.Optional[typing.Callable")  # 3.9+
+        or str(annotation).startswith(
+            "typing.Optional[collections.abc.Callable]"
+        )  # 3.9+
     ),
 )
 class _OptionalCallableParam(AnnotatedParam):
@@ -145,14 +187,30 @@ class _DataFrameParamBase(AnnotatedParam):
         super().__init__(param)
         assert_or_throw(self.required, lambda: TypeError(f"{self} must be required"))
 
+    @property
+    def is_per_row(self) -> bool:
+        return False
+
     def to_input_data(self, df: DataFrame, ctx: Any) -> Any:  # pragma: no cover
         raise NotImplementedError
 
+    def to_input_rows(
+        self,
+        df: DataFrame,
+        ctx: Any,
+    ) -> Iterable[Any]:
+        raise NotImplementedError  # pragma: no cover
+
     def to_output_df(
         self, df: Any, schema: Any, ctx: Any
     ) -> DataFrame:  # pragma: no cover
         raise NotImplementedError
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        raise NotImplementedError
+
     def count(self, df: Any) -> int:  # pragma: no cover
         raise NotImplementedError
 
@@ -182,6 +240,36 @@ class DataFrameParam(_DataFrameParamBase):
             return sum(1 for _ in df.as_array_iterable())
 
 
+@fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
+class RowParam(_DataFrameParamBase):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
+    @property
+    def is_per_row(self) -> bool:
+        return True
+
+    def count(self, df: Any) -> int:
+        return 1
+
+
+@fugue_annotated_param(Dict[str, Any])
+class DictParam(RowParam):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
+    def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
+        yield from df.as_dict_iterable()
+
+    def to_output_df(self, output: Dict[str, Any], schema: Any, ctx: Any) -> DataFrame:
+        return ArrayDataFrame([list(output.values())], schema)
+
+    def iterable_to_output_df(
+        self, dfs: Iterable[Dict[str, Any]], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        params: Dict[str, Any] = {}
+        if schema is not None:
+            params["schema"] = Schema(schema).pa_schema
+        adf = pa.Table.from_pylist(list(dfs), **params)
+        return ArrowDataFrame(adf)
+
+
 @fugue_annotated_param(AnyDataFrame)
 class _AnyDataFrameParam(DataFrameParam):
     def to_output_df(self, output: AnyDataFrame, schema: Any, ctx: Any) -> DataFrame:
@@ -207,6 +295,15 @@ class LocalDataFrameParam(DataFrameParam):
         )
         return output
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        def _dfs() -> Iterable[DataFrame]:
+            for df in dfs:
+                yield self.to_output_df(df, schema, ctx)
+
+        return LocalDataFrameIterableDataFrame(_dfs(), schema=schema)
+
     def count(self, df: LocalDataFrame) -> int:
         if df.is_bounded:
             return df.count()

fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspq]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspnq]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -503,7 +503,7 @@ class _FuncAsCoTransformer(CoTransformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = schema  # type: ignore
@@ -562,7 +562,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
 
         tr = _FuncAsOutputCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = None  # type: ignore

{fugue-0.9.1.dist-info → fugue-0.9.2.dist-info}/METADATA

@@ -1,13 +1,12 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.1
+Version: 0.9.2
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
 Author-email: hello@fugue.ai
 License: Apache-2.0
 Keywords: distributed spark dask ray sql dsl domain specific language
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -17,67 +16,81 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad >=0.9.7
-Requires-Dist: adagio >=0.2.4
-Provides-Extra: all
-Requires-Dist: qpd >=0.4.4 ; extra == 'all'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'all'
-Requires-Dist: sqlglot ; extra == 'all'
-Requires-Dist: jinja2 ; extra == 'all'
-Requires-Dist: pyspark >=3.1.1 ; extra == 'all'
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'all'
-Requires-Dist: dask-sql ; extra == 'all'
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'all'
-Requires-Dist: notebook ; extra == 'all'
-Requires-Dist: jupyterlab ; extra == 'all'
-Requires-Dist: ipython >=7.10.0 ; extra == 'all'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'all'
-Requires-Dist: pyarrow >=6.0.1 ; extra == 'all'
-Requires-Dist: pandas <2.2,>=2.0.2 ; extra == 'all'
-Requires-Dist: ibis-framework ; extra == 'all'
-Requires-Dist: polars ; extra == 'all'
-Provides-Extra: cpp_sql_parser
-Requires-Dist: fugue-sql-antlr[cpp] >=0.2.0 ; extra == 'cpp_sql_parser'
+License-File: LICENSE
+Requires-Dist: triad>=1.0.0
+Requires-Dist: adagio>=0.2.6
+Provides-Extra: sql
+Requires-Dist: qpd>=0.4.4; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: sqlglot; extra == "sql"
+Requires-Dist: jinja2; extra == "sql"
+Provides-Extra: cpp-sql-parser
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Provides-Extra: dask
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'dask'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'dask'
-Requires-Dist: pandas >=2.0.2 ; extra == 'dask'
-Requires-Dist: dask[dataframe,distributed] >=2024.4.0 ; (python_version >= "3.11.9") and extra == 'dask'
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "dask"
+Requires-Dist: pyarrow>=7.0.0; extra == "dask"
+Requires-Dist: pandas>=2.0.2; extra == "dask"
+Provides-Extra: ray
+Requires-Dist: ray[data]>=2.30.0; extra == "ray"
+Requires-Dist: duckdb>=0.5.0; extra == "ray"
+Requires-Dist: pyarrow>=7.0.0; extra == "ray"
+Requires-Dist: pandas<2.2; extra == "ray"
 Provides-Extra: duckdb
-Requires-Dist: qpd >=0.4.4 ; extra == 'duckdb'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'duckdb'
-Requires-Dist: sqlglot ; extra == 'duckdb'
-Requires-Dist: jinja2 ; extra == 'duckdb'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'duckdb'
-Requires-Dist: numpy ; extra == 'duckdb'
+Requires-Dist: qpd>=0.4.4; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: sqlglot; extra == "duckdb"
+Requires-Dist: jinja2; extra == "duckdb"
+Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
+Requires-Dist: numpy; extra == "duckdb"
+Provides-Extra: polars
+Requires-Dist: polars; extra == "polars"
 Provides-Extra: ibis
-Requires-Dist: qpd >=0.4.4 ; extra == 'ibis'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'ibis'
-Requires-Dist: sqlglot ; extra == 'ibis'
-Requires-Dist: jinja2 ; extra == 'ibis'
-Requires-Dist: ibis-framework ; extra == 'ibis'
-Requires-Dist: pandas <2.2 ; extra == 'ibis'
+Requires-Dist: qpd>=0.4.4; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: sqlglot; extra == "ibis"
+Requires-Dist: jinja2; extra == "ibis"
+Requires-Dist: ibis-framework[pandas]; extra == "ibis"
+Requires-Dist: pandas<2.2; extra == "ibis"
 Provides-Extra: notebook
-Requires-Dist: notebook ; extra == 'notebook'
-Requires-Dist: jupyterlab ; extra == 'notebook'
-Requires-Dist: ipython >=7.10.0 ; extra == 'notebook'
-Provides-Extra: polars
-Requires-Dist: polars ; extra == 'polars'
-Provides-Extra: ray
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'ray'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'ray'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'ray'
-Requires-Dist: pandas <2.2 ; extra == 'ray'
-Provides-Extra: spark
-Requires-Dist: pyspark >=3.1.1 ; extra == 'spark'
-Provides-Extra: sql
-Requires-Dist: qpd >=0.4.4 ; extra == 'sql'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'sql'
-Requires-Dist: sqlglot ; extra == 'sql'
-Requires-Dist: jinja2 ; extra == 'sql'
+Requires-Dist: notebook; extra == "notebook"
+Requires-Dist: jupyterlab; extra == "notebook"
+Requires-Dist: ipython>=7.10.0; extra == "notebook"
+Provides-Extra: all
+Requires-Dist: qpd>=0.4.4; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: sqlglot; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: pyspark>=3.1.1; extra == "all"
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "all"
+Requires-Dist: dask-sql; extra == "all"
+Requires-Dist: ray[data]>=2.30.0; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Requires-Dist: ipython>=7.10.0; extra == "all"
+Requires-Dist: duckdb>=0.5.0; extra == "all"
+Requires-Dist: pyarrow>=6.0.1; extra == "all"
+Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
+Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
+Requires-Dist: polars; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Fugue
 
@@ -355,4 +368,3 @@ View some of our latest conferences presentations and content. For a more comple
 *   [Large Scale Data Validation with Spark and Dask (PyCon US)](https://www.youtube.com/watch?v=2AdvBgjO_3Q)
 *   [FugueSQL - The Enhanced SQL Interface for Pandas, Spark, and Dask DataFrames (PyData Global)](https://www.youtube.com/watch?v=OBpnGYjNBBI)
 *   [Distributed Hybrid Parameter Tuning](https://www.youtube.com/watch?v=_GBjqskD8Qk)
-

{fugue-0.9.1.dist-info → fugue-0.9.2.dist-info}/RECORD

@@ -10,7 +10,7 @@ fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/_utils/display.py,sha256=JV8oDA7efHm1wceZulCBOY5dMvjbWHvIm6ASisKfoWY,3164
 fugue/_utils/exception.py,sha256=SFIjwjV4CIEovp3P9k7ePNOFB12A5D8hDdhtfFUeM5Y,2247
 fugue/_utils/interfaceless.py,sha256=wI0H6L4W_1uQjh9tpjgT9HzN-fbrrtXXHC1x6Q_rrPg,2203
-fugue/_utils/io.py,sha256=adrtj6Dq0ti426DNlkliApbTkp8b3bfBysAiE5MVQVc,9265
+fugue/_utils/io.py,sha256=5twd99LBzHtIMT67il1qwnEUa5n13WZmVKNd1shO4No,9649
 fugue/_utils/misc.py,sha256=_huy0eylmRTEFoReGR2M4rbAI8m79hFcfY5bDceVEXU,887
 fugue/_utils/registry.py,sha256=lrbzTdUEVnW6paBGDj-Yb-aTIbP5mjCqrXuRU9_N6os,316
 fugue/bag/__init__.py,sha256=0Q0_rnrEThrTx2U-1xGNyAg95idp_xcnywymIcW4Xck,46
@@ -31,7 +31,7 @@ fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvr
 fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=hOZF3GmwpxqwqKi9-pEOAPZSW1ZFyB47hLxRrGyOiuM,14855
+fugue/dataframe/function_wrapper.py,sha256=7Sb6XrWTD_swtHJbHDWZRxHvFNWkERynnCDzLM0wSbo,18340
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
 fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
@@ -61,7 +61,7 @@ fugue/extensions/processor/convert.py,sha256=zG0lMtHGwY5TsqK4eplbMdlTg7J_PD3HbI0
 fugue/extensions/processor/processor.py,sha256=czhQlQgMpAXXoLVAX9Q0TFUMYEEhsgufTammxcKSmOY,1665
 fugue/extensions/transformer/__init__.py,sha256=VD6d-8xW1Yl8fUPj43cBWNR9pCOlYD9xWyGIHAlHwvI,456
 fugue/extensions/transformer/constants.py,sha256=76DfpoTOGQ8gp5XtCs_xznfbr_H015-prXpHWSqMNDU,59
-fugue/extensions/transformer/convert.py,sha256=5fhktR2s13ZOpUihpy-gy7Xn2BRN6UoA5uwOzJ6YNOU,23380
+fugue/extensions/transformer/convert.py,sha256=zDDIpZawMnHFarjZNZAyiw1jfyXGuPjnvgQk9jpYLak,23384
 fugue/extensions/transformer/transformer.py,sha256=zhOUgyv5-DPxYd1CP_98WeEw-zUgwknRnPW_6di-q3g,9098
 fugue/rpc/__init__.py,sha256=3GzUl4QZQuCChjD7eaTJW8tnTwfke6ZY9r9g5nCeBZ8,167
 fugue/rpc/base.py,sha256=3Fq5SvwLZqw9NXru3r32WuJKBGFr9bl7nFgy6e9boGo,8470
@@ -82,6 +82,7 @@ fugue/workflow/api.py,sha256=uQoxPSCZ91-ST4vwuPWG7qioRGW4eo-Sgi3DdwtSL4k,12495
 fugue/workflow/input.py,sha256=V_zLDNzndmQuYJAPXtdK4n-vOp7LrimGIf_wQtwf2mc,321
 fugue/workflow/module.py,sha256=ajyqgMwX6hFMZY9xp4Bp1Q-Zdta0p5f_W_n_SNrc4LE,5547
 fugue/workflow/workflow.py,sha256=-SFCXkyxgXbS6DpQGSBox4d3Ws3psIlB6PnraJLSu9Y,88219
+fugue-0.9.2.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
 fugue_contrib/__init__.py,sha256=QJioX-r2AiU7Pvt24M-k2c4vNq29qpK-3WNUde7ucck,222
 fugue_contrib/contrib.py,sha256=3B--6oIVBMZ-GwjIOXwZqYqkloH7Cxfq1I8vkwl2yPk,267
 fugue_contrib/seaborn/__init__.py,sha256=NuVv8EI4Om4gHcHwYO8ddextLQqw24vDj8qJio3E1MU,1405
@@ -89,25 +90,26 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
+fugue_dask/_dask_sql_wrapper.py,sha256=lj38gJIOdoMV9W44gpwzLjUEtPVsQNKjRWuEkfI7-PM,2618
 fugue_dask/_io.py,sha256=pl4F7mbVgP7Rwh1FFG7xfOz2TBZRUj1l3lLvDY4jOf4,6020
-fugue_dask/_utils.py,sha256=1uplEqvpCDZDp2YdwJxa6cuGScpgG9VvN3057J02bys,8956
-fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=60IiwYRBVhN-pX3v6i9BZ8Pa4bcSh5UoklvCScM_XAM,21361
+fugue_dask/_utils.py,sha256=0R0pCh4B47kQsAS_o0QGaosIqVcZnSakm6pfMB7fSXs,9059
+fugue_dask/dataframe.py,sha256=4Dvckpc4mlld2WsEFTTemxoA1zYK8Cn6jMKxUxYQCEE,13491
+fugue_dask/execution_engine.py,sha256=mFN_IurhdBEu8C5OreqpGSRdTbTBqSpzJO2dMQzEF-o,21264
 fugue_dask/registry.py,sha256=jepWKH55VWNIWV3pOF5vpCl2OpO0rI1IULx5GM2Gk6w,2274
 fugue_dask/tester.py,sha256=E7BZjgFpJgrHsLMKzvSO5im5OwocYcratjzulJSQZl0,718
 fugue_duckdb/__init__.py,sha256=ZzhmAWbROR1YL9Kmlt7OlwkgPZzFhsSdwLV2pFmAqGI,268
 fugue_duckdb/_io.py,sha256=vnd8m8C6XeMCBJBbAdA5h695NMfsduQrvONyS0HcEFA,8475
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
-fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
-fugue_duckdb/execution_engine.py,sha256=IZDmSAtOMJGvulTStxjTmsqJyI5QRNyxBgSMlFMSrBI,20389
+fugue_duckdb/dataframe.py,sha256=LAPoPOad9hgGhjyhlMGMfrnhkyBKe06Xzn6eP1hkl-w,8504
+fugue_duckdb/execution_engine.py,sha256=3f5hbWcX1y9mAtfFixrri-snYxVIQAf4HOgo9fHbDwQ,20385
 fugue_duckdb/registry.py,sha256=9_41KO42kXqcjF4yParQ5JGyg5TckcbhH-Q2IlGpSho,3987
 fugue_duckdb/tester.py,sha256=MzTkv3sdOwOjI59LRrSGGl4w59Njv3OArTU5kSRL-P0,1526
 fugue_ibis/__init__.py,sha256=z7TkK7M2_0p9XO6jQATNDgT0aHXn5k69Ttz2ga-eQG8,190
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
 fugue_ibis/dataframe.py,sha256=k4Q6qBLBIADF5YhbvaDplXO7OkMZSHuf_Wg5o-AusEI,7796
-fugue_ibis/execution_engine.py,sha256=5I-ou5xPdomVu-srdvidvP8f7wDYbGrCV_lGffZa_ac,18679
+fugue_ibis/execution_engine.py,sha256=jRnp1m1wuTicS29A-WA043f8QwdoK8b9rwPXvTkm8r8,18751
 fugue_notebook/__init__.py,sha256=9r_-2uxu1lBeZ8GgpYCKom_OZy2soIOYZajg7JDO-HY,4326
 fugue_notebook/env.py,sha256=TYiTxYPFi-BVJJY49jDsvw9mddhK8WrifeRxBke30I8,4773
 fugue_notebook/nbextension/README.md,sha256=QLnr957YeGfwzy2r4c4qbZPaXyCbyGrKPvcqSBQYSnU,123
@@ -127,7 +129,7 @@ fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
-fugue_ray/_utils/io.py,sha256=3hFNDeBuh4bfCud40ZsGrGZLSvCSuxL_1VlqCTnn6RA,9794
+fugue_ray/_utils/io.py,sha256=y7TFtdKcqDtMw2e1u012rT8Ay0ChvAT2uJL4pCypABM,9963
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
@@ -135,7 +137,7 @@ fugue_spark/execution_engine.py,sha256=YBMtNxCvpy77xICFSg9PHMa6feNoYhWEZe8MmxznX
 fugue_spark/registry.py,sha256=_NmiV2cOooYK0YmqATEnNkPEMT9suUMtuecw2NNbIIk,4530
 fugue_spark/tester.py,sha256=VX003yGNlBukaZTQSN-w7XvgSk4rqxrWQIzno0dWrXg,2481
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
+fugue_spark/_utils/convert.py,sha256=J3HtbuzomTYTN6A11iuvsC1h2C7o3fQBW5U360xGDhE,10234
 fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
 fugue_spark/_utils/misc.py,sha256=9LsbBp6nOEhqXFLr8oWTc3VKzKk-vuVyixlRoquGnEs,858
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
@@ -143,14 +145,13 @@ fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=cOkZG6w1RHhWWxtjQhZClZQaGT6haNd576BoUmNC_cA,77960
+fugue_test/builtin_suite.py,sha256=jP3xiq2vRZNNGzoSRjcUfrUk8NVg31SU0kpJaEvP25E,79400
 fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
-fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
+fugue_test/execution_suite.py,sha256=wUiGdb8wLRd13JXo7Lo19vPOLh7t1C-L2NPLeU0k-uE,48736
 fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
-fugue_version/__init__.py,sha256=UwJXM8JY2T3tE2id0K2k_lEaVThbRTrGO1mNibyzIz8,22
-fugue-0.9.1.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.9.1.dist-info/METADATA,sha256=zu44QGPIwk28QyKe9H4Si2ANByy1sJ9cmauNrhCg4bc,18380
-fugue-0.9.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fugue-0.9.1.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
-fugue-0.9.1.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.9.1.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
+fugue-0.9.2.dist-info/METADATA,sha256=zmzlL5Fw-t0qTfcRb5jCeJb_IUJ7HVYRT0SpgcN3ncI,18558
+fugue-0.9.2.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fugue-0.9.2.dist-info/entry_points.txt,sha256=2Vxp1qew_tswacA8m0RzIliLlFOQMlzezvSXPugM_KA,295
+fugue-0.9.2.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.2.dist-info/RECORD,,

{fugue-0.9.1.dist-info → fugue-0.9.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fugue-0.9.2.dist-info/entry_points.txt

@@ -0,0 +1,11 @@
+[fugue.plugins]
+dask = fugue_dask.registry[dask]
+duckdb = fugue_duckdb.registry[duckdb]
+ibis = fugue_ibis[ibis]
+polars = fugue_polars.registry[polars]
+ray = fugue_ray.registry[ray]
+spark = fugue_spark.registry[spark]
+
+[pytest11]
+fugue_test = fugue_test
+fugue_test_fixtures = fugue_test.fixtures

fugue_dask/_dask_sql_wrapper.py

@@ -0,0 +1,76 @@
+from typing import Any, Optional
+
+import dask.dataframe as dd
+
+try:
+    from dask.dataframe.dask_expr.io.parquet import ReadParquet
+
+    HAS_DASK_EXPR = True  # newer dask
+except ImportError:  # pragma: no cover
+    HAS_DASK_EXPR = False  # older dask
+
+if not HAS_DASK_EXPR:  # pragma: no cover
+    try:
+        from dask_sql import Context as ContextWrapper  # pylint: disable-all
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+else:
+    from triad.utils.assertion import assert_or_throw
+
+    try:
+        from dask_sql import Context
+        from dask_sql.datacontainer import Statistics
+        from dask_sql.input_utils import InputUtil
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+
+    class ContextWrapper(Context):  # type: ignore
+        def create_table(
+            self,
+            table_name: str,
+            input_table: dd.DataFrame,
+            format: Optional[str] = None,  # noqa
+            persist: bool = False,
+            schema_name: Optional[str] = None,
+            statistics: Optional[Statistics] = None,
+            gpu: bool = False,
+            **kwargs: Any,
+        ) -> None:  # pragma: no cover
+            assert_or_throw(
+                isinstance(input_table, dd.DataFrame),
+                lambda: ValueError(
+                    f"input_table must be a dask dataframe, but got {type(input_table)}"
+                ),
+            )
+            assert_or_throw(
+                dd._dask_expr_enabled(), lambda: ValueError("Dask expr must be enabled")
+            )
+            schema_name = schema_name or self.schema_name
+
+            dc = InputUtil.to_dc(
+                input_table,
+                table_name=table_name,
+                format=format,
+                persist=persist,
+                gpu=gpu,
+                **kwargs,
+            )
+
+            dask_filepath = None
+            operations = input_table.find_operations(ReadParquet)
+            for op in operations:
+                dask_filepath = op._args[0]
+
+            dc.filepath = dask_filepath
+            self.schema[schema_name].filepaths[table_name.lower()] = dask_filepath
+
+            if not statistics:
+                statistics = Statistics(float("nan"))
+            dc.statistics = statistics
+
+            self.schema[schema_name].tables[table_name.lower()] = dc
+            self.schema[schema_name].statistics[table_name.lower()] = statistics

fugue_dask/_utils.py

@@ -5,7 +5,7 @@ import dask.dataframe as dd
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from dask.dataframe.core import DataFrame
+from dask.dataframe import DataFrame
 from dask.delayed import delayed
 from dask.distributed import Client, get_client
 from triad.utils.pandas_like import PD_UTILS, PandasLikeUtils
@@ -149,7 +149,7 @@ def _add_hash_index(
     if len(cols) == 0:
         cols = list(df.columns)
 
-    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:
+    def _add_hash(df: pd.DataFrame) -> pd.DataFrame:  # pragma: no cover
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
         return df.assign(
@@ -171,7 +171,7 @@ def _add_hash_index(
 
 def _add_random_index(
     df: dd.DataFrame, num: int, seed: Any = None
-) -> Tuple[dd.DataFrame, int]:
+) -> Tuple[dd.DataFrame, int]:  # pragma: no cover
     def _add_rand(df: pd.DataFrame) -> pd.DataFrame:
         if len(df) == 0:
             return df.assign(**{_FUGUE_DASK_TEMP_IDX_COLUMN: pd.Series(dtype=int)})
@@ -189,7 +189,9 @@ def _add_random_index(
 
 
 def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
-    def _get_info(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _get_info(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return pd.DataFrame(dict(no=[partition_info["number"]], ct=[len(df)]))
 
     pinfo = (
@@ -200,7 +202,9 @@ def _add_continuous_index(df: dd.DataFrame) -> Tuple[dd.DataFrame, int]:
     counts = pinfo.sort_values("no").ct.cumsum().tolist()
     starts = [0] + counts[0:-1]
 
-    def _add_index(df: pd.DataFrame, partition_info: Any) -> pd.DataFrame:
+    def _add_index(
+        df: pd.DataFrame, partition_info: Any
+    ) -> pd.DataFrame:  # pragma: no cover
         return df.assign(
             **{
                 _FUGUE_DASK_TEMP_IDX_COLUMN: np.arange(len(df))

fugue_dask/dataframe.py

@@ -379,7 +379,7 @@ def _to_array_chunks(
     assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
     _df = df if columns is None or len(columns) == 0 else df[columns]
 
-    def _to_list(pdf: pd.DataFrame) -> List[Any]:
+    def _to_list(pdf: pd.DataFrame) -> List[Any]:  # pragma: no cover
         return list(
             PD_UTILS.as_array_iterable(
                 pdf,

fugue_dask/execution_engine.py

@@ -9,9 +9,10 @@ from triad.collections import Schema
 from triad.collections.dict import IndexedOrderedDict, ParamDict
 from triad.utils.assertion import assert_or_throw
 from triad.utils.hash import to_uuid
+from triad.utils.io import makedirs
 from triad.utils.pandas_like import PandasUtils
 from triad.utils.threading import RunOnce
-from triad.utils.io import makedirs
+
 from fugue import StructuredRawSQL
 from fugue.collections.partition import (
     PartitionCursor,
@@ -61,14 +62,9 @@ class DaskSQLEngine(SQLEngine):
         return True
 
     def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        try:
-            from dask_sql import Context
-        except ImportError:  # pragma: no cover
-            raise ImportError(
-                "dask-sql is not installed. "
-                "Please install it with `pip install dask-sql`"
-            )
-        ctx = Context()
+        from ._dask_sql_wrapper import ContextWrapper
+
+        ctx = ContextWrapper()
         _dfs: Dict[str, dd.DataFrame] = {k: self._to_safe_df(v) for k, v in dfs.items()}
         sql = statement.construct(dialect=self.dialect, log=self.log)
         res = ctx.sql(
@@ -102,7 +98,8 @@ class DaskMapEngine(MapEngine):
         partition_spec: PartitionSpec,
         on_init: Optional[Callable[[int, DataFrame], Any]] = None,
         map_func_format_hint: Optional[str] = None,
-    ) -> DataFrame:
+    ) -> DataFrame:  # pragma: no cover
+        # It is well tested but not captured by coverage
         presort = partition_spec.get_sorts(
             df.schema, with_partition_keys=partition_spec.algo == "coarse"
         )
@@ -475,7 +472,7 @@ class DaskExecutionEngine(ExecutionEngine):
         # Use presort over partition_spec.presort if possible
         _presort: IndexedOrderedDict = presort or partition_spec.presort
 
-        def _partition_take(partition, n, presort):
+        def _partition_take(partition, n, presort):  # pragma: no cover
             assert_or_throw(
                 partition.shape[1] == len(meta),
                 FugueBug("hitting the dask bug where partition keys are lost"),

fugue_duckdb/dataframe.py

@@ -165,7 +165,7 @@ def _duck_as_local(df: DuckDBPyRelation) -> DuckDBPyRelation:
 
 @as_arrow.candidate(lambda df: isinstance(df, DuckDBPyRelation))
 def _duck_as_arrow(df: DuckDBPyRelation) -> pa.Table:
-    _df = df.arrow()
+    _df = df.fetch_arrow_table()
     _df = replace_types_in_table(_df, LARGE_TYPES_REPLACEMENT, recursive=True)
     return _df
 
@@ -216,7 +216,7 @@ def _drop_duckdb_columns(df: DuckDBPyRelation, columns: List[str]) -> DuckDBPyRe
 def _duck_as_array(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> List[Any]:
-    return pa_table_as_array(df.arrow(), columns=columns)
+    return pa_table_as_array(df.fetch_arrow_table(), columns=columns)
 
 
 @as_array_iterable.candidate(
@@ -225,14 +225,14 @@ def _duck_as_array(
 def _duck_as_array_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> Iterable[Any]:
-    yield from pa_table_as_array_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_array_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
 def _duck_as_dicts(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> List[Dict[str, Any]]:
-    return pa_table_as_dicts(df.arrow(), columns=columns)
+    return pa_table_as_dicts(df.fetch_arrow_table(), columns=columns)
 
 
 @as_dict_iterable.candidate(
@@ -241,7 +241,7 @@ def _duck_as_dicts(
 def _duck_as_dict_iterable(
     df: DuckDBPyRelation, columns: Optional[List[str]] = None
 ) -> Iterable[Dict[str, Any]]:
-    yield from pa_table_as_dict_iterable(df.arrow(), columns=columns)
+    yield from pa_table_as_dict_iterable(df.fetch_arrow_table(), columns=columns)
 
 
 def _assert_no_missing(df: DuckDBPyRelation, columns: Iterable[Any]) -> None:

fugue_duckdb/execution_engine.py

@@ -108,7 +108,7 @@ class DuckDBEngine(SQLEngine):
         try:
             for k, v in dfs.items():
                 duckdb.from_arrow(v.as_arrow(), connection=conn).create_view(k)
-            return ArrowDataFrame(_duck_as_arrow(conn.execute(statement)))
+            return ArrowDataFrame(_duck_as_arrow(conn.sql(statement)))
         finally:
             conn.close()
 

fugue_ibis/execution_engine.py

@@ -92,7 +92,8 @@ class IbisSQLEngine(SQLEngine):
         _df2 = self.to_df(df2)
         key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
         on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
-        if ibis.__version__ < "6":  # pragma: no cover
+        version = int(ibis.__version__.split(".")[0])
+        if version < 6:  # pragma: no cover
             suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
         else:
             # breaking change in ibis 6.0
@@ -113,7 +114,7 @@ class IbisSQLEngine(SQLEngine):
                     cols.append(
                         ibis.coalesce(tb[k], tb[k + _JOIN_RIGHT_SUFFIX]).name(k)
                     )
-            tb = tb[cols]
+            tb = tb.select(*cols)
         elif how.lower() in ["semi", "left_semi"]:
             tb = _df1.native.semi_join(_df2.native, on_fields, **suffixes)
         elif how.lower() in ["anti", "left_anti"]:
@@ -153,7 +154,7 @@ class IbisSQLEngine(SQLEngine):
         self,
         df: DataFrame,
         how: str = "any",
-        thresh: int = None,
+        thresh: Optional[int] = None,
         subset: Optional[List[str]] = None,
     ) -> DataFrame:
         schema = df.schema
@@ -161,7 +162,7 @@ class IbisSQLEngine(SQLEngine):
             schema = schema.extract(subset)
         _df = self.to_df(df)
         if thresh is None:
-            tb = _df.native.dropna(subset=subset, how=how)
+            tb = _df.native.drop_null(subset, how=how)
             return self.to_df(tb, df.schema)
         assert_or_throw(
             how == "any", ValueError("when thresh is set, how must be 'any'")
@@ -204,7 +205,7 @@ class IbisSQLEngine(SQLEngine):
             ibis.coalesce(tb[f], ibis.literal(vd[f])).name(f) if f in names else tb[f]
             for f in df.columns
         ]
-        return self.to_df(tb[cols], schema=df.schema)
+        return self.to_df(tb.select(cols), schema=df.schema)
 
     def take(
         self,
@@ -241,7 +242,7 @@ class IbisSQLEngine(SQLEngine):
                 f") WHERE __fugue_take_param<={n}"
             )
             tb = self.query_to_table(sql, {tbn: idf})
-            return self.to_df(tb[df.columns], schema=df.schema)
+            return self.to_df(tb.select(*df.columns), schema=df.schema)
 
         sorts: List[str] = []
         for k, v in _presort.items():

fugue_ray/_utils/io.py

@@ -7,7 +7,7 @@ import ray.data as rd
 from packaging import version
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
-from ray.data.datasource import FileExtensionFilter
+
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
@@ -21,6 +21,27 @@ from fugue_ray.dataframe import RayDataFrame
 
 from .._constants import RAY_VERSION
 
+try:
+    from ray.data.datasource import FileExtensionFilter
+
+    class _FileFiler(FileExtensionFilter):  # pragma: no cover
+        def __init__(
+            self, file_extensions: Union[str, List[str]], exclude: Iterable[str]
+        ):
+            super().__init__(file_extensions, allow_if_no_extension=True)
+            self._exclude = set(exclude)
+
+        def _is_valid(self, path: str) -> bool:
+            return pathlib.Path(
+                path
+            ).name not in self._exclude and self._file_has_extension(path)
+
+        def __call__(self, paths: List[str]) -> List[str]:
+            return [path for path in paths if self._is_valid(path)]
+
+except ImportError:  # pragma: no cover
+    pass  # ray >=2.10
+
 
 class RayIO(object):
     def __init__(self, engine: ExecutionEngine):
@@ -53,7 +74,7 @@ class RayIO(object):
             len(fmts) == 1, NotImplementedError("can't support multiple formats")
         )
         fmt = fmts[0]
-        files = [f.path for f in fp]
+        files = [f.as_dir_path() if f.is_dir else f.path for f in fp]
         return self._loads[fmt](files, columns, **kwargs)
 
     def save_df(
@@ -248,17 +269,3 @@ class RayIO(object):
 
     def _remote_args(self) -> Dict[str, Any]:
         return {"num_cpus": 1}
-
-
-class _FileFiler(FileExtensionFilter):  # pragma: no cover
-    def __init__(self, file_extensions: Union[str, List[str]], exclude: Iterable[str]):
-        super().__init__(file_extensions, allow_if_no_extension=True)
-        self._exclude = set(exclude)
-
-    def _is_valid(self, path: str) -> bool:
-        return pathlib.Path(
-            path
-        ).name not in self._exclude and self._file_has_extension(path)
-
-    def __call__(self, paths: List[str]) -> List[str]:
-        return [path for path in paths if self._is_valid(path)]

fugue_spark/_utils/convert.py

@@ -174,20 +174,26 @@ def pd_to_spark_df(
 
 
 def to_pandas(df: ps.DataFrame) -> pd.DataFrame:
-    if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
-        isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
-        for x in df.schema.fields
-    ):
-        return df.toPandas()
-    else:  # pragma: no cover
+    def _to_df() -> pd.DataFrame:
+        if version.parse(pd.__version__) < version.parse("2.0.0") or not any(
+            isinstance(x.dataType, (pt.TimestampType, TimestampNTZType))
+            for x in df.schema.fields
+        ):
+            return df.toPandas()
+        else:  # pragma: no cover
+
+            def serialize(dfs):
+                for df in dfs:
+                    data = pickle.dumps(df)
+                    yield pd.DataFrame([[data]], columns=["data"])
 
-        def serialize(dfs):
-            for df in dfs:
-                data = pickle.dumps(df)
-                yield pd.DataFrame([[data]], columns=["data"])
+            sdf = df.mapInPandas(serialize, schema="data binary")
+            return pd.concat(pickle.loads(x.data) for x in sdf.collect())
 
-        sdf = df.mapInPandas(serialize, schema="data binary")
-        return pd.concat(pickle.loads(x.data) for x in sdf.collect())
+    pdf = _to_df()
+    if hasattr(pdf, "attrs") and "metrics" in pdf.attrs:  # pragma: no cover
+        del pdf.attrs["metrics"]
+    return pdf
 
 
 def to_arrow(df: ps.DataFrame) -> pa.Table:

fugue_test/builtin_suite.py

@@ -486,6 +486,25 @@ class BuiltInTests(object):
                 dag.df([], "a:int,b:int").assert_eq(b)
             dag.run(self.engine)
 
+        def _test_transform_row_wise(self):  # pragma: no cover
+            # TODO: currently disabled because we don't support Dict[str, Any]
+            # as dataframe input
+            def t1(row: Dict[str, Any]) -> Dict[str, Any]:
+                row["b"] = 1
+                return row
+
+            def t2(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
+                return rows[0]
+
+            with fa.engine_context(self.engine):
+                a = pd.DataFrame([[3, 4], [1, 2], [3, 5]], columns=["a", "b"])
+                b = fa.transform(a, t1, schema="*")
+                assert sorted(fa.as_array(b)) == [[1, 1], [3, 1], [3, 1]]
+                b = fa.transform(
+                    a, t2, schema="*", partition={"by": "a", "presort": "b"}
+                )
+                assert sorted(fa.as_array(b)) == [[1, 2], [3, 4]]
+
         def test_transform_binary(self):
             with FugueWorkflow() as dag:
                 a = dag.df([[1, pickle.dumps([0, "a"])]], "a:int,b:bytes")
@@ -548,6 +567,8 @@ class BuiltInTests(object):
                 e = dag.df([[1, 2, 1, 10]], "a:int,ct1:int,ct2:int,x:int")
                 e.assert_eq(c)
 
+                a.zip(b).transform(mock_co_tf1_d, params=dict(p=10)).assert_eq(e)
+
                 # interfaceless
                 c = dag.transform(
                     a.zip(b),
@@ -676,6 +697,13 @@ class BuiltInTests(object):
                 incr()
                 yield pa.Table.from_pandas(df)
 
+            def t11(row: list[dict[str, Any]]) -> dict[str, Any]:
+                incr()
+                return row[0]
+
+            def t12(row: list[dict[str, Any]]) -> None:
+                incr()
+
             with FugueWorkflow() as dag:
                 a = dag.df([[1, 2], [3, 4]], "a:double,b:int")
                 a.out_transform(t1)  # +2
@@ -688,6 +716,8 @@ class BuiltInTests(object):
                 a.out_transform(t8, ignore_errors=[NotImplementedError])  # +1
                 a.out_transform(t9)  # +1
                 a.out_transform(t10)  # +1
+                a.out_transform(t11)  # +2
+                a.out_transform(t12)  # +2
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t2))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t3))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t4))
@@ -695,7 +725,7 @@ class BuiltInTests(object):
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(T7))
             dag.run(self.engine)
 
-            assert 13 <= incr()
+            assert 17 <= incr()
 
         def test_out_cotransform(self):  # noqa: C901
             tmpdir = str(self.tmpdir)
@@ -2001,6 +2031,13 @@ def mock_co_tf1(
     return [[df1[0]["a"], len(df1), len(df2), p]]
 
 
+@cotransformer(lambda dfs, **kwargs: "a:int,ct1:int,ct2:int,x:int")
+def mock_co_tf1_d(
+    df1: List[Dict[str, Any]], df2: List[List[Any]], p=1
+) -> Dict[str, Any]:
+    return dict(a=df1[0]["a"], ct1=len(df1), ct2=len(df2), x=p)
+
+
 def mock_co_tf2(dfs: DataFrames, p=1) -> List[List[Any]]:
     return [[dfs[0].peek_dict()["a"], dfs[0].count(), dfs[1].count(), p]]
 

fugue_test/execution_suite.py

@@ -9,6 +9,7 @@ except ImportError:  # pragma: no cover
 import copy
 import os
 import pickle
+import sys
 from datetime import datetime
 
 import pandas as pd
@@ -1194,6 +1195,7 @@ class ExecutionEngineTests(object):
             )
             self.df_eq(c, [[1.1, 6.1], [7.1, 2.1]], "a:double,c:double", throw=True)
 
+        @pytest.mark.skipif(sys.platform == "win32", reason="skip on Windows")
         def test_load_csv_folder(self):
             native = NativeExecutionEngine()
             a = ArrayDataFrame([[6.1, 1.1]], "c:double,a:double")

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.1"
+__version__ = "0.9.2"

fugue-0.9.1.dist-info/entry_points.txt

@@ -1,12 +0,0 @@
-[fugue.plugins]
-dask = fugue_dask.registry [dask]
-duckdb = fugue_duckdb.registry [duckdb]
-ibis = fugue_ibis [ibis]
-polars = fugue_polars.registry [polars]
-ray = fugue_ray.registry [ray]
-spark = fugue_spark.registry [spark]
-
-[pytest11]
-fugue_test = fugue_test
-fugue_test_fixtures = fugue_test.fixtures
-